From 08ad80b112ffde6f1c6aaf747483c31018567931 Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Mon, 9 Dec 2024 09:34:04 +0000 Subject: [PATCH 01/41] refactor sls client manager --- core/CMakeLists.txt | 2 +- core/app_config/AppConfig.h | 1 + core/common/EncodingUtil.cpp | 75 + core/common/EncodingUtil.h | 25 + core/common/EndpointUtil.cpp | 30 +- core/common/EndpointUtil.h | 2 +- core/common/HashUtil.cpp | 18 + core/common/HashUtil.h | 1 + core/common/LogtailCommonFlags.cpp | 3 - core/common/LogtailCommonFlags.h | 3 - core/common/common.cmake | 2 +- core/common/http/AsynCurlRunner.cpp | 110 +- core/common/http/AsynCurlRunner.h | 2 - core/common/http/Constant.cpp | 34 + core/common/http/Constant.h | 36 + core/common/http/Curl.cpp | 170 +- core/common/http/Curl.h | 4 + core/common/http/HttpRequest.cpp | 38 +- core/common/http/HttpRequest.h | 15 +- core/common/http/HttpResponse.h | 13 + .../common_provider/CommonConfigProvider.cpp | 53 +- .../LegacyCommonConfigProvider.cpp | 127 +- core/file_server/reader/LogFileReader.cpp | 5 +- core/monitor/Monitor.cpp | 2 +- core/monitor/profile_sender/ProfileSender.cpp | 29 +- core/pipeline/queue/SLSSenderQueueItem.h | 2 +- core/plugin/flusher/sls/DiskBufferWriter.cpp | 337 ++-- core/plugin/flusher/sls/DiskBufferWriter.h | 29 +- core/{sdk => plugin/flusher/sls}/Exception.h | 0 core/plugin/flusher/sls/FlusherSLS.cpp | 340 ++-- core/plugin/flusher/sls/FlusherSLS.h | 31 +- core/plugin/flusher/sls/SLSClientManager.cpp | 1731 +++++++++++------ core/plugin/flusher/sls/SLSClientManager.h | 349 +++- core/plugin/flusher/sls/SLSConstant.cpp | 96 + core/plugin/flusher/sls/SLSConstant.h | 99 + core/plugin/flusher/sls/SLSResponse.cpp | 85 +- core/plugin/flusher/sls/SLSResponse.h | 1 + core/plugin/flusher/sls/SLSUtil.cpp | 307 +++ core/plugin/flusher/sls/SLSUtil.h | 82 + core/plugin/flusher/sls/SendResult.cpp | 16 +- .../processor/ProcessorDesensitizeNative.cpp | 4 +- core/prometheus/PrometheusInputRunner.cpp | 51 +- core/prometheus/PrometheusInputRunner.h | 6 +- core/prometheus/schedulers/ScrapeConfig.cpp | 4 +- .../prometheus/schedulers/ScrapeScheduler.cpp | 4 +- .../schedulers/TargetSubscriberScheduler.cpp | 6 +- core/protobuf/sls/logtail_buffer_meta.proto | 11 +- core/runner/FlusherRunner.cpp | 11 - core/runner/sink/http/HttpSink.cpp | 7 +- core/sdk/Client.cpp | 455 ----- core/sdk/Client.h | 223 --- core/sdk/Common.cpp | 883 --------- core/sdk/Common.h | 323 --- core/sdk/CurlImp.cpp | 192 -- core/sdk/CurlImp.h | 42 - core/sdk/Result.cpp | 128 -- core/sdk/Result.h | 47 - core/unittest/CMakeLists.txt | 1 - core/unittest/flusher/CMakeLists.txt | 4 + core/unittest/flusher/FlusherSLSUnittest.cpp | 113 +- .../flusher/SLSClientManagerUnittest.cpp | 1380 +++++++++++++ core/unittest/sdk/CMakeLists.txt | 19 - core/unittest/sdk/SDKCommonUnittest.cpp | 290 --- .../serializer/SLSSerializerUnittest.cpp | 8 +- 64 files changed, 4510 insertions(+), 4007 deletions(-) create mode 100644 core/common/EncodingUtil.cpp create mode 100644 core/common/EncodingUtil.h create mode 100644 core/common/http/Constant.cpp create mode 100644 core/common/http/Constant.h rename core/{sdk => plugin/flusher/sls}/Exception.h (100%) create mode 100644 core/plugin/flusher/sls/SLSConstant.cpp create mode 100644 core/plugin/flusher/sls/SLSConstant.h create mode 100644 core/plugin/flusher/sls/SLSUtil.cpp create mode 100644 core/plugin/flusher/sls/SLSUtil.h delete mode 100644 core/sdk/Client.cpp delete mode 100644 core/sdk/Client.h delete mode 100644 core/sdk/Common.cpp delete mode 100644 core/sdk/Common.h delete mode 100644 core/sdk/CurlImp.cpp delete mode 100644 core/sdk/CurlImp.h delete mode 100644 core/sdk/Result.cpp delete mode 100644 core/sdk/Result.h create mode 100644 core/unittest/flusher/SLSClientManagerUnittest.cpp delete mode 100644 core/unittest/sdk/CMakeLists.txt delete mode 100644 core/unittest/sdk/SDKCommonUnittest.cpp diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 3dd6af2c93..546eee8ba3 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -123,7 +123,7 @@ set(SUB_DIRECTORIES_LIST file_server file_server/event file_server/event_handler file_server/event_listener file_server/reader file_server/polling prometheus prometheus/labels prometheus/schedulers prometheus/async ebpf ebpf/observer ebpf/security ebpf/handler - parser sls_control sdk + parser ) if (LINUX) if (ENABLE_ENTERPRISE) diff --git a/core/app_config/AppConfig.h b/core/app_config/AppConfig.h index a1f0af7ec5..7499df5e4e 100644 --- a/core/app_config/AppConfig.h +++ b/core/app_config/AppConfig.h @@ -519,6 +519,7 @@ class AppConfig { friend class InputPrometheusUnittest; friend class InputContainerStdioUnittest; friend class BatcherUnittest; + friend class SLSClientManagerUnittest; #endif }; diff --git a/core/common/EncodingUtil.cpp b/core/common/EncodingUtil.cpp new file mode 100644 index 0000000000..32c8d836c7 --- /dev/null +++ b/core/common/EncodingUtil.cpp @@ -0,0 +1,75 @@ +// Copyright 2024 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "common/EncodingUtil.h" + +#include + +using namespace std; + +namespace logtail { + +static void Base64Encoding(istream& is, ostream& os, char makeupChar, const char* alphabet) { + int out[4]; + int remain = 0; + while (!is.eof()) { + int byte1 = is.get(); + if (byte1 < 0) { + break; + } + int byte2 = is.get(); + int byte3; + if (byte2 < 0) { + byte2 = 0; + byte3 = 0; + remain = 1; + } else { + byte3 = is.get(); + if (byte3 < 0) { + byte3 = 0; + remain = 2; + } + } + out[0] = static_cast(byte1) >> 2; + out[1] = ((byte1 & 0x03) << 4) + (static_cast(byte2) >> 4); + out[2] = ((byte2 & 0x0F) << 2) + (static_cast(byte3) >> 6); + out[3] = byte3 & 0x3F; + + if (remain == 1) { + os.put(out[0] = alphabet[out[0]]); + os.put(out[1] = alphabet[out[1]]); + os.put(makeupChar); + os.put(makeupChar); + } else if (remain == 2) { + os.put(out[0] = alphabet[out[0]]); + os.put(out[1] = alphabet[out[1]]); + os.put(out[2] = alphabet[out[2]]); + os.put(makeupChar); + } else { + os.put(out[0] = alphabet[out[0]]); + os.put(out[1] = alphabet[out[1]]); + os.put(out[2] = alphabet[out[2]]); + os.put(out[3] = alphabet[out[3]]); + } + } +} + +string Base64Enconde(const string& message) { + istringstream iss(message); + ostringstream oss; + Base64Encoding(iss, oss, '=', "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"); + return oss.str(); +} + +} // namespace logtail diff --git a/core/common/EncodingUtil.h b/core/common/EncodingUtil.h new file mode 100644 index 0000000000..f467818627 --- /dev/null +++ b/core/common/EncodingUtil.h @@ -0,0 +1,25 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +namespace logtail { + +std::string Base64Enconde(const std::string& message); + +} // namespace logtail diff --git a/core/common/EndpointUtil.cpp b/core/common/EndpointUtil.cpp index 67e4b34333..16e7839160 100644 --- a/core/common/EndpointUtil.cpp +++ b/core/common/EndpointUtil.cpp @@ -25,28 +25,20 @@ namespace logtail { bool IsHttpsEndpoint(const string& endpoint) { return endpoint.find("https://") == 0; } - -string StandardizeEndpoint(const string& endpoint, const string& defaultEndpoint) { - string res = endpoint; - if (endpoint.find("https://") == 0) { - if (endpoint.size() < string("https://x").size()) { - LOG_WARNING(sLogger, ("invalid endpoint", endpoint)("use default instead", defaultEndpoint)); - return defaultEndpoint; - } - } else if (endpoint.find("http://") == 0) { - if (endpoint.size() < string("http://x").size()) { - LOG_WARNING(sLogger, ("invalid endpoint", endpoint)("use default instead", defaultEndpoint)); - return defaultEndpoint; - } + +string StandardizeEndpoint(const string& endpoint) { + auto bpos = endpoint.find("://"); + if (bpos == string::npos) { + bpos = 0; } else { - res = string("http://") + endpoint; - LOG_DEBUG(sLogger, ("add default protocol for endpoint, old", endpoint)("new", res)); + bpos += strlen("://"); } - // trim the last '/' - if (res[res.size() - 1] == '/') { - return res.substr(0, res.size() - 1); + + auto epos = endpoint.find("/", bpos); + if (epos == string::npos) { + epos = endpoint.length(); } - return res; + return endpoint.substr(bpos, epos - bpos); } string GetHostFromEndpoint(const std::string& endpoint) { diff --git a/core/common/EndpointUtil.h b/core/common/EndpointUtil.h index b678dc6792..1791706c2a 100644 --- a/core/common/EndpointUtil.h +++ b/core/common/EndpointUtil.h @@ -22,7 +22,7 @@ namespace logtail { bool IsHttpsEndpoint(const std::string& endpoint); -std::string StandardizeEndpoint(const std::string& endpoint, const std::string& defaultEndpoint); +std::string StandardizeEndpoint(const std::string& endpoint); std::string GetHostFromEndpoint(const std::string& endpoint); diff --git a/core/common/HashUtil.cpp b/core/common/HashUtil.cpp index 3b58726971..206c872f25 100644 --- a/core/common/HashUtil.cpp +++ b/core/common/HashUtil.cpp @@ -24,6 +24,8 @@ namespace logtail { +static constexpr uint32_t MD5_BYTES = 16; + /////////////////////////////////////////////// MACRO ////////////////////////////////////////////////// #define SHIFT_LEFT(a, b) ((a) << (b) | (a) >> (32 - b)) @@ -310,6 +312,22 @@ void DoMd5(const uint8_t* poolIn, const uint64_t inputBytesNum, uint8_t md5[16]) } } /// DoMd5 +static std::string HexToString(const uint8_t md5[16]) { + static const char* table = "0123456789ABCDEF"; + std::string ss(32, 'a'); + for (int i = 0; i < 16; ++i) { + ss[i * 2] = table[md5[i] >> 4]; + ss[i * 2 + 1] = table[md5[i] & 0x0F]; + } + return ss; +} + +std::string CalcMD5(const std::string& message) { + uint8_t md5[MD5_BYTES]; + DoMd5((const uint8_t*)message.data(), message.length(), md5); + return HexToString(md5); +} + bool SignatureToHash(const std::string& signature, uint64_t& sigHash, uint32_t& sigSize) { sigSize = (uint32_t)signature.size(); sigHash = (uint64_t)HashSignatureString(signature.c_str(), signature.size()); diff --git a/core/common/HashUtil.h b/core/common/HashUtil.h index 1612ffeee8..5bd7204c74 100644 --- a/core/common/HashUtil.h +++ b/core/common/HashUtil.h @@ -24,6 +24,7 @@ namespace logtail { // Hash(string(@poolIn, @inputBytesNum)) => @md5. // TODO: Same implementation in sdk module, merge them. void DoMd5(const uint8_t* poolIn, const uint64_t inputBytesNum, uint8_t md5[16]); +std::string CalcMD5(const std::string& message); bool SignatureToHash(const std::string& signature, uint64_t& sigHash, uint32_t& sigSize); bool CheckAndUpdateSignature(const std::string& signature, uint64_t& sigHash, uint32_t& sigSize); diff --git a/core/common/LogtailCommonFlags.cpp b/core/common/LogtailCommonFlags.cpp index 1d70138f27..ca328dce6c 100644 --- a/core/common/LogtailCommonFlags.cpp +++ b/core/common/LogtailCommonFlags.cpp @@ -96,10 +96,7 @@ DEFINE_FLAG_INT32(ilogtail_epoll_wait_events, "epoll_wait event number", 100); DEFINE_FLAG_INT32(ilogtail_max_epoll_events, "the max events number in epoll", 10000); // sls sender -DEFINE_FLAG_INT32(sls_client_send_timeout, "timeout time of one operation for SlsClient", 15); DEFINE_FLAG_BOOL(sls_client_send_compress, "whether compresses the data or not when put data", true); -DEFINE_FLAG_INT32(send_retrytimes, "how many times should retry if PostLogStoreLogs operation fail", 3); -DEFINE_FLAG_DOUBLE(loggroup_bytes_inflation, "", 1.2); DEFINE_FLAG_STRING(default_region_name, "for compatible with old user_log_config.json or old config server", "__default_region__"); diff --git a/core/common/LogtailCommonFlags.h b/core/common/LogtailCommonFlags.h index 2c980742a0..60bf0d1de9 100644 --- a/core/common/LogtailCommonFlags.h +++ b/core/common/LogtailCommonFlags.h @@ -34,10 +34,7 @@ DECLARE_FLAG_INT32(ilogtail_epoll_wait_events); DECLARE_FLAG_INT32(ilogtail_max_epoll_events); // sls sender -DECLARE_FLAG_INT32(sls_client_send_timeout); DECLARE_FLAG_BOOL(sls_client_send_compress); -DECLARE_FLAG_INT32(send_retrytimes); -DECLARE_FLAG_DOUBLE(loggroup_bytes_inflation); DECLARE_FLAG_STRING(default_region_name); // profile diff --git a/core/common/common.cmake b/core/common/common.cmake index cd5e9401c6..5cc621bd51 100644 --- a/core/common/common.cmake +++ b/core/common/common.cmake @@ -28,7 +28,7 @@ endif () list(APPEND THIS_SOURCE_FILES_LIST ${XX_HASH_SOURCE_FILES}) # add memory in common list(APPEND THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/memory/SourceBuffer.h) -list(APPEND THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/http/AsynCurlRunner.cpp ${CMAKE_SOURCE_DIR}/common/http/Curl.cpp ${CMAKE_SOURCE_DIR}/common/http/HttpResponse.cpp ${CMAKE_SOURCE_DIR}/common/http/HttpRequest.cpp) +list(APPEND THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/http/AsynCurlRunner.cpp ${CMAKE_SOURCE_DIR}/common/http/Curl.cpp ${CMAKE_SOURCE_DIR}/common/http/HttpResponse.cpp ${CMAKE_SOURCE_DIR}/common/http/HttpRequest.cpp ${CMAKE_SOURCE_DIR}/common/http/Constant.cpp) list(APPEND THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/timer/Timer.cpp ${CMAKE_SOURCE_DIR}/common/timer/HttpRequestTimerEvent.cpp) list(APPEND THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/compression/Compressor.cpp ${CMAKE_SOURCE_DIR}/common/compression/CompressorFactory.cpp ${CMAKE_SOURCE_DIR}/common/compression/LZ4Compressor.cpp ${CMAKE_SOURCE_DIR}/common/compression/ZstdCompressor.cpp) # remove several files in common diff --git a/core/common/http/AsynCurlRunner.cpp b/core/common/http/AsynCurlRunner.cpp index a5201fb609..8a58082423 100644 --- a/core/common/http/AsynCurlRunner.cpp +++ b/core/common/http/AsynCurlRunner.cpp @@ -16,7 +16,6 @@ #include -#include "app_config/AppConfig.h" #include "common/StringTools.h" #include "common/http/Curl.h" #include "logger/Logger.h" @@ -58,7 +57,7 @@ void AsynCurlRunner::Run() { LOG_DEBUG( sLogger, ("got request from queue, request address", request.get())("try cnt", ToString(request->mTryCnt))); - if (!AddRequestToClient(std::move(request))) { + if (!AddRequestToMultiCurlHandler(mClient, std::move(request))) { continue; } } else if (mIsFlush && mQueue.Empty()) { @@ -74,47 +73,6 @@ void AsynCurlRunner::Run() { } } -bool AsynCurlRunner::AddRequestToClient(unique_ptr&& request) { - curl_slist* headers = nullptr; - CURL* curl = CreateCurlHandler(request->mMethod, - request->mHTTPSFlag, - request->mHost, - request->mPort, - request->mUrl, - request->mQueryString, - request->mHeader, - request->mBody, - request->mResponse, - headers, - request->mTimeout, - AppConfig::GetInstance()->IsHostIPReplacePolicyEnabled(), - AppConfig::GetInstance()->GetBindInterface(), - request->mFollowRedirects, - request->mTls); - - if (curl == nullptr) { - LOG_ERROR(sLogger, ("failed to send request", "failed to init curl handler")("request address", request.get())); - request->OnSendDone(request->mResponse); - return false; - } - - request->mPrivateData = headers; - curl_easy_setopt(curl, CURLOPT_PRIVATE, request.get()); - request->mLastSendTime = std::chrono::system_clock::now(); - auto res = curl_multi_add_handle(mClient, curl); - if (res != CURLM_OK) { - LOG_ERROR(sLogger, - ("failed to send request", "failed to add the easy curl handle to multi_handle")( - "errMsg", curl_multi_strerror(res))("request address", request.get())); - request->OnSendDone(request->mResponse); - curl_easy_cleanup(curl); - return false; - } - // let runner destruct the request - request.release(); - return true; -} - void AsynCurlRunner::DoRun() { CURLMcode mc; int runningHandlers = 1; @@ -126,14 +84,14 @@ void AsynCurlRunner::DoRun() { this_thread::sleep_for(chrono::milliseconds(100)); continue; } - HandleCompletedRequests(runningHandlers); + HandleCompletedAsynRequests(mClient, runningHandlers); unique_ptr request; if (mQueue.TryPop(request)) { LOG_DEBUG(sLogger, ("got item from flusher runner, request address", request.get())("try cnt", ToString(request->mTryCnt))); - if (AddRequestToClient(std::move(request))) { + if (AddRequestToMultiCurlHandler(mClient, std::move(request))) { ++runningHandlers; } } @@ -174,66 +132,4 @@ void AsynCurlRunner::DoRun() { } } -void AsynCurlRunner::HandleCompletedRequests(int& runningHandlers) { - int msgsLeft = 0; - CURLMsg* msg = curl_multi_info_read(mClient, &msgsLeft); - while (msg) { - if (msg->msg == CURLMSG_DONE) { - bool requestReused = false; - CURL* handler = msg->easy_handle; - AsynHttpRequest* request = nullptr; - curl_easy_getinfo(handler, CURLINFO_PRIVATE, &request); - auto responseTime - = chrono::duration_cast(chrono::system_clock::now() - request->mLastSendTime) - .count(); - switch (msg->data.result) { - case CURLE_OK: { - long statusCode = 0; - curl_easy_getinfo(handler, CURLINFO_RESPONSE_CODE, &statusCode); - request->mResponse.SetStatusCode(statusCode); - request->OnSendDone(request->mResponse); - LOG_DEBUG(sLogger, - ("send http request succeeded, request address", - request)("response time", ToString(responseTime) + "ms")("try cnt", - ToString(request->mTryCnt))); - break; - } - default: - // considered as network error - if (request->mTryCnt < request->mMaxTryCnt) { - LOG_WARNING(sLogger, - ("failed to send http request", "retry immediately")("request address", request)( - "try cnt", request->mTryCnt)("errMsg", curl_easy_strerror(msg->data.result))); - // free first，becase mPrivateData will be reset in AddRequestToClient - if (request->mPrivateData) { - curl_slist_free_all((curl_slist*)request->mPrivateData); - request->mPrivateData = nullptr; - } - ++request->mTryCnt; - AddRequestToClient(unique_ptr(request)); - ++runningHandlers; - requestReused = true; - } else { - request->OnSendDone(request->mResponse); - LOG_DEBUG( - sLogger, - ("failed to send http request", "abort")("request address", request)( - "response time", ToString(responseTime) + "ms")("try cnt", ToString(request->mTryCnt))); - } - break; - } - - curl_multi_remove_handle(mClient, handler); - curl_easy_cleanup(handler); - if (!requestReused) { - if (request->mPrivateData) { - curl_slist_free_all((curl_slist*)request->mPrivateData); - } - delete request; - } - } - msg = curl_multi_info_read(mClient, &msgsLeft); - } -} - } // namespace logtail diff --git a/core/common/http/AsynCurlRunner.h b/core/common/http/AsynCurlRunner.h index a5c02eb4d6..16a3dc3f93 100644 --- a/core/common/http/AsynCurlRunner.h +++ b/core/common/http/AsynCurlRunner.h @@ -48,9 +48,7 @@ class AsynCurlRunner { ~AsynCurlRunner() = default; void Run(); - bool AddRequestToClient(std::unique_ptr&& request); void DoRun(); - void HandleCompletedRequests(int& runningHandlers); CURLM* mClient = nullptr; SafeQueue> mQueue; diff --git a/core/common/http/Constant.cpp b/core/common/http/Constant.cpp new file mode 100644 index 0000000000..0a1421182c --- /dev/null +++ b/core/common/http/Constant.cpp @@ -0,0 +1,34 @@ +// Copyright 2024 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "common/http/Constant.h" + +using namespace std; + +namespace logtail { + +const string HTTP_POST = "POST"; +const string HTTP_GET = "GET"; + +const string HOST = "Host"; +const string DATE = "Date"; +const string USER_AGENT = "User-Agent"; +const string CONTENT_TYPE = "Content-Type"; +const string CONTENT_LENGTH = "Content-Length"; +const string AUTHORIZATION = "Authorization"; +const string SIGNATURE = "Signature"; + +const string TYPE_LOG_PROTOBUF = "application/x-protobuf"; + +} // namespace logtail diff --git a/core/common/http/Constant.h b/core/common/http/Constant.h new file mode 100644 index 0000000000..e96716c9a2 --- /dev/null +++ b/core/common/http/Constant.h @@ -0,0 +1,36 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +namespace logtail { + +extern const std::string HTTP_POST; +extern const std::string HTTP_GET; + +extern const std::string HOST; +extern const std::string DATE; +extern const std::string USER_AGENT; +extern const std::string CONTENT_LENGTH; +extern const std::string CONTENT_TYPE; +extern const std::string AUTHORIZATION; +extern const std::string SIGNATURE; + +extern const std::string TYPE_LOG_PROTOBUF; + +} // namespace logtail diff --git a/core/common/http/Curl.cpp b/core/common/http/Curl.cpp index 19778c2abc..437f5ba9ae 100644 --- a/core/common/http/Curl.cpp +++ b/core/common/http/Curl.cpp @@ -20,6 +20,7 @@ #include "app_config/AppConfig.h" #include "common/DNSCache.h" +#include "common/StringTools.h" #include "common/http/HttpResponse.h" #include "logger/Logger.h" @@ -57,21 +58,21 @@ static size_t header_write_callback(char* buffer, return sizes; } -CURL* CreateCurlHandler(const std::string& method, +CURL* CreateCurlHandler(const string& method, bool httpsFlag, - const std::string& host, + const string& host, int32_t port, - const std::string& url, - const std::string& queryString, - const std::map& header, - const std::string& body, + const string& url, + const string& queryString, + const map& header, + const string& body, HttpResponse& response, curl_slist*& headers, uint32_t timeout, bool replaceHostWithIp, - const std::string& intf, + const string& intf, bool followRedirects, - std::optional tls) { + optional tls) { static DnsCache* dnsCache = DnsCache::GetInstance(); CURL* curl = curl_easy_init(); @@ -80,7 +81,7 @@ CURL* CreateCurlHandler(const std::string& method, } string totalUrl = httpsFlag ? "https://" : "http://"; - std::string hostIP; + string hostIP; if (replaceHostWithIp && dnsCache->GetIPFromDnsCache(host, hostIP)) { totalUrl.append(hostIP); } else { @@ -144,7 +145,7 @@ CURL* CreateCurlHandler(const std::string& method, return curl; } -bool SendHttpRequest(std::unique_ptr&& request, HttpResponse& response) { +bool SendHttpRequest(unique_ptr&& request, HttpResponse& response) { curl_slist* headers = NULL; CURL* curl = CreateCurlHandler(request->mMethod, request->mHTTPSFlag, @@ -191,4 +192,153 @@ bool SendHttpRequest(std::unique_ptr&& request, HttpResponse& respo return success; } +bool AddRequestToMultiCurlHandler(CURLM* multiCurl, unique_ptr&& request) { + curl_slist* headers = NULL; + CURL* curl = CreateCurlHandler(request->mMethod, + request->mHTTPSFlag, + request->mHost, + request->mPort, + request->mUrl, + request->mQueryString, + request->mHeader, + request->mBody, + request->mResponse, + headers, + request->mTimeout, + AppConfig::GetInstance()->IsHostIPReplacePolicyEnabled(), + AppConfig::GetInstance()->GetBindInterface(), + request->mFollowRedirects, + request->mTls); + if (curl == NULL) { + LOG_ERROR(sLogger, ("failed to send request", "failed to init curl handler")("request address", request.get())); + request->OnSendDone(request->mResponse); + return false; + } + + request->mPrivateData = headers; + curl_easy_setopt(curl, CURLOPT_PRIVATE, request.get()); + request->mLastSendTime = chrono::system_clock::now(); + CURLMcode res = curl_multi_add_handle(multiCurl, curl); + if (res != CURLM_OK) { + LOG_ERROR(sLogger, + ("failed to send request", "failed to add the easy curl handle to multi_handle")( + "errMsg", curl_multi_strerror(res))("request address", request.get())); + request->OnSendDone(request->mResponse); + curl_easy_cleanup(curl); + return false; + } + // let callback destruct the request + request.release(); + return true; +} + +void HandleCompletedAsynRequests(CURLM* multiCurl, int& runningHandlers) { + int msgsLeft = 0; + CURLMsg* msg = curl_multi_info_read(multiCurl, &msgsLeft); + while (msg) { + if (msg->msg == CURLMSG_DONE) { + bool requestReused = false; + CURL* handler = msg->easy_handle; + AsynHttpRequest* request = nullptr; + curl_easy_getinfo(handler, CURLINFO_PRIVATE, &request); + auto responseTimeMs + = chrono::duration_cast(chrono::system_clock::now() - request->mLastSendTime); + switch (msg->data.result) { + case CURLE_OK: { + long statusCode = 0; + curl_easy_getinfo(handler, CURLINFO_RESPONSE_CODE, &statusCode); + request->mResponse.SetStatusCode(statusCode); + request->mResponse.SetResponseTime(responseTimeMs); + request->OnSendDone(request->mResponse); + LOG_DEBUG(sLogger, + ("send http request succeeded, request address", + request)("response time", ToString(responseTimeMs.count()) + "ms")( + "try cnt", ToString(request->mTryCnt))); + break; + } + default: + // considered as network error + if (request->mTryCnt < request->mMaxTryCnt) { + LOG_WARNING(sLogger, + ("failed to send http request", "retry immediately")("request address", request)( + "try cnt", request->mTryCnt)("errMsg", curl_easy_strerror(msg->data.result))); + // free first，becase mPrivateData will be reset in AddRequestToMultiCurlHandler + if (request->mPrivateData) { + curl_slist_free_all((curl_slist*)request->mPrivateData); + request->mPrivateData = nullptr; + } + ++request->mTryCnt; + AddRequestToMultiCurlHandler(multiCurl, unique_ptr(request)); + ++runningHandlers; + requestReused = true; + } else { + request->OnSendDone(request->mResponse); + LOG_DEBUG(sLogger, + ("failed to send http request", "abort")("request address", request)( + "response time", + ToString(responseTimeMs.count()) + "ms")("try cnt", ToString(request->mTryCnt))); + } + break; + } + + curl_multi_remove_handle(multiCurl, handler); + curl_easy_cleanup(handler); + if (!requestReused) { + if (request->mPrivateData) { + curl_slist_free_all((curl_slist*)request->mPrivateData); + } + delete request; + } + } + msg = curl_multi_info_read(multiCurl, &msgsLeft); + } +} + +void SendAsynRequests(CURLM* multiCurl) { + CURLMcode mc; + int runningHandlers = 0; + do { + if ((mc = curl_multi_perform(multiCurl, &runningHandlers)) != CURLM_OK) { + LOG_ERROR( + sLogger, + ("failed to call curl_multi_perform", "sleep 100ms and retry")("errMsg", curl_multi_strerror(mc))); + this_thread::sleep_for(chrono::milliseconds(100)); + continue; + } + HandleCompletedAsynRequests(multiCurl, runningHandlers); + + long curlTimeout = -1; + if ((mc = curl_multi_timeout(multiCurl, &curlTimeout)) != CURLM_OK) { + LOG_WARNING( + sLogger, + ("failed to call curl_multi_timeout", "use default timeout 1s")("errMsg", curl_multi_strerror(mc))); + } + struct timeval timeout { + 1, 0 + }; + if (curlTimeout >= 0) { + timeout.tv_sec = curlTimeout / 1000; + timeout.tv_usec = (curlTimeout % 1000) * 1000; + } + + int maxfd = -1; + fd_set fdread; + fd_set fdwrite; + fd_set fdexcep; + FD_ZERO(&fdread); + FD_ZERO(&fdwrite); + FD_ZERO(&fdexcep); + if ((mc = curl_multi_fdset(multiCurl, &fdread, &fdwrite, &fdexcep, &maxfd)) != CURLM_OK) { + LOG_ERROR(sLogger, ("failed to call curl_multi_fdset", "sleep 100ms")("errMsg", curl_multi_strerror(mc))); + } + if (maxfd == -1) { + // sleep min(timeout, 100ms) according to libcurl + int64_t sleepMs = (curlTimeout >= 0 && curlTimeout < 100) ? curlTimeout : 100; + this_thread::sleep_for(chrono::milliseconds(sleepMs)); + } else { + select(maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout); + } + } while (runningHandlers); +} + } // namespace logtail diff --git a/core/common/http/Curl.h b/core/common/http/Curl.h index e1f311c017..4d4fdd54c9 100644 --- a/core/common/http/Curl.h +++ b/core/common/http/Curl.h @@ -46,4 +46,8 @@ CURL* CreateCurlHandler(const std::string& method, bool SendHttpRequest(std::unique_ptr&& request, HttpResponse& response); +bool AddRequestToMultiCurlHandler(CURLM* multiCurl, std::unique_ptr&& request); +void SendAsynRequests(CURLM* multiCurl); +void HandleCompletedAsynRequests(CURLM* multiCurl, int& runningHandlers); + } // namespace logtail diff --git a/core/common/http/HttpRequest.cpp b/core/common/http/HttpRequest.cpp index 3f6fd215d7..34a4f3428e 100644 --- a/core/common/http/HttpRequest.cpp +++ b/core/common/http/HttpRequest.cpp @@ -19,4 +19,40 @@ DEFINE_FLAG_INT32(default_http_request_max_try_cnt, "", 3); using namespace std; -namespace logtail {} // namespace logtail +namespace logtail { + +static unsigned char ToHex(unsigned char x) { + return x > 9 ? x + 55 : x + 48; +} + +static string UrlEncode(const string& str) { + string strTemp; + size_t length = str.length(); + for (size_t i = 0; i < length; i++) { + if (isalnum((unsigned char)str[i]) || (str[i] == '-') || (str[i] == '_') || (str[i] == '.') || (str[i] == '~')) + strTemp += str[i]; + else if (str[i] == ' ') + strTemp += "+"; + else { + strTemp += '%'; + strTemp += ToHex((unsigned char)str[i] >> 4); + strTemp += ToHex((unsigned char)str[i] % 16); + } + } + return strTemp; +} + +string GetQueryString(const map& parameters) { + string res; + for (auto it = parameters.begin(); it != parameters.end(); ++it) { + if (it != parameters.begin()) { + res.append("&"); + } + res.append(it->first); + res.append("="); + res.append(UrlEncode(it->second)); + } + return res; +} + +} // namespace logtail diff --git a/core/common/http/HttpRequest.h b/core/common/http/HttpRequest.h index 13ef2db70c..f0a2abeb85 100644 --- a/core/common/http/HttpRequest.h +++ b/core/common/http/HttpRequest.h @@ -102,13 +102,24 @@ struct AsynHttpRequest : public HttpRequest { uint32_t maxTryCnt = static_cast(INT32_FLAG(default_http_request_max_try_cnt)), bool followRedirects = false, std::optional tls = std::nullopt) - : HttpRequest( - method, httpsFlag, host, port, url, query, header, body, timeout, maxTryCnt, followRedirects, std::move(tls)), + : HttpRequest(method, + httpsFlag, + host, + port, + url, + query, + header, + body, + timeout, + maxTryCnt, + followRedirects, + std::move(tls)), mResponse(std::move(response)) {} virtual bool IsContextValid() const = 0; virtual void OnSendDone(HttpResponse& response) = 0; }; +std::string GetQueryString(const std::map& parameters); } // namespace logtail diff --git a/core/common/http/HttpResponse.h b/core/common/http/HttpResponse.h index 70b1d228b7..e5be450a50 100644 --- a/core/common/http/HttpResponse.h +++ b/core/common/http/HttpResponse.h @@ -16,6 +16,7 @@ #pragma once +#include #include #include #include @@ -75,12 +76,24 @@ class HttpResponse { } void SetStatusCode(int32_t code) { mStatusCode = code; } + void SetResponseTime(const std::chrono::milliseconds& time) { mResponseTime = time; } + std::chrono::milliseconds GetResponseTime() const { return mResponseTime; } + +#ifdef APSARA_UNIT_TEST_MAIN + template + void SetBody(const T& body) { + *mBody = body; + } + + void AddHeader(const std::string& key, const std::string& value) { mHeader[key] = value; } +#endif private: int32_t mStatusCode = 0; // 0 means no response from server std::map mHeader; std::unique_ptr> mBody; size_t (*mWriteCallback)(char*, size_t, size_t, void*) = nullptr; + std::chrono::milliseconds mResponseTime = std::chrono::milliseconds::max(); }; } // namespace logtail diff --git a/core/config/common_provider/CommonConfigProvider.cpp b/core/config/common_provider/CommonConfigProvider.cpp index e86808d7a2..2e278c6ecf 100644 --- a/core/config/common_provider/CommonConfigProvider.cpp +++ b/core/config/common_provider/CommonConfigProvider.cpp @@ -26,6 +26,8 @@ #include "common/StringTools.h" #include "common/UUIDUtil.h" #include "common/YamlUtil.h" +#include "common/http/Constant.h" +#include "common/http/Curl.h" #include "common/version.h" #include "config/ConfigUtil.h" #include "config/PipelineConfig.h" @@ -33,9 +35,6 @@ #include "constants/Constants.h" #include "logger/Logger.h" #include "monitor/Monitor.h" -#include "sdk/Common.h" -#include "sdk/CurlImp.h" -#include "sdk/Exception.h" using namespace std; @@ -43,7 +42,9 @@ DEFINE_FLAG_INT32(heartbeat_interval, "second", 10); namespace logtail { -std::string CommonConfigProvider::configVersion = "version"; +const string AGENT = "/Agent"; + +string CommonConfigProvider::configVersion = "version"; void CommonConfigProvider::Init(const string& dir) { sName = "common config provider"; @@ -300,7 +301,7 @@ configserver::proto::v2::HeartbeatRequest CommonConfigProvider::PrepareHeartbeat bool CommonConfigProvider::SendHeartbeat(const configserver::proto::v2::HeartbeatRequest& heartbeatReq, configserver::proto::v2::HeartbeatResponse& heartbeatResponse) { - string operation = sdk::CONFIGSERVERAGENT; + string operation = AGENT; operation.append("/").append("Heartbeat"); string reqBody; heartbeatReq.SerializeToString(&reqBody); @@ -323,31 +324,25 @@ bool CommonConfigProvider::SendHttpRequest(const string& operation, // LCOV_EXCL_START ConfigServerAddress configServerAddress = GetOneConfigServerAddress(false); map httpHeader; - httpHeader[sdk::CONTENT_TYPE] = sdk::TYPE_LOG_PROTOBUF; - sdk::HttpMessage httpResponse; - httpResponse.header[sdk::X_LOG_REQUEST_ID] = requestId; - sdk::CurlClient client; - - try { - client.Send(sdk::HTTP_POST, - configServerAddress.host, - configServerAddress.port, - operation, - "", - httpHeader, - reqBody, - INT32_FLAG(sls_client_send_timeout), - httpResponse, - "", - false); - resp.swap(httpResponse.content); - return true; - } catch (const sdk::LOGException& e) { + httpHeader[CONTENT_TYPE] = TYPE_LOG_PROTOBUF; + + HttpResponse httpResponse; + if (!logtail::SendHttpRequest(make_unique(HTTP_POST, + false, + configServerAddress.host, + configServerAddress.port, + operation, + "", + httpHeader, + reqBody), + httpResponse)) { LOG_WARNING(sLogger, - (configType, "fail")("reqBody", reqBody)("errCode", e.GetErrorCode())("errMsg", e.GetMessage())( - "host", configServerAddress.host)("port", configServerAddress.port)); + (configType, "fail")("reqBody", + reqBody)("host", configServerAddress.host)("port", configServerAddress.port)); return false; } + resp = *httpResponse.GetBody(); + return true; // LCOV_EXCL_STOP } @@ -498,7 +493,7 @@ bool CommonConfigProvider::FetchInstanceConfigFromServer( reqConfig->set_name(config.name()); reqConfig->set_version(config.version()); } - string operation = sdk::CONFIGSERVERAGENT; + string operation = AGENT; operation.append("/FetchInstanceConfig"); string reqBody; fetchConfigRequest.SerializeToString(&reqBody); @@ -525,7 +520,7 @@ bool CommonConfigProvider::FetchPipelineConfigFromServer( reqConfig->set_name(config.name()); reqConfig->set_version(config.version()); } - string operation = sdk::CONFIGSERVERAGENT; + string operation = AGENT; operation.append("/FetchPipelineConfig"); string reqBody; fetchConfigRequest.SerializeToString(&reqBody); diff --git a/core/config/common_provider/LegacyCommonConfigProvider.cpp b/core/config/common_provider/LegacyCommonConfigProvider.cpp index 334b4dc85b..9231cee6db 100644 --- a/core/config/common_provider/LegacyCommonConfigProvider.cpp +++ b/core/config/common_provider/LegacyCommonConfigProvider.cpp @@ -22,14 +22,14 @@ #include "app_config/AppConfig.h" #include "application/Application.h" +#include "common/EncodingUtil.h" #include "common/LogtailCommonFlags.h" #include "common/StringTools.h" +#include "common/http/Constant.h" +#include "common/http/Curl.h" #include "common/version.h" #include "logger/Logger.h" #include "monitor/Monitor.h" -#include "sdk/Common.h" -#include "sdk/CurlImp.h" -#include "sdk/Exception.h" using namespace std; @@ -37,6 +37,8 @@ DEFINE_FLAG_INT32(config_update_interval, "second", 10); namespace logtail { +const string AGENT = "/Agent"; + void LegacyCommonConfigProvider::Init(const string& dir) { ConfigProvider::Init(dir); @@ -157,7 +159,7 @@ google::protobuf::RepeatedPtrField LegacyCommonConfigProvider::SendHeartbeat(const ConfigServerAddress& configServerAddress) { configserver::proto::HeartBeatRequest heartBeatReq; configserver::proto::AgentAttributes attributes; - string requestID = sdk::Base64Enconde(string("heartbeat").append(to_string(time(NULL)))); + string requestID = Base64Enconde(string("heartbeat").append(to_string(time(NULL)))); heartBeatReq.set_request_id(requestID); heartBeatReq.set_agent_id(Application::GetInstance()->GetInstanceId()); heartBeatReq.set_agent_type("iLogtail"); @@ -179,46 +181,41 @@ LegacyCommonConfigProvider::SendHeartbeat(const ConfigServerAddress& configServe } heartBeatReq.mutable_pipeline_configs()->MergeFrom(pipelineConfigs); - string operation = sdk::CONFIGSERVERAGENT; + string operation = AGENT; operation.append("/").append("HeartBeat"); map httpHeader; - httpHeader[sdk::CONTENT_TYPE] = sdk::TYPE_LOG_PROTOBUF; + httpHeader[CONTENT_TYPE] = TYPE_LOG_PROTOBUF; string reqBody; heartBeatReq.SerializeToString(&reqBody); - sdk::HttpMessage httpResponse; - httpResponse.header[sdk::X_LOG_REQUEST_ID] = "ConfigServer"; - sdk::CurlClient client; google::protobuf::RepeatedPtrField emptyResult; - try { - client.Send(sdk::HTTP_POST, - configServerAddress.host, - configServerAddress.port, - operation, - "", - httpHeader, - reqBody, - INT32_FLAG(sls_client_send_timeout), - httpResponse, - "", - false); - configserver::proto::HeartBeatResponse heartBeatResp; - heartBeatResp.ParseFromString(httpResponse.content); - - if (0 != strcmp(heartBeatResp.request_id().c_str(), requestID.c_str())) - return emptyResult; - - LOG_DEBUG(sLogger, - ("SendHeartbeat", "success")("reqBody", reqBody)("requestId", heartBeatResp.request_id())( - "statusCode", heartBeatResp.code())); - - return heartBeatResp.pipeline_check_results(); - } catch (const sdk::LOGException& e) { + HttpResponse httpResponse; + if (!logtail::SendHttpRequest(make_unique(HTTP_POST, + false, + configServerAddress.host, + configServerAddress.port, + operation, + "", + httpHeader, + reqBody), + httpResponse)) { LOG_WARNING(sLogger, - ("SendHeartbeat", "fail")("reqBody", reqBody)("errCode", e.GetErrorCode())( - "errMsg", e.GetMessage())("host", configServerAddress.host)("port", configServerAddress.port)); + ("SendHeartbeat", + "fail")("reqBody", reqBody)("host", configServerAddress.host)("port", configServerAddress.port)); return emptyResult; } + + configserver::proto::HeartBeatResponse heartBeatResp; + heartBeatResp.ParseFromString(*httpResponse.GetBody()); + + if (0 != strcmp(heartBeatResp.request_id().c_str(), requestID.c_str())) + return emptyResult; + + LOG_DEBUG(sLogger, + ("SendHeartbeat", "success")("reqBody", reqBody)("requestId", heartBeatResp.request_id())( + "statusCode", heartBeatResp.code())); + + return heartBeatResp.pipeline_check_results(); } google::protobuf::RepeatedPtrField LegacyCommonConfigProvider::FetchPipelineConfig( @@ -226,7 +223,7 @@ google::protobuf::RepeatedPtrField LegacyComm const google::protobuf::RepeatedPtrField& requestConfigs) { configserver::proto::FetchPipelineConfigRequest fetchConfigReq; string requestID - = sdk::Base64Enconde(Application::GetInstance()->GetInstanceId().append("_").append(to_string(time(NULL)))); + = Base64Enconde(Application::GetInstance()->GetInstanceId().append("_").append(to_string(time(NULL)))); fetchConfigReq.set_request_id(requestID); fetchConfigReq.set_agent_id(Application::GetInstance()->GetInstanceId()); @@ -242,47 +239,39 @@ google::protobuf::RepeatedPtrField LegacyComm } fetchConfigReq.mutable_req_configs()->MergeFrom(configInfos); - string operation = sdk::CONFIGSERVERAGENT; + string operation = AGENT; operation.append("/").append("FetchPipelineConfig"); map httpHeader; - httpHeader[sdk::CONTENT_TYPE] = sdk::TYPE_LOG_PROTOBUF; + httpHeader[CONTENT_TYPE] = TYPE_LOG_PROTOBUF; string reqBody; fetchConfigReq.SerializeToString(&reqBody); - sdk::HttpMessage httpResponse; - httpResponse.header[sdk::X_LOG_REQUEST_ID] = "ConfigServer"; - sdk::CurlClient client; google::protobuf::RepeatedPtrField emptyResult; - try { - client.Send(sdk::HTTP_POST, - configServerAddress.host, - configServerAddress.port, - operation, - "", - httpHeader, - reqBody, - INT32_FLAG(sls_client_send_timeout), - httpResponse, - "", - false); - - configserver::proto::FetchPipelineConfigResponse fetchConfigResp; - fetchConfigResp.ParseFromString(httpResponse.content); - - if (0 != strcmp(fetchConfigResp.request_id().c_str(), requestID.c_str())) - return emptyResult; - - LOG_DEBUG(sLogger, - ("GetConfigUpdateInfos", "success")("reqBody", reqBody)("requestId", fetchConfigResp.request_id())( - "statusCode", fetchConfigResp.code())); - - return fetchConfigResp.config_details(); - } catch (const sdk::LOGException& e) { - LOG_WARNING(sLogger, - ("GetConfigUpdateInfos", "fail")("reqBody", reqBody)("errCode", e.GetErrorCode())("errMsg", - e.GetMessage())); + HttpResponse httpResponse; + if (!logtail::SendHttpRequest(make_unique(HTTP_POST, + false, + configServerAddress.host, + configServerAddress.port, + operation, + "", + httpHeader, + reqBody), + httpResponse)) { + LOG_WARNING(sLogger, ("GetConfigUpdateInfos", "fail")("reqBody", reqBody)); return emptyResult; } + + configserver::proto::FetchPipelineConfigResponse fetchConfigResp; + fetchConfigResp.ParseFromString(*httpResponse.GetBody()); + + if (0 != strcmp(fetchConfigResp.request_id().c_str(), requestID.c_str())) + return emptyResult; + + LOG_DEBUG(sLogger, + ("GetConfigUpdateInfos", "success")("reqBody", reqBody)("requestId", fetchConfigResp.request_id())( + "statusCode", fetchConfigResp.code())); + + return fetchConfigResp.config_details(); } void LegacyCommonConfigProvider::UpdateRemoteConfig( diff --git a/core/file_server/reader/LogFileReader.cpp b/core/file_server/reader/LogFileReader.cpp index 08b7693354..d5024ca48c 100644 --- a/core/file_server/reader/LogFileReader.cpp +++ b/core/file_server/reader/LogFileReader.cpp @@ -53,7 +53,6 @@ #include "pipeline/queue/QueueKeyManager.h" #include "plugin/processor/inner/ProcessorParseContainerLogNative.h" #include "rapidjson/document.h" -#include "sdk/Common.h" using namespace sls_logs; using namespace std; @@ -90,6 +89,8 @@ namespace logtail { size_t LogFileReader::BUFFER_SIZE = 1024 * 512; // 512KB +const int64_t kFirstHashKeySeqID = 1; + LogFileReader* LogFileReader::CreateLogFileReader(const string& hostLogPathDir, const string& hostLogPathFile, const DevInode& devInode, @@ -439,7 +440,7 @@ void LogFileReader::initExactlyOnce(uint32_t concurrency) { auto partitionRange = detail::getPartitionRange(partIdx, mEOOption->concurrency, kPartitionCount); auto partitionID = partitionRange.first + rand() % (partitionRange.second - partitionRange.first + 1); rangeCpt->data.set_hash_key(GenerateHashKey(baseHashKey, partitionID, kPartitionCount)); - rangeCpt->data.set_sequence_id(sdk::kFirstHashKeySeqID); + rangeCpt->data.set_sequence_id(kFirstHashKeySeqID); rangeCpt->data.set_committed(false); } LOG_DEBUG(sLogger, diff --git a/core/monitor/Monitor.cpp b/core/monitor/Monitor.cpp index 34fabe9059..bcb05ebf6f 100644 --- a/core/monitor/Monitor.cpp +++ b/core/monitor/Monitor.cpp @@ -25,6 +25,7 @@ #include "app_config/AppConfig.h" #include "application/Application.h" #include "common/DevInode.h" +#include "common/EncodingUtil.h" #include "common/ExceptionBase.h" #include "common/LogtailCommonFlags.h" #include "common/MachineInfoUtil.h" @@ -41,7 +42,6 @@ #include "plugin/flusher/sls/FlusherSLS.h" #include "protobuf/sls/sls_logs.pb.h" #include "runner/FlusherRunner.h" -#include "sdk/Common.h" #ifdef __ENTERPRISE__ #include "config/provider/EnterpriseConfigProvider.h" #endif diff --git a/core/monitor/profile_sender/ProfileSender.cpp b/core/monitor/profile_sender/ProfileSender.cpp index d899cbae00..9d332ffe3e 100644 --- a/core/monitor/profile_sender/ProfileSender.cpp +++ b/core/monitor/profile_sender/ProfileSender.cpp @@ -25,9 +25,8 @@ #ifdef __ENTERPRISE__ #include "EnterpriseProfileSender.h" #endif -#include "sdk/Exception.h" -#include "plugin/flusher/sls/SLSClientManager.h" #include "app_config/AppConfig.h" +#include "plugin/flusher/sls/SLSClientManager.h" // TODO: temporarily used #include "common/compression/CompressorFactory.h" @@ -148,7 +147,7 @@ void ProfileSender::SendRunningStatus(sls_logs::LogGroup& logGroup) { string region = "cn-shanghai"; string project = "ilogtail-community-edition"; string logstore = "ilogtail-online"; - string endpoint = region + ".log.aliyuncs.com"; + string host = project + "." + region + ".log.aliyuncs.com"; Json::Value logtailStatus; logtailStatus["__topic__"] = "logtail_status_profile"; @@ -166,26 +165,14 @@ void ProfileSender::SendRunningStatus(sls_logs::LogGroup& logGroup) { } logtailStatus["__logs__"][0] = status; string logBody = logtailStatus.toStyledString(); - sdk::Client client("", endpoint, INT32_FLAG(sls_client_send_timeout)); - client.SetPort(AppConfig::GetInstance()->GetDataServerPort()); - try { - string res; - if (!CompressLz4(logBody, res)) { - LOG_ERROR(sLogger, ("lz4 compress data", "fail")); - return; - } - sdk::PostLogStoreLogsResponse resp - = client.PostLogUsingWebTracking(project, logstore, sls_logs::SLS_CMP_LZ4, res, logBody.size()); - - LOG_DEBUG(sLogger, - ("SendToProfileProject", - "success")("logBody", logBody)("requestId", resp.requestId)("statusCode", resp.statusCode)); - } catch (const sdk::LOGException& e) { - LOG_DEBUG(sLogger, - ("SendToProfileProject", "fail")("logBody", logBody)("errCode", e.GetErrorCode())("errMsg", - e.GetMessage())); + string res; + if (!CompressLz4(logBody, res)) { + LOG_ERROR(sLogger, ("lz4 compress data", "fail")); + return; } + + PutWebTracking(host, true, logstore, "lz4", res, logBody.size()); } } // namespace logtail diff --git a/core/pipeline/queue/SLSSenderQueueItem.h b/core/pipeline/queue/SLSSenderQueueItem.h index 1d34efe40f..2ee185dbb0 100644 --- a/core/pipeline/queue/SLSSenderQueueItem.h +++ b/core/pipeline/queue/SLSSenderQueueItem.h @@ -32,7 +32,7 @@ struct SLSSenderQueueItem : public SenderQueueItem { std::string mLogstore; RangeCheckpointPtr mExactlyOnceCheckpoint; - std::string mCurrentEndpoint; + std::string mCurrentHost; bool mRealIpFlag = false; int32_t mLastLogWarningTime = 0; // temporaily used diff --git a/core/plugin/flusher/sls/DiskBufferWriter.cpp b/core/plugin/flusher/sls/DiskBufferWriter.cpp index 2d8e6e22f4..077f91754a 100644 --- a/core/plugin/flusher/sls/DiskBufferWriter.cpp +++ b/core/plugin/flusher/sls/DiskBufferWriter.cpp @@ -29,9 +29,9 @@ #include "pipeline/queue/SLSSenderQueueItem.h" #include "plugin/flusher/sls/FlusherSLS.h" #include "plugin/flusher/sls/SLSClientManager.h" +#include "plugin/flusher/sls/SLSConstant.h" #include "protobuf/sls/sls_logs.pb.h" #include "provider/Provider.h" -#include "sdk/Exception.h" DEFINE_FLAG_INT32(write_secondary_wait_timeout, "interval of dump seconary buffer from memory to file, seconds", 2); DEFINE_FLAG_INT32(buffer_file_alive_interval, "the max alive time of a bufferfile, 5 minutes", 300); @@ -41,13 +41,54 @@ DEFINE_FLAG_INT32(secondary_buffer_count_limit, "data ready for write buffer fil DEFINE_FLAG_INT32(send_retry_sleep_interval, "sleep microseconds when sync send fail, 50ms", 50000); DEFINE_FLAG_INT32(buffer_check_period, "check logtail local storage buffer period", 60); DEFINE_FLAG_INT32(unauthorized_wait_interval, "", 1); +DEFINE_FLAG_INT32(send_retrytimes, "how many times should retry if PostLogStoreLogs operation fail", 3); DECLARE_FLAG_INT32(discard_send_fail_interval); using namespace std; - namespace logtail { +#ifdef __ENTERPRISE__ +static EndpointMode GetEndpointMode(sls_logs::EndpointMode mode) { + switch (mode) { + case sls_logs::EndpointMode::DEFAULT: + return EndpointMode::DEFAULT; + case sls_logs::EndpointMode::ACCELERATE: + return EndpointMode::ACCELERATE; + case sls_logs::EndpointMode::CUSTOM: + return EndpointMode::CUSTOM; + } +} + +static sls_logs::EndpointMode GetEndpointMode(EndpointMode mode) { + switch (mode) { + case EndpointMode::DEFAULT: + return sls_logs::EndpointMode::DEFAULT; + case EndpointMode::ACCELERATE: + return sls_logs::EndpointMode::ACCELERATE; + case EndpointMode::CUSTOM: + return sls_logs::EndpointMode::CUSTOM; + } +} +#endif + +static const string& GetSLSCompressTypeString(sls_logs::SlsCompressType compressType) { + switch (compressType) { + case sls_logs::SLS_CMP_NONE: { + static string none = ""; + return none; + } + case sls_logs::SLS_CMP_ZSTD: { + static string zstd = "zstd"; + return zstd; + } + default: { + static string lz4 = "lz4"; + return lz4; + } + } +} + const int32_t DiskBufferWriter::BUFFER_META_BASE_SIZE = 65536; void DiskBufferWriter::Init() { @@ -146,13 +187,6 @@ void DiskBufferWriter::BufferSenderThread() { LOG_INFO(sLogger, ("disk buffer sender", "started")); unique_lock lock(mBufferSenderThreadRunningMux); while (mIsSendBufferThreadRunning) { - if (!SLSClientManager::GetInstance()->HasNetworkAvailable()) { - if (mStopCV.wait_for( - lock, chrono::seconds(mCheckPeriod), [this]() { return !mIsSendBufferThreadRunning; })) { - break; - } - continue; - } vector filesToSend; if (!LoadFileToSend(mBufferDivideTime, filesToSend)) { if (mStopCV.wait_for( @@ -200,6 +234,7 @@ void DiskBufferWriter::BufferSenderThread() { "check header of buffer file failed, delete file: " + fileName); } } + mCandidateHostsInfos.clear(); // mIsSendingBuffer = false; lock.lock(); if (mStopCV.wait_for(lock, chrono::seconds(mCheckPeriod), [this]() { return !mIsSendBufferThreadRunning; })) { @@ -386,7 +421,7 @@ bool DiskBufferWriter::ReadNextEncryption(int32_t& pos, } } else { bufferMeta.set_project(encodedInfo); - bufferMeta.set_endpoint(FlusherSLS::GetDefaultRegion()); // new mode + bufferMeta.set_region(FlusherSLS::GetDefaultRegion()); // new mode bufferMeta.set_aliuid(""); } if (!bufferMeta.has_compresstype()) { @@ -395,6 +430,14 @@ bool DiskBufferWriter::ReadNextEncryption(int32_t& pos, if (!bufferMeta.has_telemetrytype()) { bufferMeta.set_telemetrytype(sls_logs::SLS_TELEMETRY_TYPE_LOGS); } +#ifdef __ENTERPRISE__ + if (!bufferMeta.has_endpointmode()) { + bufferMeta.set_endpointmode(sls_logs::EndpointMode::DEFAULT); + } +#endif + if (!bufferMeta.has_endpoint()) { + bufferMeta.set_endpoint(""); + } buffer = new char[meta.mEncryptionSize + 1]; nbytes = fread(buffer, sizeof(char), meta.mEncryptionSize, fin); @@ -480,20 +523,59 @@ void DiskBufferWriter::SendEncryptionBuffer(const std::string& filename, int32_t } } if (!sendResult) { - string errorCode; - SendResult res = SendBufferFileData(bufferMeta, logData, errorCode); - if (res == SEND_OK) - sendResult = true; - else if (res == SEND_DISCARD_ERROR || res == SEND_PARAMETER_INVALID) { - AlarmManager::GetInstance()->SendAlarm(SEND_DATA_FAIL_ALARM, - string("send buffer file fail, rawsize:") - + ToString(bufferMeta.rawsize()) - + "errorCode: " + errorCode, - bufferMeta.project(), - bufferMeta.logstore(), - ""); - sendResult = true; - discardCount++; + time_t beginTime = time(nullptr); + while (true) { + string host; + auto response = SendBufferFileData(bufferMeta, logData, host); + auto sendRes = ConvertErrorCode(response.mErrorCode); + bool hasAuthError = false; + switch (sendRes) { + case SEND_OK: + sendResult = true; + break; + case SEND_NETWORK_ERROR: + case SEND_SERVER_ERROR: + LOG_WARNING(sLogger, + ("send data to SLS fail", "retry later")("error_code", response.mErrorCode)( + "error_message", response.mErrorMsg)("endpoint", host)( + "projectName", bufferMeta.project())("logstore", bufferMeta.logstore())( + "rawsize", bufferMeta.rawsize())); + usleep(INT32_FLAG(send_retry_sleep_interval)); + break; + case SEND_QUOTA_EXCEED: + AlarmManager::GetInstance()->SendAlarm(SEND_QUOTA_EXCEED_ALARM, + "error_code: " + response.mErrorCode + + ", error_message: " + response.mErrorMsg, + bufferMeta.project(), + bufferMeta.logstore(), + ""); + // no region + if (!GetProfileSender()->IsProfileData("", bufferMeta.project(), bufferMeta.logstore())) + LOG_WARNING( + sLogger, + ("send data to SLS fail, error_code", + response.mErrorCode)("error_message", response.mErrorMsg)("endpoint", host)( + "projectName", bufferMeta.project())("logstore", bufferMeta.logstore())( + "rawsize", bufferMeta.rawsize())); + usleep(INT32_FLAG(quota_exceed_wait_interval)); + break; + case SEND_UNAUTHORIZED: + hasAuthError = true; + usleep(INT32_FLAG(unauthorized_wait_interval)); + break; + default: + sendResult = true; + discardCount++; + break; + } + SLSClientManager::GetInstance()->UpdateAccessKeyStatus(bufferMeta.aliuid(), !hasAuthError); + if (time(nullptr) - beginTime >= INT32_FLAG(discard_send_fail_interval)) { + sendResult = true; + discardCount++; + } + if (sendResult) { + break; + } } } } @@ -656,7 +738,7 @@ bool DiskBufferWriter::SendToBufferFile(SenderQueueItem* dataPtr) { sls_logs::LogtailBufferMeta bufferMeta; bufferMeta.set_project(flusher->mProject); - bufferMeta.set_endpoint(flusher->mRegion); + bufferMeta.set_region(flusher->mRegion); bufferMeta.set_aliuid(flusher->mAliuid); bufferMeta.set_logstore(data->mLogstore); bufferMeta.set_datatype(int32_t(data->mType)); @@ -664,6 +746,10 @@ bool DiskBufferWriter::SendToBufferFile(SenderQueueItem* dataPtr) { bufferMeta.set_shardhashkey(data->mShardHashKey); bufferMeta.set_compresstype(ConvertCompressType(flusher->GetCompressType())); bufferMeta.set_telemetrytype(flusher->mTelemetryType); +#ifdef __ENTERPRISE__ + bufferMeta.set_endpointmode(GetEndpointMode(flusher->mEndpointMode)); +#endif + bufferMeta.set_endpoint(flusher->mEndpoint); string encodedInfo; bufferMeta.SerializeToString(&encodedInfo); @@ -702,138 +788,85 @@ bool DiskBufferWriter::SendToBufferFile(SenderQueueItem* dataPtr) { return true; } -SendResult DiskBufferWriter::SendBufferFileData(const sls_logs::LogtailBufferMeta& bufferMeta, - const std::string& logData, - std::string& errorCode) { +SLSResponse DiskBufferWriter::SendBufferFileData(const sls_logs::LogtailBufferMeta& bufferMeta, + const std::string& logData, + std::string& host) { RateLimiter::FlowControl(bufferMeta.rawsize(), mSendLastTime, mSendLastByte, false); - string region = bufferMeta.endpoint(); - if (region.find("http://") == 0) // old buffer file which record the endpoint + string region = bufferMeta.region(); + // old buffer file which record the endpoint + if (region.find("http://") == 0) { region = SLSClientManager::GetInstance()->GetRegionFromEndpoint(region); + } - sdk::Client* sendClient = SLSClientManager::GetInstance()->GetClient(region, bufferMeta.aliuid()); - SendResult sendRes; - const string& endpoint = sendClient->GetRawSlsHost(); - if (endpoint.empty()) - sendRes = SEND_NETWORK_ERROR; - else { - sendRes = SendToNetSync(sendClient, region, endpoint, bufferMeta, logData, errorCode); + SLSClientManager::AuthType type; + string accessKeyId, accessKeySecret; + if (!SLSClientManager::GetInstance()->GetAccessKey(bufferMeta.aliuid(), type, accessKeyId, accessKeySecret)) { + SLSResponse response; + response.mErrorCode = LOGE_UNAUTHORIZED; + response.mErrorMsg = "can not get valid access key"; + return response; } - return sendRes; -} -SendResult DiskBufferWriter::SendToNetSync(sdk::Client* sendClient, - const std::string& region, - const std::string& endpoint, - const sls_logs::LogtailBufferMeta& bufferMeta, - const std::string& logData, - std::string& errorCode) { - int32_t retryTimes = 0; - time_t beginTime = time(NULL); - while (true) { - ++retryTimes; - try { - if (bufferMeta.datatype() == int(RawDataType::EVENT_GROUP)) { - if (bufferMeta.has_telemetrytype() - && bufferMeta.telemetrytype() == sls_logs::SLS_TELEMETRY_TYPE_METRICS) { - sendClient->PostMetricStoreLogs(bufferMeta.project(), - bufferMeta.logstore(), - bufferMeta.compresstype(), - logData, - bufferMeta.rawsize()); - } else if (bufferMeta.has_shardhashkey() && !bufferMeta.shardhashkey().empty()) - sendClient->PostLogStoreLogs(bufferMeta.project(), - bufferMeta.logstore(), - bufferMeta.compresstype(), - logData, - bufferMeta.rawsize(), - bufferMeta.shardhashkey()); - else - sendClient->PostLogStoreLogs(bufferMeta.project(), - bufferMeta.logstore(), - bufferMeta.compresstype(), - logData, - bufferMeta.rawsize()); - } else { - if (bufferMeta.has_shardhashkey() && !bufferMeta.shardhashkey().empty()) - sendClient->PostLogStoreLogPackageList(bufferMeta.project(), - bufferMeta.logstore(), - bufferMeta.compresstype(), - logData, - bufferMeta.shardhashkey()); - else - sendClient->PostLogStoreLogPackageList( - bufferMeta.project(), bufferMeta.logstore(), bufferMeta.compresstype(), logData); - } - return SEND_OK; - } catch (sdk::LOGException& ex) { - errorCode = ex.GetErrorCode(); - SendResult sendRes = ConvertErrorCode(errorCode); - bool hasAuthError = false; - switch (sendRes) { - case SEND_NETWORK_ERROR: - case SEND_SERVER_ERROR: - SLSClientManager::GetInstance()->UpdateEndpointStatus(region, endpoint, false); - SLSClientManager::GetInstance()->ResetClientEndpoint(bufferMeta.aliuid(), region, time(NULL)); - LOG_WARNING(sLogger, - ("send data to SLS fail", "retry later")("error_code", errorCode)( - "error_message", ex.GetMessage())("endpoint", sendClient->GetRawSlsHost())( - "projectName", bufferMeta.project())("logstore", bufferMeta.logstore())( - "RetryTimes", retryTimes)("rawsize", bufferMeta.rawsize())); - usleep(INT32_FLAG(send_retry_sleep_interval)); - break; - case SEND_QUOTA_EXCEED: - AlarmManager::GetInstance()->SendAlarm(SEND_QUOTA_EXCEED_ALARM, - "error_code: " + errorCode - + ", error_message: " + ex.GetMessage(), - bufferMeta.project(), - bufferMeta.logstore(), - ""); - // no region - if (!GetProfileSender()->IsProfileData("", bufferMeta.project(), bufferMeta.logstore())) - LOG_WARNING(sLogger, - ("send data to SLS fail, error_code", errorCode)("error_message", ex.GetMessage())( - "endpoint", sendClient->GetRawSlsHost())("projectName", bufferMeta.project())( - "logstore", bufferMeta.logstore())("RetryTimes", retryTimes)( - "rawsize", bufferMeta.rawsize())); - usleep(INT32_FLAG(quota_exceed_wait_interval)); - break; - case SEND_UNAUTHORIZED: - hasAuthError = true; - usleep(INT32_FLAG(unauthorized_wait_interval)); - break; - default: - break; - } - SLSClientManager::GetInstance()->UpdateAccessKeyStatus(bufferMeta.aliuid(), !hasAuthError); - if (time(nullptr) - beginTime >= INT32_FLAG(discard_send_fail_interval)) { - sendRes = SEND_DISCARD_ERROR; - } - if (sendRes != SEND_NETWORK_ERROR && sendRes != SEND_SERVER_ERROR && sendRes != SEND_QUOTA_EXCEED - && sendRes != SEND_UNAUTHORIZED) { - return sendRes; - } - { - lock_guard lock(mBufferSenderThreadRunningMux); - if (!mIsSendBufferThreadRunning) { - return sendRes; - } - } - } catch (...) { - if (retryTimes >= INT32_FLAG(send_retrytimes)) { - LOG_ERROR(sLogger, - ("send data fail", "unknown excepiton")("endpoint", sendClient->GetRawSlsHost())( - "projectName", bufferMeta.project())("logstore", bufferMeta.logstore())( - "rawsize", bufferMeta.rawsize())); - return SEND_DISCARD_ERROR; - } else { - LOG_DEBUG(sLogger, - ("send data fail", "unknown excepiton, retry later")("endpoint", sendClient->GetRawSlsHost())( - "projectName", bufferMeta.project())("logstore", bufferMeta.logstore())( - "rawsize", bufferMeta.rawsize())); - usleep(INT32_FLAG(send_retry_sleep_interval)); - } - } +#ifdef __ENTERPRISE__ + if (bufferMeta.endpointmode() == sls_logs::EndpointMode::DEFAULT) { + mRegion = region; + SLSClientManager::GetInstance()->UpdateRemoteRegionEndpoints( + region, {bufferMeta.endpoint()}, SLSClientManager::RemoteEndpointUpdateAction::CREATE); + } + auto info = SLSClientManager::GetInstance()->GetCandidateHostsInfo( + region, bufferMeta.project(), GetEndpointMode(bufferMeta.endpointmode())); +#else + auto info = SLSClientManager::GetInstance()->GetCandidateHostsInfo(bufferMeta.project(), bufferMeta.endpoint()); + if (info == nullptr) { + SLSResponse response; + response.mErrorCode = LOGE_REQUEST_ERROR; + response.mErrorMsg = "can not get available host"; + return response; + } +#endif + mCandidateHostsInfos.insert(info); + + host = info->GetCurrentHost(); + if (host.empty()) { + SLSResponse response; + response.mErrorCode = LOGE_REQUEST_ERROR; + response.mErrorMsg = "can not get available host"; + return response; + } + + bool httpsFlag = SLSClientManager::GetInstance()->UsingHttps(region); + + RawDataType dataType; + if (bufferMeta.datatype() == 0) { + dataType = RawDataType::EVENT_GROUP_LIST; + } else { + dataType = RawDataType::EVENT_GROUP; + } + if (bufferMeta.has_telemetrytype() && bufferMeta.telemetrytype() == sls_logs::SLS_TELEMETRY_TYPE_METRICS) { + return PostLogStoreLogs(accessKeyId, + accessKeySecret, + type, + host, + httpsFlag, + bufferMeta.project(), + bufferMeta.logstore(), + GetSLSCompressTypeString(bufferMeta.compresstype()), + dataType, + logData, + bufferMeta.rawsize(), + bufferMeta.has_shardhashkey() ? bufferMeta.shardhashkey() : ""); + } else { + return PostMetricStoreLogs(accessKeyId, + accessKeySecret, + type, + host, + httpsFlag, + bufferMeta.project(), + bufferMeta.logstore(), + GetSLSCompressTypeString(bufferMeta.compresstype()), + logData, + bufferMeta.rawsize()); } } -} // namespace logtail \ No newline at end of file +} // namespace logtail diff --git a/core/plugin/flusher/sls/DiskBufferWriter.h b/core/plugin/flusher/sls/DiskBufferWriter.h index 0ed9367c5e..abfbf49396 100644 --- a/core/plugin/flusher/sls/DiskBufferWriter.h +++ b/core/plugin/flusher/sls/DiskBufferWriter.h @@ -25,9 +25,9 @@ #include "common/SafeQueue.h" #include "pipeline/queue/SenderQueueItem.h" +#include "plugin/flusher/sls/SLSClientManager.h" #include "plugin/flusher/sls/SendResult.h" #include "protobuf/sls/logtail_buffer_meta.pb.h" -#include "sdk/Client.h" namespace logtail { @@ -64,15 +64,8 @@ class DiskBufferWriter { void BufferWriterThread(); void BufferSenderThread(); - SendResult SendToNetSync(sdk::Client* sendClient, - const std::string& region, - const std::string& endpoint, - const sls_logs::LogtailBufferMeta& bufferMeta, - const std::string& logData, - std::string& errorCode); - SendResult SendBufferFileData(const sls_logs::LogtailBufferMeta& bufferMeta, - const std::string& logData, - std::string& errorCode); + SLSResponse + SendBufferFileData(const sls_logs::LogtailBufferMeta& bufferMeta, const std::string& logData, std::string& host); bool SendToBufferFile(SenderQueueItem* dataPtr); bool LoadFileToSend(time_t timeLine, std::vector& filesToSend); bool CreateNewFile(); @@ -100,12 +93,26 @@ class DiskBufferWriter { bool mIsSendBufferThreadRunning = true; mutable std::condition_variable mStopCV; + struct PointerHash { + std::size_t operator()(const std::shared_ptr& ptr) const { + return std::hash()(ptr.get()); + } + }; + + struct PointerEqual { + bool operator()(const std::shared_ptr& lhs, + const std::shared_ptr& rhs) const { + return lhs.get() == rhs.get(); + } + }; + + std::unordered_set, PointerHash, PointerEqual> mCandidateHostsInfos; + mutable std::mutex mBufferFileLock; std::string mBufferFilePath; std::string mBufferFileName; volatile time_t mBufferDivideTime = 0; - // volatile bool mIsSendingBuffer = false; int64_t mCheckPeriod = 0; int64_t mSendLastTime = 0; diff --git a/core/sdk/Exception.h b/core/plugin/flusher/sls/Exception.h similarity index 100% rename from core/sdk/Exception.h rename to core/plugin/flusher/sls/Exception.h diff --git a/core/plugin/flusher/sls/FlusherSLS.cpp b/core/plugin/flusher/sls/FlusherSLS.cpp index 47977e47a6..50d58e57ec 100644 --- a/core/plugin/flusher/sls/FlusherSLS.cpp +++ b/core/plugin/flusher/sls/FlusherSLS.cpp @@ -21,6 +21,7 @@ #include "common/ParamExtractor.h" #include "common/TimeUtil.h" #include "common/compression/CompressorFactory.h" +#include "common/http/Constant.h" #include "sls_logs.pb.h" #ifdef __ENTERPRISE__ #include "config/provider/EnterpriseConfigProvider.h" @@ -32,11 +33,12 @@ #include "pipeline/queue/SenderQueueManager.h" #include "plugin/flusher/sls/PackIdManager.h" #include "plugin/flusher/sls/SLSClientManager.h" +#include "plugin/flusher/sls/SLSConstant.h" #include "plugin/flusher/sls/SLSResponse.h" +#include "plugin/flusher/sls/SLSUtil.h" #include "plugin/flusher/sls/SendResult.h" #include "provider/Provider.h" #include "runner/FlusherRunner.h" -#include "sdk/Common.h" // TODO: temporarily used here #include "pipeline/PipelineManager.h" #include "plugin/flusher/sls/DiskBufferWriter.h" @@ -68,6 +70,8 @@ enum class OperationOnFail { RETRY_IMMEDIATELY, RETRY_LATER, DISCARD }; static const int ON_FAIL_LOG_WARNING_INTERVAL_SECOND = 10; +static constexpr int64_t kInvalidHashKeySeqID = 0; + static const char* GetOperationString(OperationOnFail op) { switch (op) { case OperationOnFail::RETRY_IMMEDIATELY: @@ -210,7 +214,6 @@ void FlusherSLS::SetDefaultRegion(const string& region) { mutex FlusherSLS::sProjectRegionMapLock; unordered_map FlusherSLS::sProjectRefCntMap; -unordered_map FlusherSLS::sRegionRefCntMap; unordered_map FlusherSLS::sProjectRegionMap; string FlusherSLS::GetAllProjects() { @@ -222,11 +225,6 @@ string FlusherSLS::GetAllProjects() { return result; } -bool FlusherSLS::IsRegionContainingConfig(const string& region) { - lock_guard lock(sProjectRegionMapLock); - return sRegionRefCntMap.find(region) != sRegionRefCntMap.end(); -} - std::string FlusherSLS::GetProjectRegion(const std::string& project) { lock_guard lock(sProjectRegionMapLock); auto iter = sProjectRegionMap.find(project); @@ -239,7 +237,6 @@ std::string FlusherSLS::GetProjectRegion(const std::string& project) { void FlusherSLS::IncreaseProjectRegionReferenceCnt(const string& project, const string& region) { lock_guard lock(sProjectRegionMapLock); ++sProjectRefCntMap[project]; - ++sRegionRefCntMap[region]; sProjectRegionMap[project] = region; } @@ -252,32 +249,6 @@ void FlusherSLS::DecreaseProjectRegionReferenceCnt(const string& project, const sProjectRegionMap.erase(project); } } - - auto regionRefCnt = sRegionRefCntMap.find(region); - if (regionRefCnt != sRegionRefCntMap.end()) { - if (--regionRefCnt->second == 0) { - sRegionRefCntMap.erase(regionRefCnt); - } - } -} - -mutex FlusherSLS::sRegionStatusLock; -unordered_map FlusherSLS::sAllRegionStatus; - -void FlusherSLS::UpdateRegionStatus(const string& region, bool status) { - LOG_DEBUG(sLogger, ("update region status, region", region)("is network in good condition", ToString(status))); - lock_guard lock(sRegionStatusLock); - sAllRegionStatus[region] = status; -} - -bool FlusherSLS::GetRegionStatus(const string& region) { - lock_guard lock(sRegionStatusLock); - auto rst = sAllRegionStatus.find(region); - if (rst == sAllRegionStatus.end()) { - return true; - } else { - return rst->second; - } } bool FlusherSLS::sIsResourceInited = false; @@ -288,10 +259,10 @@ const unordered_set FlusherSLS::sNativeParam = {"Project", "Logstore", "Region", "Endpoint", + "EndpointMode", "Aliuid", "CompressType", "TelemetryType", - "FlowControlExpireTime", "MaxSendRate", "ShardHashKeys", "Batch"}; @@ -326,26 +297,66 @@ bool FlusherSLS::Init(const Json::Value& config, Json::Value& optionalGoPipeline mContext->GetRegion()); } + // Region + if ( #ifdef __ENTERPRISE__ - if (EnterpriseConfigProvider::GetInstance()->IsDataServerPrivateCloud()) { - mRegion = STRING_FLAG(default_region_name); - } else { + !EnterpriseConfigProvider::GetInstance()->IsDataServerPrivateCloud() && #endif - // Region - if (!GetOptionalStringParam(config, "Region", mRegion, errorMsg)) { - PARAM_WARNING_DEFAULT(mContext->GetLogger(), - mContext->GetAlarm(), - errorMsg, - mRegion, - sName, - mContext->GetConfigName(), - mContext->GetProjectName(), - mContext->GetLogstoreName(), - mContext->GetRegion()); - } + !GetOptionalStringParam(config, "Region", mRegion, errorMsg)) { + PARAM_WARNING_DEFAULT(mContext->GetLogger(), + mContext->GetAlarm(), + errorMsg, + mRegion, + sName, + mContext->GetConfigName(), + mContext->GetProjectName(), + mContext->GetLogstoreName(), + mContext->GetRegion()); + } - // Endpoint #ifdef __ENTERPRISE__ + // Aliuid + if (!GetOptionalStringParam(config, "Aliuid", mAliuid, errorMsg)) { + PARAM_WARNING_IGNORE(mContext->GetLogger(), + mContext->GetAlarm(), + errorMsg, + sName, + mContext->GetConfigName(), + mContext->GetProjectName(), + mContext->GetLogstoreName(), + mContext->GetRegion()); + } + + // EndpointMode + string endpointMode = "default"; + if (!GetOptionalStringParam(config, "EndpointMode", endpointMode, errorMsg)) { + PARAM_WARNING_DEFAULT(mContext->GetLogger(), + mContext->GetAlarm(), + errorMsg, + endpointMode, + sName, + mContext->GetConfigName(), + mContext->GetProjectName(), + mContext->GetLogstoreName(), + mContext->GetRegion()); + } + if (endpointMode == "accelerate") { + mEndpointMode = EndpointMode::ACCELERATE; + } else if (endpointMode != "default") { + PARAM_WARNING_DEFAULT(mContext->GetLogger(), + mContext->GetAlarm(), + "string param EndpointMode is not valid", + "default", + sName, + mContext->GetConfigName(), + mContext->GetProjectName(), + mContext->GetLogstoreName(), + mContext->GetRegion()); + } + if (mEndpointMode == EndpointMode::DEFAULT) { + // for local pipeline whose flusher region is neither specified in local info nor included by config provider, + // param Endpoint should be used, and the mode is set to default. + // warning: if inconsistency exists among configs, only the first config would be considered in this situation. if (!GetOptionalStringParam(config, "Endpoint", mEndpoint, errorMsg)) { PARAM_WARNING_IGNORE(mContext->GetLogger(), mContext->GetAlarm(), @@ -355,8 +366,13 @@ bool FlusherSLS::Init(const Json::Value& config, Json::Value& optionalGoPipeline mContext->GetProjectName(), mContext->GetLogstoreName(), mContext->GetRegion()); - } else { + } + SLSClientManager::GetInstance()->UpdateRemoteRegionEndpoints( + mRegion, {mEndpoint}, SLSClientManager::RemoteEndpointUpdateAction::CREATE); + } + mCandidateHostsInfo = SLSClientManager::GetInstance()->GetCandidateHostsInfo(mRegion, mProject, mEndpointMode); #else + // Endpoint if (!GetMandatoryStringParam(config, "Endpoint", mEndpoint, errorMsg)) { PARAM_ERROR_RETURN(mContext->GetLogger(), mContext->GetAlarm(), @@ -366,35 +382,23 @@ bool FlusherSLS::Init(const Json::Value& config, Json::Value& optionalGoPipeline mContext->GetProjectName(), mContext->GetLogstoreName(), mContext->GetRegion()); - } else { -#endif - mEndpoint = TrimString(mEndpoint); - if (!mEndpoint.empty()) { - SLSClientManager::GetInstance()->AddEndpointEntry(mRegion, - StandardizeEndpoint(mEndpoint, mEndpoint), - false, - SLSClientManager::EndpointSourceType::REMOTE); - } - } -#ifdef __ENTERPRISE__ } - - // Aliuid - if (!GetOptionalStringParam(config, "Aliuid", mAliuid, errorMsg)) { - PARAM_WARNING_IGNORE(mContext->GetLogger(), - mContext->GetAlarm(), - errorMsg, - sName, - mContext->GetConfigName(), - mContext->GetProjectName(), - mContext->GetLogstoreName(), - mContext->GetRegion()); + mEndpoint = TrimString(mEndpoint); + if (mEndpoint.empty()) { + PARAM_ERROR_RETURN(mContext->GetLogger(), + mContext->GetAlarm(), + "param Endpoint is empty", + sName, + mContext->GetConfigName(), + mContext->GetProjectName(), + mContext->GetLogstoreName(), + mContext->GetRegion()); } + mCandidateHostsInfo = SLSClientManager::GetInstance()->GetCandidateHostsInfo(mProject, mEndpoint); #endif // TelemetryType string telemetryType; - if (!GetOptionalStringParam(config, "TelemetryType", telemetryType, errorMsg)) { PARAM_WARNING_DEFAULT(mContext->GetLogger(), mContext->GetAlarm(), @@ -541,7 +545,6 @@ bool FlusherSLS::Start() { InitResource(); IncreaseProjectRegionReferenceCnt(mProject, mRegion); - SLSClientManager::GetInstance()->IncreaseAliuidReferenceCntForRegion(mRegion, mAliuid); return true; } @@ -549,7 +552,6 @@ bool FlusherSLS::Stop(bool isPipelineRemoving) { Flusher::Stop(isPipelineRemoving); DecreaseProjectRegionReferenceCnt(mProject, mRegion); - SLSClientManager::GetInstance()->DecreaseAliuidReferenceCntForRegion(mRegion, mAliuid); return true; } @@ -577,70 +579,45 @@ bool FlusherSLS::FlushAll() { } bool FlusherSLS::BuildRequest(SenderQueueItem* item, unique_ptr& req, bool* keepItem) const { - auto data = static_cast(item); - sdk::Client* sendClient = SLSClientManager::GetInstance()->GetClient(mRegion, mAliuid); - - int32_t curTime = time(NULL); - static int32_t lastResetEndpointTime = 0; - data->mCurrentEndpoint = sendClient->GetRawSlsHost(); - if (data->mCurrentEndpoint.empty()) { - if (curTime - lastResetEndpointTime >= 30) { - SLSClientManager::GetInstance()->ResetClientEndpoint(mAliuid, mRegion, curTime); - data->mCurrentEndpoint = sendClient->GetRawSlsHost(); - lastResetEndpointTime = curTime; - } - } if (mSendCnt) { mSendCnt->Add(1); } - if (BOOL_FLAG(send_prefer_real_ip)) { - if (curTime - sendClient->GetSlsRealIpUpdateTime() >= INT32_FLAG(send_check_real_ip_interval)) { - SLSClientManager::GetInstance()->UpdateSendClientRealIp(sendClient, mRegion); - } - data->mRealIpFlag = sendClient->GetRawSlsHostFlag(); + + SLSClientManager::AuthType type; + string accessKeyId, accessKeySecret; + if (!SLSClientManager::GetInstance()->GetAccessKey(mAliuid, type, accessKeyId, accessKeySecret)) { + *keepItem = true; + return false; } - if (data->mType == RawDataType::EVENT_GROUP) { - if (mTelemetryType == sls_logs::SLS_TELEMETRY_TYPE_METRICS) { - req = sendClient->CreatePostMetricStoreLogsRequest( - mProject, data->mLogstore, ConvertCompressType(GetCompressType()), data->mData, data->mRawSize, item); + auto data = static_cast(item); + if (BOOL_FLAG(send_prefer_real_ip)) { + data->mCurrentHost = SLSClientManager::GetInstance()->GetRealIp(mRegion); + if (data->mCurrentHost.empty()) { + data->mCurrentHost = mCandidateHostsInfo->GetCurrentHost(); + data->mRealIpFlag = false; } else { - if (data->mShardHashKey.empty()) { - req = sendClient->CreatePostLogStoreLogsRequest(mProject, - data->mLogstore, - ConvertCompressType(GetCompressType()), - data->mData, - data->mRawSize, - item); - } else { - auto& exactlyOnceCpt = data->mExactlyOnceCheckpoint; - int64_t hashKeySeqID = exactlyOnceCpt ? exactlyOnceCpt->data.sequence_id() : sdk::kInvalidHashKeySeqID; - req = sendClient->CreatePostLogStoreLogsRequest(mProject, - data->mLogstore, - ConvertCompressType(GetCompressType()), - data->mData, - data->mRawSize, - item, - data->mShardHashKey, - hashKeySeqID); - } + data->mRealIpFlag = true; } } else { - if (data->mShardHashKey.empty()) - req = sendClient->CreatePostLogStoreLogPackageListRequest( - mProject, data->mLogstore, ConvertCompressType(GetCompressType()), data->mData, item); - else - req = sendClient->CreatePostLogStoreLogPackageListRequest(mProject, - data->mLogstore, - ConvertCompressType(GetCompressType()), - data->mData, - item, - data->mShardHashKey); + data->mCurrentHost = mCandidateHostsInfo->GetCurrentHost(); } - if (!req) { + if (data->mCurrentHost.empty()) { *keepItem = true; + GetRegionConcurrencyLimiter(mRegion)->OnFail(); return false; } + + switch (mTelemetryType) { + case sls_logs::SLS_TELEMETRY_TYPE_METRICS: + req = CreatePostMetricStoreLogsRequest(accessKeyId, accessKeySecret, type, data); + break; + case sls_logs::SLS_TELEMETRY_TYPE_LOGS: + req = CreatePostLogStoreLogsRequest(accessKeyId, accessKeySecret, type, data); + break; + default: + break; + } return true; } @@ -648,24 +625,7 @@ void FlusherSLS::OnSendDone(const HttpResponse& response, SenderQueueItem* item) if (mSendDoneCnt) { mSendDoneCnt->Add(1); } - SLSResponse slsResponse; - if (AppConfig::GetInstance()->IsResponseVerificationEnabled() && !IsSLSResponse(response)) { - slsResponse.mStatusCode = 0; - slsResponse.mErrorCode = sdk::LOGE_REQUEST_ERROR; - slsResponse.mErrorMsg = "invalid response body"; - } else { - slsResponse.Parse(response); - - if (AppConfig::GetInstance()->EnableLogTimeAutoAdjust()) { - static uint32_t sCount = 0; - if (sCount++ % 10000 == 0 || slsResponse.mErrorCode == sdk::LOGE_REQUEST_TIME_EXPIRED) { - time_t serverTime = GetServerTime(response); - if (serverTime > 0) { - UpdateTimeDelta(serverTime); - } - } - } - } + SLSResponse slsResponse = ParseHttpResponse(response); auto data = static_cast(item); string configName = HasContext() ? GetContext().GetConfigName() : ""; @@ -688,8 +648,8 @@ void FlusherSLS::OnSendDone(const HttpResponse& response, SenderQueueItem* item) + "ms")( "total send time", ToString(chrono::duration_cast(curSystemTime - item->mFirstEnqueTime).count()) - + "ms")("try cnt", data->mTryCnt)("endpoint", data->mCurrentEndpoint)("is profile data", - isProfileData)); + + "ms")("try cnt", data->mTryCnt)("endpoint", data->mCurrentHost)("is profile data", + isProfileData)); GetRegionConcurrencyLimiter(mRegion)->OnSuccess(); GetProjectConcurrencyLimiter(mProject)->OnSuccess(); GetLogstoreConcurrencyLimiter(mProject, mLogstore)->OnSuccess(); @@ -702,7 +662,6 @@ void FlusherSLS::OnSendDone(const HttpResponse& response, SenderQueueItem* item) OperationOnFail operation; SendResult sendResult = ConvertErrorCode(slsResponse.mErrorCode); ostringstream failDetail, suggestion; - string failEndpoint = data->mCurrentEndpoint; if (sendResult == SEND_NETWORK_ERROR || sendResult == SEND_SERVER_ERROR) { if (sendResult == SEND_NETWORK_ERROR) { failDetail << "network error"; @@ -716,21 +675,10 @@ void FlusherSLS::OnSendDone(const HttpResponse& response, SenderQueueItem* item) } } suggestion << "check network connection to endpoint"; - if (BOOL_FLAG(send_prefer_real_ip) && data->mRealIpFlag) { + if (data->mRealIpFlag) { // connect refused, use vip directly failDetail << ", real ip may be stale, force update"; - // just set force update flag - SLSClientManager::GetInstance()->ForceUpdateRealIp(mRegion); - } - if (sendResult == SEND_NETWORK_ERROR) { - // only set network stat when no real ip - if (!BOOL_FLAG(send_prefer_real_ip) || !data->mRealIpFlag) { - SLSClientManager::GetInstance()->UpdateEndpointStatus(mRegion, data->mCurrentEndpoint, false); - if (SLSClientManager::GetInstance()->GetServerSwitchPolicy() - == SLSClientManager::EndpointSwitchPolicy::DESIGNATED_FIRST) { - SLSClientManager::GetInstance()->ResetClientEndpoint(mAliuid, mRegion, curTime); - } - } + SLSClientManager::GetInstance()->UpdateOutdatedRealIpRegions(mRegion); } operation = data->mBufferOrNot ? OperationOnFail::RETRY_LATER : OperationOnFail::DISCARD; GetRegionConcurrencyLimiter(mRegion)->OnFail(); @@ -738,7 +686,7 @@ void FlusherSLS::OnSendDone(const HttpResponse& response, SenderQueueItem* item) GetLogstoreConcurrencyLimiter(mProject, mLogstore)->OnSuccess(); } else if (sendResult == SEND_QUOTA_EXCEED) { BOOL_FLAG(global_network_success) = true; - if (slsResponse.mErrorCode == sdk::LOGE_SHARD_WRITE_QUOTA_EXCEED) { + if (slsResponse.mErrorCode == LOGE_SHARD_WRITE_QUOTA_EXCEED) { failDetail << "shard write quota exceed"; suggestion << "Split logstore shards. https://help.aliyun.com/zh/sls/user-guide/expansion-of-resources"; GetLogstoreConcurrencyLimiter(mProject, mLogstore)->OnFail(); @@ -821,7 +769,7 @@ void FlusherSLS::OnSendDone(const HttpResponse& response, SenderQueueItem* item) cpt->IncreaseSequenceID(); } while (0); } else if (AppConfig::GetInstance()->EnableLogTimeAutoAdjust() - && sdk::LOGE_REQUEST_TIME_EXPIRED == slsResponse.mErrorCode) { + && LOGE_REQUEST_TIME_EXPIRED == slsResponse.mErrorCode) { failDetail << "write request expired, will retry"; suggestion << "check local system time"; operation = OperationOnFail::RETRY_IMMEDIATELY; @@ -858,7 +806,7 @@ void FlusherSLS::OnSendDone(const HttpResponse& response, SenderQueueItem* item) ToString(chrono::duration_cast(curSystemTime - data->mLastSendTime).count()) \ + "ms")("total send time", \ ToString(chrono::duration_cast(curSystemTime - data->mFirstEnqueTime).count()) \ - + "ms")("endpoint", data->mCurrentEndpoint)("is profile data", isProfileData) + + "ms")("endpoint", data->mCurrentHost)("is profile data", isProfileData) switch (operation) { case OperationOnFail::RETRY_IMMEDIATELY: @@ -866,7 +814,7 @@ void FlusherSLS::OnSendDone(const HttpResponse& response, SenderQueueItem* item) FlusherRunner::GetInstance()->PushToHttpSink(item, false); break; case OperationOnFail::RETRY_LATER: - if (slsResponse.mErrorCode == sdk::LOGE_REQUEST_TIMEOUT + if (slsResponse.mErrorCode == LOGE_REQUEST_TIMEOUT || curTime - data->mLastLogWarningTime > ON_FAIL_LOG_WARNING_INTERVAL_SECOND) { LOG_WARNING(sLogger, LOG_PATTERN); data->mLastLogWarningTime = curTime; @@ -884,7 +832,7 @@ void FlusherSLS::OnSendDone(const HttpResponse& response, SenderQueueItem* item) + "\trequestId: " + slsResponse.mRequestId + "\tstatusCode: " + ToString(slsResponse.mStatusCode) + "\terrorCode: " + slsResponse.mErrorCode + "\terrorMessage: " + slsResponse.mErrorMsg - + "\tconfig: " + configName + "\tendpoint: " + data->mCurrentEndpoint, + + "\tconfig: " + configName + "\tendpoint: " + data->mCurrentHost, mProject, data->mLogstore, mRegion); @@ -1148,7 +1096,7 @@ string FlusherSLS::GetShardHashKey(const BatchedEvents& g) const { key += "_"; } } - return sdk::CalcMD5(key); + return CalcMD5(key); } void FlusherSLS::AddPackId(BatchedEvents& g) const { @@ -1160,6 +1108,60 @@ void FlusherSLS::AddPackId(BatchedEvents& g) const { g.mTags.Insert(LOG_RESERVED_KEY_PACKAGE_ID, StringView(packId.data, packId.size)); } +unique_ptr FlusherSLS::CreatePostLogStoreLogsRequest(const string& accessKeyId, + const string& accessKeySecret, + SLSClientManager::AuthType type, + SLSSenderQueueItem* item) const { + optional seqId; + if (item->mExactlyOnceCheckpoint) { + seqId = item->mExactlyOnceCheckpoint->data.sequence_id(); + } + string path, query; + map header; + PreparePostLogStoreLogsRequest(accessKeyId, + accessKeySecret, + type, + item->mCurrentHost, + item->mRealIpFlag, + mProject, + item->mLogstore, + CompressTypeToString(mCompressor->GetCompressType()), + item->mType, + item->mData, + item->mRawSize, + item->mShardHashKey, + seqId, + path, + query, + header); + bool httpsFlag = SLSClientManager::GetInstance()->UsingHttps(mRegion); + return make_unique( + HTTP_POST, httpsFlag, item->mCurrentHost, httpsFlag ? 443 : 80, path, query, header, item->mData, item); +} + +unique_ptr FlusherSLS::CreatePostMetricStoreLogsRequest(const string& accessKeyId, + const string& accessKeySecret, + SLSClientManager::AuthType type, + SLSSenderQueueItem* item) const { + string path; + map header; + PreparePostMetricStoreLogsRequest(accessKeyId, + accessKeySecret, + type, + item->mCurrentHost, + item->mRealIpFlag, + mProject, + item->mLogstore, + CompressTypeToString(mCompressor->GetCompressType()), + item->mData, + item->mRawSize, + path, + header); + bool httpsFlag = SLSClientManager::GetInstance()->UsingHttps(mRegion); + return make_unique( + HTTP_POST, httpsFlag, item->mCurrentHost, httpsFlag ? 443 : 80, path, "", header, item->mData, item); +} + sls_logs::SlsCompressType ConvertCompressType(CompressType type) { sls_logs::SlsCompressType compressType = sls_logs::SLS_CMP_NONE; switch (type) { diff --git a/core/plugin/flusher/sls/FlusherSLS.h b/core/plugin/flusher/sls/FlusherSLS.h index 6a71b0f526..61f3010415 100644 --- a/core/plugin/flusher/sls/FlusherSLS.h +++ b/core/plugin/flusher/sls/FlusherSLS.h @@ -30,7 +30,9 @@ #include "pipeline/batch/Batcher.h" #include "pipeline/limiter/ConcurrencyLimiter.h" #include "pipeline/plugin/interface/HttpFlusher.h" +#include "pipeline/queue/SLSSenderQueueItem.h" #include "pipeline/serializer/SLSSerializer.h" +#include "plugin/flusher/sls/SLSClientManager.h" #include "protobuf/sls/sls_logs.pb.h" namespace logtail { @@ -48,13 +50,8 @@ class FlusherSLS : public HttpFlusher { static std::string GetDefaultRegion(); static void SetDefaultRegion(const std::string& region); static std::string GetAllProjects(); - static bool IsRegionContainingConfig(const std::string& region); static std::string GetProjectRegion(const std::string& project); - // TODO: should be moved to enterprise config provider - static bool GetRegionStatus(const std::string& region); - static void UpdateRegionStatus(const std::string& region, bool status); - static const std::string sName; FlusherSLS(); @@ -77,8 +74,11 @@ class FlusherSLS : public HttpFlusher { std::string mProject; std::string mLogstore; std::string mRegion; - std::string mEndpoint; std::string mAliuid; +#ifdef __ENTERPRISE__ + EndpointMode mEndpointMode = EndpointMode::DEFAULT; +#endif + std::string mEndpoint; sls_logs::SlsTelemetryType mTelemetryType = sls_logs::SlsTelemetryType::SLS_TELEMETRY_TYPE_LOGS; std::vector mShardHashKeys; uint32_t mMaxSendRate = 0; // preserved only for exactly once @@ -87,8 +87,6 @@ class FlusherSLS : public HttpFlusher { std::unique_ptr mCompressor; private: - static const std::unordered_set sNativeParam; - static void InitResource(); static void IncreaseProjectRegionReferenceCnt(const std::string& project, const std::string& region); @@ -99,18 +97,15 @@ class FlusherSLS : public HttpFlusher { static std::unordered_map> sRegionConcurrencyLimiterMap; static std::unordered_map> sLogstoreConcurrencyLimiterMap; + static const std::unordered_set sNativeParam; + static std::mutex sDefaultRegionLock; static std::string sDefaultRegion; static std::mutex sProjectRegionMapLock; static std::unordered_map sProjectRefCntMap; - static std::unordered_map sRegionRefCntMap; static std::unordered_map sProjectRegionMap; - // TODO: should be moved to enterprise config provider - static std::mutex sRegionStatusLock; - static std::unordered_map sAllRegionStatus; - static bool sIsResourceInited; void GenerateGoPlugin(const Json::Value& config, Json::Value& res) const; @@ -121,9 +116,19 @@ class FlusherSLS : public HttpFlusher { std::string GetShardHashKey(const BatchedEvents& g) const; void AddPackId(BatchedEvents& g) const; + std::unique_ptr CreatePostLogStoreLogsRequest(const std::string& accessKeyId, + const std::string& accessKeySecret, + SLSClientManager::AuthType type, + SLSSenderQueueItem* item) const; + std::unique_ptr CreatePostMetricStoreLogsRequest(const std::string& accessKeyId, + const std::string& accessKeySecret, + SLSClientManager::AuthType type, + SLSSenderQueueItem* item) const; + Batcher mBatcher; std::unique_ptr mGroupSerializer; std::unique_ptr>> mGroupListSerializer; + std::shared_ptr mCandidateHostsInfo; CounterPtr mSendCnt; CounterPtr mSendDoneCnt; diff --git a/core/plugin/flusher/sls/SLSClientManager.cpp b/core/plugin/flusher/sls/SLSClientManager.cpp index 7e7930a547..0b19ca1be4 100644 --- a/core/plugin/flusher/sls/SLSClientManager.cpp +++ b/core/plugin/flusher/sls/SLSClientManager.cpp @@ -18,657 +18,983 @@ #include #endif -#include - #include "app_config/AppConfig.h" #include "common/EndpointUtil.h" #include "common/Flags.h" +#include "common/HashUtil.h" #include "common/LogtailCommonFlags.h" #include "common/StringTools.h" -#include "common/TimeUtil.h" +#include "common/http/Constant.h" +#include "common/http/Curl.h" #include "common/version.h" #include "logger/Logger.h" #include "monitor/Monitor.h" +#include "pipeline/queue/SenderQueueItem.h" #ifdef __ENTERPRISE__ #include "plugin/flusher/sls/EnterpriseSLSClientManager.h" #endif -#include "plugin/flusher/sls/FlusherSLS.h" -#include "plugin/flusher/sls/SendResult.h" -#include "sdk/Exception.h" - -// for windows compatability, to avoid conflict with the same function defined in windows.h -#ifdef SetPort -#undef SetPort -#endif +#include "plugin/flusher/sls/PackIdManager.h" +#include "plugin/flusher/sls/SLSConstant.h" +#include "plugin/flusher/sls/SLSUtil.h" -DEFINE_FLAG_STRING(data_endpoint_policy, - "policy for switching between data server endpoints, possible options include " - "'designated_first'(default) and 'designated_locked'", - "designated_first"); -DEFINE_FLAG_INT32(sls_host_update_interval, "seconds", 5); -DEFINE_FLAG_INT32(send_client_timeout_interval, "recycle clients avoid memory increment", 12 * 3600); -DEFINE_FLAG_INT32(test_network_normal_interval, "if last check is normal, test network again after seconds ", 30); -DEFINE_FLAG_INT32(test_unavailable_endpoint_interval, "test unavailable endpoint interval", 60); -DEFINE_FLAG_INT32(send_switch_real_ip_interval, "seconds", 60); -DEFINE_FLAG_BOOL(send_prefer_real_ip, "use real ip to send data", false); +DEFINE_FLAG_STRING(custom_user_agent, "custom user agent appended at the end of the exsiting ones", ""); +// DEFINE_FLAG_INT32(sls_hosts_probe_max_try_cnt, "", 3); +// DEFINE_FLAG_INT32(sls_hosts_probe_timeout, "", 3); +// DEFINE_FLAG_INT32(sls_all_hosts_probe_interval_sec, "seconds", 5 * 60); +// DEFINE_FLAG_INT32(sls_hosts_probe_interval_sec, "", 60); DEFINE_FLAG_STRING(default_access_key_id, "", ""); DEFINE_FLAG_STRING(default_access_key, "", ""); -DEFINE_FLAG_STRING(custom_user_agent, "custom user agent appended at the end of the exsiting ones", ""); +// DEFINE_FLAG_INT32(send_switch_real_ip_interval, "seconds", 60); +// DEFINE_FLAG_BOOL(send_prefer_real_ip, "use real ip to send data", false); using namespace std; namespace logtail { -bool SLSClientManager::RegionEndpointsInfo::AddDefaultEndpoint(const std::string& endpoint, - const EndpointSourceType& endpointType, - bool& isDefault) { - if (mDefaultEndpoint.empty() - || (endpointType == EndpointSourceType::LOCAL && mDefaultEndpointType == EndpointSourceType::REMOTE)) { - mDefaultEndpoint = endpoint; - mDefaultEndpointType = endpointType; - isDefault = true; +// TODO: move endpoint functions to EnterpriseEndpointUtil.h +// bool IsCustomEndpoint(const string& endpoint) { +// if (StartWith(endpoint, "log.") || StartWith(endpoint, "log-intranet.") || StartWith(endpoint, "log-internal.") +// || EndWith(endpoint, ".aliyuncs.com") || EndWith(endpoint, ".aliyun-inc.com")) { +// return false; +// } +// return true; +// } + +// enum class EndpointAddressType { INNER, INTRANET, PUBLIC }; +// const string kLogEndpointSuffix = ".log.aliyuncs.com"; + +// EndpointAddressType GetEndpointAddressType(const string& address) { +// // 一国一云 OXS区访问 & VPC访问 +// if (StartWith(address, "log-intranet.") || StartWith(address, "log-internal.")) { +// return EndpointAddressType::INTRANET; +// } +// // 一国一云 公网访问 +// if (StartWith(address, "log.")) { +// return EndpointAddressType::PUBLIC; +// } +// if (EndWith(address, "-intranet" + kLogEndpointSuffix) || EndWith(address, "-internal" + kLogEndpointSuffix)) { +// return EndpointAddressType::INTRANET; +// } +// if (!EndWith(address, "-share" + kLogEndpointSuffix) && EndWith(address, kLogEndpointSuffix)) { +// return EndpointAddressType::PUBLIC; +// } +// return EndpointAddressType::INNER; +// } + +// static const string globalAccelerationEndpoint = "log.global.aliyuncs.com"; + +const string& EndpointModeToString(EndpointMode mode) { + switch (mode) { + case EndpointMode::CUSTOM: + static string customStr = "custom"; + return customStr; + case EndpointMode::ACCELERATE: + static string accelerateStr = "accelerate"; + return accelerateStr; + case EndpointMode::DEFAULT: + static string defaultStr = "default"; + return defaultStr; + default: + static string unknownStr = "unknown"; + return unknownStr; } - return AddEndpoint(endpoint, true, false); } -bool SLSClientManager::RegionEndpointsInfo::AddEndpoint(const std::string& endpoint, bool status, bool proxy) { - if (mEndpointInfoMap.find(endpoint) == mEndpointInfoMap.end()) { - mEndpointInfoMap.emplace(std::make_pair(endpoint, EndpointInfo(status, proxy))); - return true; - } - return false; +chrono::milliseconds HostInfo::GetLatency() const { + lock_guard lock(mLatencyMux); + return mLatency; } -void SLSClientManager::RegionEndpointsInfo::UpdateEndpointInfo(const std::string& endpoint, - bool status, - std::optional latency, - bool createFlag) { - auto iter = mEndpointInfoMap.find(endpoint); - if (iter == mEndpointInfoMap.end()) { - if (createFlag) { - AddEndpoint(endpoint, status); - } - } else { - (iter->second).UpdateInfo(status, latency); - } +void HostInfo::SetLatency(const chrono::milliseconds& latency) { + lock_guard lock(mLatencyMux); + mLatency = latency; } -void SLSClientManager::RegionEndpointsInfo::RemoveEndpoint(const std::string& endpoint) { - mEndpointInfoMap.erase(endpoint); - if (mDefaultEndpoint == endpoint) { - mDefaultEndpoint.clear(); - } +void HostInfo::SetForbidden() { + lock_guard lock(mLatencyMux); + mLatency = chrono::milliseconds::max(); } -std::string SLSClientManager::RegionEndpointsInfo::GetAvailableEndpointWithTopPriority() const { - if (!mDefaultEndpoint.empty()) { - auto iter = mEndpointInfoMap.find(mDefaultEndpoint); - if (iter != mEndpointInfoMap.end() && (iter->second).mValid) { - return mDefaultEndpoint; +bool HostInfo::IsForbidden() const { + lock_guard lock(mLatencyMux); + return mLatency == chrono::milliseconds::max(); +} + +void CandidateHostsInfo::UpdateHosts(const CandidateEndpoints& regionEndpoints) { + lock_guard lock(mCandidateHostsMux); + switch (mMode) { +#ifdef __ENTERPRISE__ + case EndpointMode::DEFAULT: { + vector endpoints(regionEndpoints.mLocalEndpoints); + for (const auto& endpoint : regionEndpoints.mRemoteEndpoints) { + bool found = false; + for (const auto& existedEndpoint : regionEndpoints.mLocalEndpoints) { + if (existedEndpoint == endpoint) { + found = true; + break; + } + } + if (!found) { + endpoints.emplace_back(endpoint); + } + } + vector> infos; + for (const auto& endpoint : endpoints) { + string host = mProject.empty() ? endpoint : mProject + "." + endpoint; + if (mCandidateHosts.empty()) { + infos.emplace_back().emplace_back(host); + } else { + bool found = false; + for (const auto& item : mCandidateHosts) { + if (!item.empty() && item[0].GetHostname() == host) { + found = true; + infos.emplace_back(item); + break; + } + } + if (!found) { + infos.emplace_back().emplace_back(host); + } + } + } + mCandidateHosts.swap(infos); + break; } - } - std::string proxyEndpoint; - for (auto iter = mEndpointInfoMap.begin(); iter != mEndpointInfoMap.end(); ++iter) { - if (!(iter->second).mValid) { - continue; + case EndpointMode::ACCELERATE: { + vector endpoints{globalAccelerationEndpoint}; + for (const auto& item : regionEndpoints.mRemoteEndpoints) { + if (GetEndpointAddressType(item) == EndpointAddressType::PUBLIC) { + endpoints.emplace_back(item); + } + } + vector infos; + for (const auto& endpoint : endpoints) { + string host = mProject.empty() ? endpoint : mProject + "." + endpoint; + if (mCandidateHosts.empty()) { + infos.emplace_back(host); + } else { + bool found = false; + for (const auto& item : mCandidateHosts[0]) { + if (item.GetHostname() == host) { + found = true; + infos.emplace_back(item); + break; + } + } + if (!found) { + infos.emplace_back(host); + } + } + } + if (mCandidateHosts.empty()) { + mCandidateHosts.emplace_back(infos); + } else { + mCandidateHosts[0].swap(infos); + } + break; } - if ((iter->second).mValid && !(iter->second).mProxy) { - return iter->first; +#endif + case EndpointMode::CUSTOM: { + vector infos; + for (const auto& endpoint : regionEndpoints.mLocalEndpoints) { + string host = mProject.empty() ? endpoint : mProject + "." + endpoint; + if (mCandidateHosts.empty()) { + infos.emplace_back(host); + } else { + bool found = false; + for (const auto& item : mCandidateHosts[0]) { + if (item.GetHostname() == host) { + found = true; + infos.emplace_back(item); + break; + } + } + if (!found) { + infos.emplace_back(host); + } + } + } + if (mCandidateHosts.empty()) { + mCandidateHosts.emplace_back(infos); + } else { + mCandidateHosts[0].swap(infos); + } + break; } - proxyEndpoint = iter->first; - } - if (!proxyEndpoint.empty()) { - return proxyEndpoint; - } - if (!mDefaultEndpoint.empty()) { - return mDefaultEndpoint; - } - if (!mEndpointInfoMap.empty()) { - return mEndpointInfoMap.begin()->first; + default: + break; } - return mDefaultEndpoint; } -SLSClientManager* SLSClientManager::GetInstance() { -#ifdef __ENTERPRISE__ - static auto ptr = unique_ptr(new EnterpriseSLSClientManager()); -#else - static auto ptr = unique_ptr(new SLSClientManager()); -#endif - return ptr.get(); +bool CandidateHostsInfo::UpdateHostInfo(const string& hostname, const chrono::milliseconds& latency) { + lock_guard lock(mCandidateHostsMux); + for (auto& item : mCandidateHosts) { + for (auto& entry : item) { + if (entry.GetHostname() == hostname) { + entry.SetLatency(latency); + return true; + } + } + } + return false; } -void SLSClientManager::Init() { - InitEndpointSwitchPolicy(); - GenerateUserAgent(); - if (mDataServerSwitchPolicy == EndpointSwitchPolicy::DESIGNATED_FIRST) { - mProbeNetworkClient.reset(new sdk::Client("", - "", - INT32_FLAG(sls_client_send_timeout))); - mProbeNetworkClient->SetPort(AppConfig::GetInstance()->GetDataServerPort()); - mProbeNetworkThreadRes = async(launch::async, &SLSClientManager::ProbeNetworkThread, this); +void CandidateHostsInfo::GetProbeHosts(vector& hosts) const { + if (!HasValidHost()) { + return GetAllHosts(hosts); } - if (BOOL_FLAG(send_prefer_real_ip)) { - mUpdateRealIpClient.reset(new sdk::Client("", - "", - INT32_FLAG(sls_client_send_timeout))); - mUpdateRealIpClient->SetPort(AppConfig::GetInstance()->GetDataServerPort()); - mUpdateRealIpThreadRes = async(launch::async, &SLSClientManager::UpdateRealIpThread, this); + { + lock_guard lock(mCandidateHostsMux); + for (const auto& item : mCandidateHosts) { + for (const auto& entry : item) { + if (!entry.IsForbidden()) { + hosts.emplace_back(entry.GetHostname()); + } + } + } } } -void SLSClientManager::Stop() { - if (mDataServerSwitchPolicy == EndpointSwitchPolicy::DESIGNATED_FIRST) { - lock_guard lock(mProbeNetworkThreadRunningMux); - mIsProbeNetworkThreadRunning = false; - } - if (BOOL_FLAG(send_prefer_real_ip)) { - lock_guard lock(mUpdateRealIpThreadRunningMux); - mIsUpdateRealIpThreadRunning = false; - } - mStopCV.notify_all(); - if (mDataServerSwitchPolicy == EndpointSwitchPolicy::DESIGNATED_FIRST && mProbeNetworkThreadRes.valid()) { - future_status s = mProbeNetworkThreadRes.wait_for(chrono::seconds(1)); - if (s == future_status::ready) { - LOG_INFO(sLogger, ("sls endpoint probe", "stopped successfully")); - } else { - LOG_WARNING(sLogger, ("sls endpoint probe", "forced to stopped")); +void CandidateHostsInfo::GetAllHosts(vector& hosts) const { + lock_guard lock(mCandidateHostsMux); + for (const auto& item : mCandidateHosts) { + for (const auto& entry : item) { + hosts.emplace_back(entry.GetHostname()); } } - if (BOOL_FLAG(send_prefer_real_ip) && mUpdateRealIpThreadRes.valid()) { - future_status s = mUpdateRealIpThreadRes.wait_for(chrono::seconds(1)); - if (s == future_status::ready) { - LOG_INFO(sLogger, ("sls real ip update", "stopped successfully")); - } else { - LOG_WARNING(sLogger, ("sls real ip update", "forced to stopped")); +} + +void CandidateHostsInfo::SelectBestHost() { + lock_guard lock(mCandidateHostsMux); + for (size_t i = 0; i < mCandidateHosts.size(); ++i) { + const auto& hosts = mCandidateHosts[i]; + chrono::milliseconds minLatency = chrono::milliseconds::max(); + size_t minIdx = numeric_limits::max(); + for (size_t j = 0; j < hosts.size(); ++j) { + if (!hosts[j].IsForbidden() && hosts[j].GetLatency() < minLatency) { + minLatency = hosts[j].GetLatency(); + minIdx = j; + } + } + if (minIdx != numeric_limits::max()) { + const auto& hostname = hosts[minIdx].GetHostname(); + if (GetCurrentHost() != hostname) { + SetCurrentHost(hostname); + LOG_INFO(sLogger, + ("switch to the best host", hostname)("latency", minLatency.count())("project", mProject)( + "region", mRegion)("endpoint mode", EndpointModeToString(mMode))); + } + return; } } + SetCurrentHost(""); + LOG_INFO(sLogger, + ("no valid host", "stop sending data and retry later")("project", mProject)("region", mRegion)( + "endpoint mode", EndpointModeToString(mMode))); } -void SLSClientManager::InitEndpointSwitchPolicy() { - if (STRING_FLAG(data_endpoint_policy) == "designated_locked") { - mDataServerSwitchPolicy = EndpointSwitchPolicy::DESIGNATED_LOCKED; - } else if (STRING_FLAG(data_endpoint_policy) == "designated_first") { - mDataServerSwitchPolicy = EndpointSwitchPolicy::DESIGNATED_FIRST; - } else { - LOG_WARNING(sLogger, - ("param data_endpoint_policy is invalid, action", "use default value instead")("default value", - "designated_first")); - } +string CandidateHostsInfo::GetCurrentHost() const { + lock_guard lock(mCurrentHostMux); + return mCurrentHost; } -vector SLSClientManager::GetRegionAliuids(const std::string& region) { - lock_guard lock(mRegionAliuidRefCntMapLock); - vector aliuids; - for (const auto& item : mRegionAliuidRefCntMap[region]) { - aliuids.push_back(item.first); +string CandidateHostsInfo::GetFirstHost() const { + lock_guard lock(mCandidateHostsMux); + if (mCandidateHosts.empty() || mCandidateHosts[0].empty()) { + return ""; } - return aliuids; + return mCandidateHosts[0][0].GetHostname(); } -void SLSClientManager::IncreaseAliuidReferenceCntForRegion(const std::string& region, const std::string& aliuid) { - lock_guard lock(mRegionAliuidRefCntMapLock); - ++mRegionAliuidRefCntMap[region][aliuid]; +bool CandidateHostsInfo::HasValidHost() const { + lock_guard lock(mCurrentHostMux); + return !mCurrentHost.empty(); } -void SLSClientManager::DecreaseAliuidReferenceCntForRegion(const std::string& region, const std::string& aliuid) { - lock_guard lock(mRegionAliuidRefCntMapLock); - auto outerIter = mRegionAliuidRefCntMap.find(region); - if (outerIter == mRegionAliuidRefCntMap.end()) { - // should not happen - return; - } - auto innerIter = outerIter->second.find(aliuid); - if (innerIter == outerIter->second.end()) { - // should not happen - return; - } - if (--innerIter->second == 0) { - outerIter->second.erase(innerIter); - } - if (outerIter->second.empty()) { - mRegionAliuidRefCntMap.erase(outerIter); - } +void CandidateHostsInfo::SetCurrentHost(const string& host) { + lock_guard lock(mCurrentHostMux); + mCurrentHost = host; } -sdk::Client* SLSClientManager::GetClient(const string& region, const string& aliuid, bool createIfNotFound) { - string key = region + "_" + aliuid; - { - lock_guard lock(mClientMapMux); - auto iter = mClientMap.find(key); - if (iter != mClientMap.end()) { - (iter->second).second = time(NULL); - return (iter->second).first.get(); - } - } - if (!createIfNotFound) { - return nullptr; - } +// SLSClientManager::ProbeNetworkHttpRequest::ProbeNetworkHttpRequest( +// const string& region, const string& project, EndpointMode mode, const string& host, bool httpsFlag) +// : AsynHttpRequest(HTTP_GET, +// httpsFlag, +// host, +// httpsFlag ? 443 : 80, +// HEALTH, +// "", +// {}, +// "", +// HttpResponse(), +// INT32_FLAG(sls_hosts_probe_timeout), +// INT32_FLAG(sls_hosts_probe_max_try_cnt)), +// mRegion(region), +// mProject(project), +// mMode(mode), +// mHost(host) { +// } - string endpoint = GetAvailableEndpointWithTopPriority(region); - auto client = make_unique(aliuid, - endpoint, - INT32_FLAG(sls_client_send_timeout)); - ResetClientPort(region, client.get()); - LOG_INFO(sLogger, - ("init endpoint for sender, region", region)("uid", aliuid)("hostname", GetHostFromEndpoint(endpoint))( - "use https", ToString(client->IsUsingHTTPS()))); - auto ptr = client.get(); - { - lock_guard lock(mClientMapMux); - mClientMap.insert(make_pair(key, make_pair(std::move(client), time(nullptr)))); - } - return ptr; -} +// void SLSClientManager::ProbeNetworkHttpRequest::OnSendDone(HttpResponse& response) { +// if (mProject.empty()) { +// SLSClientManager::GetInstance()->UpdateHostInfo(mRegion, mHost, response.GetResponseTime()); +// } else { +// SLSClientManager::GetInstance()->UpdateHostInfo(mProject, mMode, mHost, response.GetResponseTime()); +// } +// } -bool SLSClientManager::ResetClientEndpoint(const string& aliuid, const string& region, time_t curTime) { - sdk::Client* sendClient = GetClient(region, aliuid, false); - if (sendClient == nullptr) { - return false; - } - if (curTime - sendClient->GetSlsHostUpdateTime() < INT32_FLAG(sls_host_update_interval)) { - return false; - } - sendClient->SetSlsHostUpdateTime(curTime); - string endpoint = GetAvailableEndpointWithTopPriority(region); - if (endpoint.empty()) { - return false; - } - string originalEndpoint = sendClient->GetRawSlsHost(); - if (originalEndpoint == endpoint) { - return false; - } - sendClient->SetSlsHost(endpoint); - ResetClientPort(region, sendClient); - LOG_INFO( - sLogger, - ("reset endpoint for sender, region", region)("uid", aliuid)("from", GetHostFromEndpoint(originalEndpoint))( - "to", GetHostFromEndpoint(endpoint))("use https", ToString(sendClient->IsUsingHTTPS()))); - return true; -} +// SLSClientManager::SLSClientManager() : mGetEndpointRealIp(GetEndpointRealIp) { +// } -void SLSClientManager::ResetClientPort(const string& region, sdk::Client* sendClient) { - sendClient->SetPort(AppConfig::GetInstance()->GetDataServerPort()); - if (AppConfig::GetInstance()->GetDataServerPort() == 80) { - lock_guard lock(mRegionEndpointEntryMapLock); - auto iter = mRegionEndpointEntryMap.find(region); - if (iter != mRegionEndpointEntryMap.end()) { - const string& defaultEndpoint = iter->second.mDefaultEndpoint; - if (!defaultEndpoint.empty()) { - if (IsHttpsEndpoint(defaultEndpoint)) { - sendClient->SetPort(443); - } - } else { - if (IsHttpsEndpoint(sendClient->GetRawSlsHost())) { - sendClient->SetPort(443); - } - } - } - } +SLSClientManager* SLSClientManager::GetInstance() { +#ifdef __ENTERPRISE__ + static auto ptr = unique_ptr(new EnterpriseSLSClientManager()); +#else + static auto ptr = unique_ptr(new SLSClientManager()); +#endif + return ptr.get(); } -void SLSClientManager::CleanTimeoutClient() { - lock_guard lock(mClientMapMux); - time_t curTime = time(nullptr); - for (auto iter = mClientMap.begin(); iter != mClientMap.end();) { - if ((curTime - (iter->second).second) > INT32_FLAG(send_client_timeout_interval)) { - iter = mClientMap.erase(iter); - } else { - ++iter; - } - } +void SLSClientManager::Init() { + GenerateUserAgent(); + + // mUnInitializedHostProbeClient = curl_multi_init(); + // if (mUnInitializedHostProbeClient == nullptr) { + // LOG_ERROR(sLogger, ("failed to init uninitialized host probe", "failed to init curl client")); + // } else { + // mUnInitializedHostProbeThreadRes = async(launch::async, &SLSClientManager::UnInitializedHostProbeThread, + // this); + // } + + // mHostProbeClient = curl_multi_init(); + // if (mHostProbeClient == nullptr) { + // LOG_ERROR(sLogger, ("failed to init host probe", "failed to init curl client")); + // } else { + // mHostProbeThreadRes = async(launch::async, &SLSClientManager::HostProbeThread, this); + // } + + // if (BOOL_FLAG(send_prefer_real_ip)) { + // mUpdateRealIpThreadRes = async(launch::async, &SLSClientManager::UpdateRealIpThread, this); + // } } -bool SLSClientManager::GetAccessKey(const std::string& aliuid, +// void SLSClientManager::Stop() { +// if (mUnInitializedHostProbeClient) { +// lock_guard lock(mUnInitializedHostProbeThreadRunningMux); +// mIsUnInitializedHostProbeThreadRunning = false; +// } +// if (mHostProbeClient) { +// lock_guard lock(mHostProbeThreadRunningMux); +// mIsHostProbeThreadRunning = false; +// } +// if (BOOL_FLAG(send_prefer_real_ip)) { +// lock_guard lock(mUpdateRealIpThreadRunningMux); +// mIsUpdateRealIpThreadRunning = false; +// } +// mStopCV.notify_all(); + +// if (mUnInitializedHostProbeClient) { +// future_status s = mUnInitializedHostProbeThreadRes.wait_for(chrono::seconds(1)); +// if (s == future_status::ready) { +// LOG_INFO(sLogger, ("sls uninitialized host probe", "stopped successfully")); +// } else { +// LOG_WARNING(sLogger, ("sls uninitialized host probe", "forced to stopped")); +// } +// } +// if (mHostProbeClient) { +// future_status s = mHostProbeThreadRes.wait_for(chrono::seconds(1)); +// if (s == future_status::ready) { +// LOG_INFO(sLogger, ("sls host probe", "stopped successfully")); +// } else { +// LOG_WARNING(sLogger, ("sls host probe", "forced to stopped")); +// } +// } +// if (BOOL_FLAG(send_prefer_real_ip) && mUpdateRealIpThreadRes.valid()) { +// future_status s = mUpdateRealIpThreadRes.wait_for(chrono::seconds(1)); +// if (s == future_status::ready) { +// LOG_INFO(sLogger, ("sls real ip update", "stopped successfully")); +// } else { +// LOG_WARNING(sLogger, ("sls real ip update", "forced to stopped")); +// } +// } +// } + +bool SLSClientManager::GetAccessKey(const string& aliuid, AuthType& type, - std::string& accessKeyId, - std::string& accessKeySecret) { + string& accessKeyId, + string& accessKeySecret) { accessKeyId = STRING_FLAG(default_access_key_id); accessKeySecret = STRING_FLAG(default_access_key); type = AuthType::AK; return true; } -void SLSClientManager::AddEndpointEntry(const string& region, - const string& endpoint, - bool isProxy, - const EndpointSourceType& endpointType) { - lock_guard lock(mRegionEndpointEntryMapLock); - RegionEndpointsInfo& info = mRegionEndpointEntryMap[region]; - if (!isProxy) { - bool isDefault = false; - if (info.AddDefaultEndpoint(endpoint, endpointType, isDefault)) { - LOG_INFO(sLogger, - ("add data server endpoint, region", region)("endpoint", endpoint)( - "isDefault", isDefault ? "yes" : "no")("isProxy", "false")("#endpoint", - info.mEndpointInfoMap.size())); - } - } else { - if (info.AddEndpoint(endpoint, true, isProxy)) { - LOG_INFO(sLogger, - ("add data server endpoint, region", region)("endpoint", endpoint)("isProxy", ToString(isProxy))( - "#endpoint", info.mEndpointInfoMap.size())); - } - } -} +// void SLSClientManager::UpdateLocalRegionEndpointsAndHttpsInfo(const string& region, +// const vector& rawEndpoints) { +// vector endpoints; +// for (const auto& item : rawEndpoints) { +// auto tmp = StandardizeEndpoint(item); +// if (!tmp.empty()) { +// endpoints.emplace_back(tmp); +// } +// } -void SLSClientManager::UpdateEndpointStatus(const string& region, - const string& endpoint, - bool status, - optional latency) { - lock_guard lock(mRegionEndpointEntryMapLock); - auto iter = mRegionEndpointEntryMap.find(region); - if (iter != mRegionEndpointEntryMap.end()) { - (iter->second).UpdateEndpointInfo(endpoint, status, latency, false); - } -} +// lock_guard lock(mRegionCandidateEndpointsMapMux); +// auto& candidate = mRegionCandidateEndpointsMap[region]; +// candidate.mMode = EndpointMode::DEFAULT; +// candidate.mLocalEndpoints.clear(); +// for (const auto& item : endpoints) { +// // if both acclerate and custom endpoints are given, we ignore custom endpoints +// if (item == globalAccelerationEndpoint) { +// candidate.mMode = EndpointMode::ACCELERATE; +// break; +// } +// if (IsCustomEndpoint(item)) { +// candidate.mMode = EndpointMode::CUSTOM; +// candidate.mLocalEndpoints.emplace_back(item); +// } +// } +// if (candidate.mMode == EndpointMode::ACCELERATE) { +// candidate.mLocalEndpoints.clear(); +// } else if (candidate.mMode == EndpointMode::DEFAULT) { +// candidate.mLocalEndpoints = endpoints; +// } -string SLSClientManager::GetAvailableEndpointWithTopPriority(const string& region) const { - static string emptyStr = ""; - lock_guard lock(mRegionEndpointEntryMapLock); - auto iter = mRegionEndpointEntryMap.find(region); - if (iter != mRegionEndpointEntryMap.end()) { - return (iter->second).GetAvailableEndpointWithTopPriority(); - } - return emptyStr; -} +// // as long as one endpoint is https, we treat the region as https +// bool isHttps = false; +// for (const auto& item : rawEndpoints) { +// if (IsHttpsEndpoint(item)) { +// isHttps = true; +// break; +// } +// } +// { +// lock_guard lock(mHttpsRegionsMux); +// if (isHttps) { +// mHttpsRegions.insert(region); +// } else { +// mHttpsRegions.erase(region); +// } +// } +// } -string SLSClientManager::GetRegionFromEndpoint(const string& endpoint) { - lock_guard lock(mRegionEndpointEntryMapLock); - for (auto iter = mRegionEndpointEntryMap.begin(); iter != mRegionEndpointEntryMap.end(); ++iter) { - for (auto epIter = ((iter->second).mEndpointInfoMap).begin(); epIter != ((iter->second).mEndpointInfoMap).end(); - ++epIter) { - if (epIter->first == endpoint) - return iter->first; - } - } - return STRING_FLAG(default_region_name); -} +// void SLSClientManager::UpdateRemoteRegionEndpoints(const string& region, +// const vector& rawEndpoints, +// RemoteEndpointUpdateAction action) { +// vector endpoints; +// for (const auto& item : rawEndpoints) { +// auto tmp = StandardizeEndpoint(item); +// if (!tmp.empty()) { +// endpoints.emplace_back(tmp); +// } +// } +// lock_guard lock(mRegionCandidateEndpointsMapMux); -bool SLSClientManager::HasNetworkAvailable() { - static time_t lastCheckTime = time(nullptr); - time_t curTime = time(nullptr); - if (curTime - lastCheckTime >= 3600) { - lastCheckTime = curTime; - return true; - } - { - lock_guard lock(mRegionEndpointEntryMapLock); - for (auto iter = mRegionEndpointEntryMap.begin(); iter != mRegionEndpointEntryMap.end(); ++iter) { - for (auto epIter = ((iter->second).mEndpointInfoMap).begin(); - epIter != ((iter->second).mEndpointInfoMap).end(); - ++epIter) { - if ((epIter->second).mValid) { - return true; - } +// if (action == RemoteEndpointUpdateAction::CREATE +// && mRegionCandidateEndpointsMap.find(region) != mRegionCandidateEndpointsMap.end()) { +// return; +// } +// auto& candidate = mRegionCandidateEndpointsMap[region]; +// if (action == RemoteEndpointUpdateAction::APPEND) { +// for (const auto& item : endpoints) { +// bool found = false; +// for (const auto& existed : candidate.mRemoteEndpoints) { +// if (existed == item) { +// found = true; +// break; +// } +// } +// if (!found) { +// candidate.mRemoteEndpoints.emplace_back(item); +// } +// } +// } else { +// candidate.mRemoteEndpoints = endpoints; +// } +// { +// lock_guard lock(mCandidateHostsInfosMapMux); +// for (auto& item : mRegionCandidateHostsInfosMap[region]) { +// if (item.expired()) { +// continue; +// } +// auto info = item.lock(); +// info->UpdateHosts(candidate); +// } +// } +// if (BOOL_FLAG(send_prefer_real_ip)) { +// lock_guard lock(mRegionRealIpCandidateHostsInfosMapMux); +// auto it = mRegionRealIpCandidateHostsInfosMap.find(region); +// if (it == mRegionRealIpCandidateHostsInfosMap.end()) { +// it = mRegionRealIpCandidateHostsInfosMap.try_emplace(region, "", region, EndpointMode::DEFAULT).first; +// } +// it->second.UpdateHosts(candidate); +// } +// } + +// shared_ptr +// SLSClientManager::GetCandidateHostsInfo(const string& region, const string& project, EndpointMode mode) { +// if (mode == EndpointMode::DEFAULT) { +// // when flusher does not specify the mode, we use the mode of the region, which is set before flusher init +// and +// // will not change. +// lock_guard lock(mRegionCandidateEndpointsMapMux); +// auto it = mRegionCandidateEndpointsMap.find(region); +// if (it != mRegionCandidateEndpointsMap.end()) { +// mode = it->second.mMode; +// } +// } +// { +// lock_guard lock(mCandidateHostsInfosMapMux); +// auto& hostsInfo = mProjectCandidateHostsInfosMap[project]; +// for (auto& item : hostsInfo) { +// if (item.expired()) { +// continue; +// } +// auto info = item.lock(); +// if (info->GetMode() == mode) { +// return info; +// } +// } +// } + +// auto info = make_shared(project, region, mode); +// CandidateEndpoints endpoints; +// { +// lock_guard lock(mRegionCandidateEndpointsMapMux); +// auto it = mRegionCandidateEndpointsMap.find(region); +// if (it != mRegionCandidateEndpointsMap.end()) { +// endpoints = it->second; +// } +// } +// info->UpdateHosts(endpoints); +// { +// lock_guard lock(mCandidateHostsInfosMapMux); +// mProjectCandidateHostsInfosMap[project].emplace_back(info); +// mRegionCandidateHostsInfosMap[region].emplace_back(info); +// } +// { +// lock_guard lock(mUnInitializedCandidateHostsInfosMux); +// mUnInitializedCandidateHostsInfos.emplace_back(info); +// } +// return info; +// } + +// bool SLSClientManager::UpdateHostInfo(const string& project, +// EndpointMode mode, +// const string& host, +// const chrono::milliseconds& latency) { +// lock_guard lock(mCandidateHostsInfosMapMux); +// auto it = mProjectCandidateHostsInfosMap.find(project); +// if (it == mProjectCandidateHostsInfosMap.end()) { +// return false; +// } +// for (auto& entry : it->second) { +// if (entry.expired()) { +// continue; +// } +// auto info = entry.lock(); +// if (info->GetMode() == mode) { +// info->UpdateHostInfo(host, latency); +// return true; +// } +// } +// return false; +// } + +// bool SLSClientManager::UpdateHostInfo(const string& region, const string& host, const chrono::milliseconds& latency) +// { +// lock_guard lock(mRegionRealIpCandidateHostsInfosMapMux); +// auto it = mRegionRealIpCandidateHostsInfosMap.find(region); +// if (it == mRegionRealIpCandidateHostsInfosMap.end()) { +// return false; +// } +// return it->second.UpdateHostInfo(host, latency); +// } + +shared_ptr SLSClientManager::GetCandidateHostsInfo(const string& project, const string& endpoint) { + if (endpoint.empty()) { + // this should only occur on first update, where we try find any available info + lock_guard lock(mCandidateHostsInfosMapMux); + auto& hostsInfo = mProjectCandidateHostsInfosMap[project]; + for (auto& item : hostsInfo) { + if (item.expired()) { + continue; } + return item.lock(); } + return nullptr; } - return false; -} -void SLSClientManager::ProbeNetworkThread() { - LOG_INFO(sLogger, ("sls endpoint probe", "started")); - // pair represents the weight of each endpoint - map>> unavaliableEndpoints; - set unavaliableRegions; - int32_t lastCheckAllTime = 0; - unique_lock lock(mProbeNetworkThreadRunningMux); - while (mIsProbeNetworkThreadRunning) { - unavaliableEndpoints.clear(); - unavaliableRegions.clear(); - { - lock_guard lock(mRegionEndpointEntryMapLock); - for (auto iter = mRegionEndpointEntryMap.begin(); iter != mRegionEndpointEntryMap.end(); ++iter) { - auto& endpoints = unavaliableEndpoints[iter->first]; - bool unavaliable = true; - for (auto epIter = ((iter->second).mEndpointInfoMap).begin(); - epIter != ((iter->second).mEndpointInfoMap).end(); - ++epIter) { - if (!(epIter->second).mValid) { - if (epIter->first == iter->second.mDefaultEndpoint) { - endpoints.emplace_back(0, epIter->first); - } else { - endpoints.emplace_back(10, epIter->first); - } - } else { - unavaliable = false; - } - } - sort(endpoints.begin(), endpoints.end()); - if (unavaliable) { - unavaliableRegions.insert(iter->first); - } - } - } - if (unavaliableEndpoints.empty()) { - if (mStopCV.wait_for(lock, chrono::seconds(INT32_FLAG(test_network_normal_interval)), [this]() { - return !mIsProbeNetworkThreadRunning; - })) { - break; + string standardEndpoint = StandardizeEndpoint(endpoint); + { + lock_guard lock(mCandidateHostsInfosMapMux); + auto& hostsInfo = mProjectCandidateHostsInfosMap[project]; + for (auto& item : hostsInfo) { + if (item.expired()) { + continue; } - continue; - } - int32_t curTime = time(NULL); - // bool wakeUp = false; - for (const auto& value : unavaliableEndpoints) { - const string& region = value.first; - vector uids = GetRegionAliuids(region); - bool endpointChanged = false; - for (const auto& item : value.second) { - const string& endpoint = item.second; - const int32_t priority = item.first; - if (unavaliableRegions.find(region) == unavaliableRegions.end()) { - if (!endpointChanged && priority != 10) { - if (TestEndpoint(region, endpoint)) { - for (const auto& uid : uids) { - ResetClientEndpoint(uid, region, curTime); - } - endpointChanged = true; - } - } else { - if (curTime - lastCheckAllTime >= 1800) { - TestEndpoint(region, endpoint); - } - } - } else { - if (TestEndpoint(region, endpoint)) { - // wakeUp = true; - // Sender::GetInstance()->OnRegionRecover(region); - if (!endpointChanged) { - for (const auto& uid : uids) { - ResetClientEndpoint(uid, region, curTime); - } - endpointChanged = true; - } - } - } + auto info = item.lock(); + if (info->GetFirstHost() == project + "." + standardEndpoint) { + return info; } } - // if (wakeUp && (!mIsSendingBuffer)) { - // mSenderQueue.Signal(); - // } - if (curTime - lastCheckAllTime >= 1800) { - lastCheckAllTime = curTime; - } - if (mStopCV.wait_for(lock, chrono::seconds(INT32_FLAG(test_unavailable_endpoint_interval)), [this]() { - return !mIsProbeNetworkThreadRunning; - })) { - break; - } } -} - -bool SLSClientManager::TestEndpoint(const string& region, const string& endpoint) { - // TODO: this should be removed, since control-plane status is not the same as data-plane - if (!FlusherSLS::GetRegionStatus(region)) { - return false; - } - if (endpoint.empty()) { - return false; - } - mProbeNetworkClient->SetSlsHost(endpoint); - ResetClientPort(region, mProbeNetworkClient.get()); - - bool status = true; - uint64_t beginTime = GetCurrentTimeInMicroSeconds(); - try { - status = mProbeNetworkClient->TestNetwork(); - } catch (sdk::LOGException& ex) { - if (ConvertErrorCode(ex.GetErrorCode()) == SEND_NETWORK_ERROR) { - status = false; - } - } catch (...) { - LOG_ERROR(sLogger, ("test network", "send fail")("exception", "unknown")); - status = false; + auto info = make_shared(project, "", EndpointMode::CUSTOM); + info->UpdateHosts({EndpointMode::CUSTOM, {standardEndpoint}}); + { + lock_guard lock(mCandidateHostsInfosMapMux); + mProjectCandidateHostsInfosMap[project].emplace_back(info); } - uint32_t latency = (GetCurrentTimeInMicroSeconds() - beginTime) / 1000; - LOG_DEBUG(sLogger, ("TestEndpoint, region", region)("endpoint", endpoint)("status", status)("latency", latency)); - UpdateEndpointStatus(region, endpoint, status, latency); - return status; + return info; } -void SLSClientManager::ForceUpdateRealIp(const string& region) { - lock_guard lock(mRegionRealIpLock); - auto iter = mRegionRealIpMap.find(region); - if (iter != mRegionRealIpMap.end()) { - iter->second->mForceFlushFlag = true; +bool SLSClientManager::UsingHttps(const string& region) const { + if (AppConfig::GetInstance()->GetDataServerPort() == 443) { + return true; } + // lock_guard lock(mHttpsRegionsMux); + // return mHttpsRegions.find(region) != mHttpsRegions.end(); } -void SLSClientManager::UpdateSendClientRealIp(sdk::Client* client, const string& region) { - string realIp; - RealIpInfo* pInfo = NULL; - { - lock_guard lock(mRegionRealIpLock); - auto iter = mRegionRealIpMap.find(region); - if (iter != mRegionRealIpMap.end()) { - pInfo = iter->second; - } else { - pInfo = new RealIpInfo; - mRegionRealIpMap.insert(make_pair(region, pInfo)); - } - realIp = pInfo->mRealIp; - } - if (!realIp.empty()) { - client->SetSlsHost(realIp); - client->SetSlsRealIpUpdateTime(time(NULL)); - } else if (pInfo->mLastUpdateTime >= client->GetSlsRealIpUpdateTime()) { - const string& defaultEndpoint = GetAvailableEndpointWithTopPriority(region); - if (!defaultEndpoint.empty()) { - client->SetSlsHost(defaultEndpoint); - client->SetSlsRealIpUpdateTime(time(NULL)); - } - } +string SLSClientManager::GetRegionFromEndpoint(const string& endpoint) { + // lock_guard lock(mRegionEndpointEntryMapLock); + // for (auto iter = mRegionEndpointEntryMap.begin(); iter != mRegionEndpointEntryMap.end(); ++iter) { + // for (auto epIter = ((iter->second).mEndpointInfoMap).begin(); epIter != + // ((iter->second).mEndpointInfoMap).end(); + // ++epIter) { + // if (epIter->first == endpoint) + // return iter->first; + // } + // } + return STRING_FLAG(default_region_name); } -void SLSClientManager::UpdateRealIpThread() { - LOG_INFO(sLogger, ("sls real ip update", "started")); - int32_t lastUpdateRealIpTime = 0; - vector regionEndpointArray; - vector regionArray; - unique_lock lock(mUpdateRealIpThreadRunningMux); - while (mIsUpdateRealIpThreadRunning) { - int32_t curTime = time(NULL); - bool updateFlag = curTime - lastUpdateRealIpTime > INT32_FLAG(send_switch_real_ip_interval); - { - // check force update - lock_guard lock(mRegionRealIpLock); - auto iter = mRegionRealIpMap.begin(); - for (; iter != mRegionRealIpMap.end(); ++iter) { - if (iter->second->mForceFlushFlag) { - iter->second->mForceFlushFlag = false; - updateFlag = true; - LOG_INFO(sLogger, ("force update real ip", iter->first)); - } - } - } - if (updateFlag) { - LOG_DEBUG(sLogger, ("start update real ip", "")); - regionEndpointArray.clear(); - regionArray.clear(); - { - lock_guard lock(mRegionEndpointEntryMapLock); - auto iter = mRegionEndpointEntryMap.begin(); - for (; iter != mRegionEndpointEntryMap.end(); ++iter) { - regionEndpointArray.push_back((iter->second).GetAvailableEndpointWithTopPriority()); - regionArray.push_back(iter->first); - } - } - for (size_t i = 0; i < regionEndpointArray.size(); ++i) { - // no available endpoint - if (regionEndpointArray[i].empty()) { - continue; - } +// void SLSClientManager::UnInitializedHostProbeThread() { +// LOG_INFO(sLogger, ("sls uninitialized host probe", "started")); +// unique_lock lock(mUnInitializedHostProbeThreadRunningMux); +// while (mIsUnInitializedHostProbeThreadRunning) { +// DoProbeUnInitializedHost(); +// if (mStopCV.wait_for( +// lock, chrono::milliseconds(10), [this]() { return !mIsUnInitializedHostProbeThreadRunning; })) { +// return; +// } +// } - EndpointStatus status = UpdateRealIp(regionArray[i], regionEndpointArray[i]); - if (status == EndpointStatus::STATUS_ERROR) { - UpdateEndpointStatus(regionArray[i], regionEndpointArray[i], false); - } - } - lastUpdateRealIpTime = time(NULL); - } - if (mStopCV.wait_for(lock, chrono::seconds(1), [this]() { return !mIsUpdateRealIpThreadRunning; })) { - break; - } - } -} +// auto mc = curl_multi_cleanup(mUnInitializedHostProbeClient); +// if (mc != CURLM_OK) { +// LOG_ERROR(sLogger, ("failed to cleanup curl multi handle", "exit anyway")("errMsg", +// curl_multi_strerror(mc))); +// } +// } -SLSClientManager::EndpointStatus SLSClientManager::UpdateRealIp(const string& region, const string& endpoint) { - mUpdateRealIpClient->SetSlsHost(endpoint); - EndpointStatus status = EndpointStatus::STATUS_ERROR; - int64_t beginTime = GetCurrentTimeInMicroSeconds(); - try { - sdk::GetRealIpResponse rsp; - rsp = mUpdateRealIpClient->GetRealIp(); - - if (!rsp.realIp.empty()) { - SetRealIp(region, rsp.realIp); - status = EndpointStatus::STATUS_OK_WITH_IP; - } else { - status = EndpointStatus::STATUS_OK_WITH_ENDPOINT; - static int32_t sUpdateRealIpWarningCount = 0; - if (sUpdateRealIpWarningCount++ % 100 == 0) { - sUpdateRealIpWarningCount %= 100; - LOG_WARNING(sLogger, - ("get real ip request succeeded but server did not give real ip, region", - region)("endpoint", endpoint)); - } +// void SLSClientManager::DoProbeUnInitializedHost() { +// vector> infos; +// { +// lock_guard lock(mUnInitializedCandidateHostsInfosMux); +// infos.swap(mUnInitializedCandidateHostsInfos); +// } +// if (infos.empty()) { +// return; +// } +// #ifndef APSARA_UNIT_TEST_MAIN +// for (auto& item : infos) { +// if (item.expired()) { +// continue; +// } +// auto info = item.lock(); +// bool httpsFlag = UsingHttps(info->GetRegion()); +// auto req = make_unique( +// info->GetRegion(), info->GetProject(), info->GetMode(), info->GetFirstHost(), httpsFlag); +// AddRequestToMultiCurlHandler(mUnInitializedHostProbeClient, std::move(req)); +// } +// SendAsynRequests(mUnInitializedHostProbeClient); +// #else +// for (auto& item : infos) { +// if (item.expired()) { +// continue; +// } +// auto info = item.lock(); +// bool httpsFlag = UsingHttps(info->GetRegion()); +// auto req = make_unique( +// info->GetRegion(), info->GetProject(), info->GetMode(), info->GetFirstHost(), httpsFlag); +// HttpResponse response = mDoProbeNetwork(req); +// req->OnSendDone(response); +// } +// #endif - // we should set real ip to empty string if server did not give real ip - SetRealIp(region, ""); - } - } - // GetRealIp's implement should not throw LOGException, but we catch it to hold implement changing - catch (sdk::LOGException& ex) { - const string& errorCode = ex.GetErrorCode(); - LOG_DEBUG(sLogger, ("get real ip", "send fail")("errorCode", errorCode)("errorMessage", ex.GetMessage())); - SendResult sendRst = ConvertErrorCode(errorCode); - if (sendRst == SEND_NETWORK_ERROR) - status = EndpointStatus::STATUS_ERROR; - } catch (...) { - LOG_ERROR(sLogger, ("get real ip", "send fail")("exception", "unknown")); - } - int64_t endTime = GetCurrentTimeInMicroSeconds(); - int32_t latency = int32_t((endTime - beginTime) / 1000); // ms - LOG_DEBUG(sLogger, - ("Get real ip, region", region)("endpoint", endpoint)("status", int(status))("latency", latency)); - return status; -} +// for (auto& item : infos) { +// if (item.expired()) { +// continue; +// } +// item.lock()->SelectBestHost(); +// } +// { +// lock_guard lock(mPartiallyInitializedCandidateHostsInfosMux); +// mPartiallyInitializedCandidateHostsInfos.insert( +// mPartiallyInitializedCandidateHostsInfos.end(), infos.begin(), infos.end()); +// } +// } -void SLSClientManager::SetRealIp(const string& region, const string& ip) { - lock_guard lock(mRegionRealIpLock); - RealIpInfo* pInfo = NULL; - auto iter = mRegionRealIpMap.find(region); - if (iter != mRegionRealIpMap.end()) { - pInfo = iter->second; - } else { - pInfo = new RealIpInfo; - mRegionRealIpMap.insert(make_pair(region, pInfo)); - } - LOG_DEBUG(sLogger, ("set real ip, last", pInfo->mRealIp)("now", ip)("region", region)); - pInfo->SetRealIp(ip); -} +// void SLSClientManager::HostProbeThread() { +// LOG_INFO(sLogger, ("sls host probe", "started")); +// unique_lock lock(mHostProbeThreadRunningMux); +// while (mIsHostProbeThreadRunning) { +// DoProbeHost(); +// if (mStopCV.wait_for(lock, chrono::seconds(1), [this]() { return !mIsHostProbeThreadRunning; })) { +// return; +// } +// } + +// auto mc = curl_multi_cleanup(mHostProbeClient); +// if (mc != CURLM_OK) { +// LOG_ERROR(sLogger, ("failed to cleanup curl multi handle", "exit anyway")("errMsg", +// curl_multi_strerror(mc))); +// } +// } + +// void SLSClientManager::DoProbeHost() { +// static time_t lastProbeAllEndpointsTime = time(nullptr); +// static time_t lastProbeAvailableEndpointsTime = time(nullptr); +// bool shouldTestAllEndpoints = false; +// bool shouldTestAvailableEndpoints = false; +// time_t beginTime = time(nullptr); +// if (beginTime - lastProbeAllEndpointsTime >= INT32_FLAG(sls_all_hosts_probe_interval_sec)) { +// shouldTestAllEndpoints = true; +// lastProbeAllEndpointsTime = beginTime; +// } else if (beginTime - lastProbeAvailableEndpointsTime >= INT32_FLAG(sls_hosts_probe_interval_sec)) { +// shouldTestAvailableEndpoints = true; +// lastProbeAvailableEndpointsTime = beginTime; +// } else if (!HasPartiallyInitializedCandidateHostsInfos()) { +// ClearExpiredCandidateHostsInfos(); +// PackIdManager::GetInstance()->CleanTimeoutEntry(); +// return; +// } + +// // collect hosts to be probed +// map, pair>> projectHostMap; +// map> regionHostsMap; // only for realip +// if (shouldTestAllEndpoints || shouldTestAvailableEndpoints) { +// { +// lock_guard lk(mCandidateHostsInfosMapMux); +// for (const auto& item : mProjectCandidateHostsInfosMap) { +// for (const auto& entry : item.second) { +// if (entry.expired()) { +// continue; +// } +// auto info = entry.lock(); +// auto& hosts = projectHostMap[make_pair(item.first, info->GetMode())]; +// hosts.first = info->GetRegion(); +// if (shouldTestAllEndpoints) { +// info->GetAllHosts(hosts.second); +// } else { +// info->GetProbeHosts(hosts.second); +// } +// } +// } +// } +// if (BOOL_FLAG(send_prefer_real_ip)) { +// lock_guard lk(mRegionRealIpCandidateHostsInfosMapMux); +// for (const auto& item : mRegionRealIpCandidateHostsInfosMap) { +// auto& hosts = regionHostsMap[item.second.GetRegion()]; +// if (shouldTestAllEndpoints) { +// item.second.GetAllHosts(hosts); +// } else { +// item.second.GetProbeHosts(hosts); +// } +// } +// } +// } else { +// lock_guard lk(mPartiallyInitializedCandidateHostsInfosMux); +// for (const auto& item : mPartiallyInitializedCandidateHostsInfos) { +// if (item.expired()) { +// continue; +// } +// auto info = item.lock(); +// auto& hosts = projectHostMap[make_pair(info->GetProject(), info->GetMode())]; +// hosts.first = info->GetRegion(); +// info->GetAllHosts(hosts.second); +// } +// mPartiallyInitializedCandidateHostsInfos.clear(); +// } + +// // do probe and update latency +// for (const auto& item : projectHostMap) { +// bool httpsFlag = UsingHttps(item.second.first); +// #ifndef APSARA_UNIT_TEST_MAIN +// for (const auto& host : item.second.second) { +// auto req = make_unique( +// item.second.first, item.first.first, item.first.second, host, httpsFlag); +// AddRequestToMultiCurlHandler(mHostProbeClient, std::move(req)); +// } +// SendAsynRequests(mHostProbeClient); +// #else +// for (const auto& host : item.second.second) { +// auto req = make_unique( +// item.second.first, item.first.first, item.first.second, host, httpsFlag); +// HttpResponse response = mDoProbeNetwork(req); +// req->OnSendDone(response); +// } +// #endif +// } +// if (BOOL_FLAG(send_prefer_real_ip)) { +// for (const auto& item : regionHostsMap) { +// bool httpsFlag = UsingHttps(item.first); +// #ifndef APSARA_UNIT_TEST_MAIN +// for (const auto& host : item.second) { +// auto req = make_unique(item.first, "", EndpointMode::DEFAULT, host, +// httpsFlag); AddRequestToMultiCurlHandler(mHostProbeClient, std::move(req)); +// } +// SendAsynRequests(mHostProbeClient); +// #else +// for (const auto& host : item.second) { +// auto req = make_unique(item.first, "", EndpointMode::DEFAULT, host, +// httpsFlag); HttpResponse response = mDoProbeNetwork(req); req->OnSendDone(response); +// } +// #endif +// } +// } + +// // update selected host +// { +// lock_guard lk(mCandidateHostsInfosMapMux); +// for (const auto& item : mProjectCandidateHostsInfosMap) { +// for (auto& entry : item.second) { +// if (auto p = entry.lock()) { +// p->SelectBestHost(); +// } +// } +// } +// } +// if (BOOL_FLAG(send_prefer_real_ip) && (shouldTestAllEndpoints || shouldTestAvailableEndpoints)) { +// lock_guard lk(mRegionRealIpCandidateHostsInfosMapMux); +// for (auto& item : mRegionRealIpCandidateHostsInfosMap) { +// item.second.SelectBestHost(); +// } +// } +// } + +// bool SLSClientManager::HasPartiallyInitializedCandidateHostsInfos() const { +// lock_guard lk(mPartiallyInitializedCandidateHostsInfosMux); +// return !mPartiallyInitializedCandidateHostsInfos.empty(); +// } + +// void SLSClientManager::ClearExpiredCandidateHostsInfos() { +// lock_guard lk(mCandidateHostsInfosMapMux); +// for (auto iter = mProjectCandidateHostsInfosMap.begin(); iter != mProjectCandidateHostsInfosMap.end();) { +// for (auto it = iter->second.begin(); it != iter->second.end();) { +// if (it->expired()) { +// it = iter->second.erase(it); +// } else { +// ++it; +// } +// } +// if (iter->second.empty()) { +// iter = mProjectCandidateHostsInfosMap.erase(iter); +// } else { +// ++iter; +// } +// } +// for (auto iter = mRegionCandidateHostsInfosMap.begin(); iter != mRegionCandidateHostsInfosMap.end();) { +// for (auto it = iter->second.begin(); it != iter->second.end();) { +// if (it->expired()) { +// it = iter->second.erase(it); +// } else { +// ++it; +// } +// } +// if (iter->second.empty()) { +// iter = mRegionCandidateHostsInfosMap.erase(iter); +// } else { +// ++iter; +// } +// } +// } + +// void SLSClientManager::UpdateRealIpThread() { +// LOG_INFO(sLogger, ("sls real ip update", "started")); +// unique_lock lock(mUpdateRealIpThreadRunningMux); +// while (mIsUpdateRealIpThreadRunning) { +// DoUpdateRealIp(); +// if (mStopCV.wait_for(lock, chrono::seconds(1), [this]() { return !mIsUpdateRealIpThreadRunning; })) { +// break; +// } +// } +// } + +// void SLSClientManager::DoUpdateRealIp() { +// vector> hosts; +// static time_t lastUpdateRealIpTime = 0; +// if (time(nullptr) - lastUpdateRealIpTime >= INT32_FLAG(send_switch_real_ip_interval)) { +// { +// lock_guard lock(mOutdatedRealIpRegionsMux); +// mOutdatedRealIpRegions.clear(); +// } +// { +// lock_guard lock(mRegionRealIpCandidateHostsInfosMapMux); +// for (const auto& item : mRegionRealIpCandidateHostsInfosMap) { +// hosts.emplace_back(item.first, item.second.GetCurrentHost()); +// } +// } +// lastUpdateRealIpTime = time(nullptr); +// } else if (HasOutdatedRealIpRegions()) { +// vector regions; +// { +// lock_guard lock(mOutdatedRealIpRegionsMux); +// regions.swap(mOutdatedRealIpRegions); +// } +// { +// lock_guard lock(mRegionRealIpCandidateHostsInfosMapMux); +// for (const auto& item : regions) { +// auto it = mRegionRealIpCandidateHostsInfosMap.find(item); +// if (it != mRegionRealIpCandidateHostsInfosMap.end()) { +// hosts.emplace_back(item, it->second.GetCurrentHost()); +// } +// } +// } +// } +// for (const auto& item : hosts) { +// if (item.second.empty()) { +// continue; +// } +// string ip; +// if (!mGetEndpointRealIp(item.second, ip)) { +// LOG_WARNING(sLogger, +// ("failed to get real ip", "retry later")("region", item.first)("endpoint", item.second)); +// continue; +// } +// SetRealIp(item.first, ip); +// } +// } + +// bool SLSClientManager::HasOutdatedRealIpRegions() const { +// lock_guard lock(mOutdatedRealIpRegionsMux); +// return !mOutdatedRealIpRegions.empty(); +// } + +// void SLSClientManager::UpdateOutdatedRealIpRegions(const string& region) { +// lock_guard lock(mOutdatedRealIpRegionsMux); +// mOutdatedRealIpRegions.emplace_back(region); +// } + +// string SLSClientManager::GetRealIp(const string& region) const { +// lock_guard lock(mRegionRealIpMux); +// auto it = mRegionRealIpMap.find(region); +// if (it == mRegionRealIpMap.end()) { +// return ""; +// } +// return it->second; +// } + +// void SLSClientManager::SetRealIp(const string& region, const string& ip) { +// lock_guard lock(mRegionRealIpMux); +// auto it = mRegionRealIpMap.find(region); +// if (it == mRegionRealIpMap.end()) { +// mRegionRealIpMap.emplace(region, ip); +// } else if (it->second != ip) { +// LOG_INFO(sLogger, ("set real ip for region", region)("from", it->second)("to", ip)); +// it->second = ip; +// } +// } void SLSClientManager::GenerateUserAgent() { string os; @@ -714,7 +1040,7 @@ string SLSClientManager::GetRunningEnvironment() { // containers in K8S will possess the above env if (AppConfig::GetInstance()->IsPurageContainerMode()) { env = "K8S-Daemonset"; - } else if (TryCurlEndpoint("http://100.100.100.200/latest/meta-data")) { + } else if (PingEndpoint("100.100.100.200", "/latest/meta-data")) { // containers in ACK can be connected to the above address, see // https://help.aliyun.com/document_detail/108460.html#section-akf-lwh-1gb. // Note: we can not distinguish ACK from K8S built on ECS @@ -724,7 +1050,7 @@ string SLSClientManager::GetRunningEnvironment() { } } else if (AppConfig::GetInstance()->IsPurageContainerMode() || getenv("ALIYUN_LOGTAIL_CONFIG")) { env = "Docker"; - } else if (TryCurlEndpoint("http://100.100.100.200/latest/meta-data")) { + } else if (PingEndpoint("100.100.100.200", "/latest/meta-data")) { env = "ECS"; } else { env = "Others"; @@ -732,37 +1058,270 @@ string SLSClientManager::GetRunningEnvironment() { return env; } -bool SLSClientManager::TryCurlEndpoint(const string& endpoint) { - CURL* curl; - for (size_t retryTimes = 1; retryTimes <= 5; retryTimes++) { - curl = curl_easy_init(); - if (curl) { - break; - } - this_thread::sleep_for(chrono::seconds(1)); +bool SLSClientManager::PingEndpoint(const string& host, const string& path) { + map header; + HttpResponse response; + return SendHttpRequest(make_unique(HTTP_GET, false, host, 80, path, "", header, "", 3, 1, true), + response); +} + +#ifdef APSARA_UNIT_TEST_MAIN +void SLSClientManager::Clear() { + mRegionCandidateEndpointsMap.clear(); + mHttpsRegions.clear(); + mRegionCandidateHostsInfosMap.clear(); + mProjectCandidateHostsInfosMap.clear(); + mUnInitializedCandidateHostsInfos.clear(); + mRegionRealIpMap.clear(); + mOutdatedRealIpRegions.clear(); + mRegionRealIpCandidateHostsInfosMap.clear(); +} +#endif + +void PreparePostLogStoreLogsRequest(const string& accessKeyId, + const string& accessKeySecret, + SLSClientManager::AuthType type, + const string& host, + bool isHostIp, + const string& project, + const string& logstore, + const string& compressType, + RawDataType dataType, + const string& body, + size_t rawSize, + const string& shardHashKey, + optional seqId, + string& path, + string& query, + map& header) { + path = LOGSTORES; + path.append("/").append(logstore); + if (shardHashKey.empty()) { + path.append("/shards/lb"); + } else { + path.append("/shards/route"); + } + + if (isHostIp) { + header[HOST] = project + "." + host; + } else { + header[HOST] = host; + } + header[USER_AGENT] = SLSClientManager::GetInstance()->GetUserAgent(); + header[DATE] = GetDateString(); + header[CONTENT_TYPE] = TYPE_LOG_PROTOBUF; + header[CONTENT_LENGTH] = to_string(body.size()); + header[CONTENT_MD5] = CalcMD5(body); + header[X_LOG_APIVERSION] = LOG_API_VERSION; + header[X_LOG_SIGNATUREMETHOD] = HMAC_SHA1; + if (!compressType.empty()) { + header[X_LOG_COMPRESSTYPE] = compressType; + } + if (dataType == RawDataType::EVENT_GROUP) { + header[X_LOG_BODYRAWSIZE] = to_string(rawSize); + } else { + header[X_LOG_BODYRAWSIZE] = to_string(body.size()); + header[X_LOG_MODE] = LOG_MODE_BATCH_GROUP; + } + if (type == SLSClientManager::AuthType::ANONYMOUS) { + header[X_LOG_KEYPROVIDER] = MD5_SHA1_SALT_KEYPROVIDER; } - if (curl) { - curl_easy_setopt(curl, CURLOPT_URL, endpoint.c_str()); - curl_easy_setopt(curl, CURLOPT_NOBODY, 1L); - curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); - curl_easy_setopt(curl, CURLOPT_TIMEOUT, 3); - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L); - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L); - - if (curl_easy_perform(curl) != CURLE_OK) { - curl_easy_cleanup(curl); - return false; + map parameterList; + if (!shardHashKey.empty()) { + parameterList["key"] = shardHashKey; + if (!seqId.has_value()) { + parameterList["seqid"] = to_string(seqId.value()); } - curl_easy_cleanup(curl); - return true; } + query = GetQueryString(parameterList); - LOG_WARNING( - sLogger, - ("curl handler cannot be initialized during user environment identification", "user agent may be mislabeled")); - return false; + string signature = GetUrlSignature(HTTP_POST, path, header, parameterList, body, accessKeySecret); + header[AUTHORIZATION] = LOG_HEADSIGNATURE_PREFIX + accessKeyId + ':' + signature; +} + +void PreparePostMetricStoreLogsRequest(const string& accessKeyId, + const string& accessKeySecret, + SLSClientManager::AuthType type, + const string& host, + bool isHostIp, + const string& project, + const string& logstore, + const string& compressType, + const string& body, + size_t rawSize, + string& path, + map& header) { + path = METRICSTORES; + path.append("/").append(project).append("/").append(logstore).append("/api/v1/write"); + + if (isHostIp) { + header[HOST] = project + "." + host; + } else { + header[HOST] = host; + } + header[USER_AGENT] = SLSClientManager::GetInstance()->GetUserAgent(); + header[DATE] = GetDateString(); + header[CONTENT_TYPE] = TYPE_LOG_PROTOBUF; + header[CONTENT_LENGTH] = to_string(body.size()); + header[CONTENT_MD5] = CalcMD5(body); + header[X_LOG_APIVERSION] = LOG_API_VERSION; + header[X_LOG_SIGNATUREMETHOD] = HMAC_SHA1; + if (!compressType.empty()) { + header[X_LOG_COMPRESSTYPE] = compressType; + } + header[X_LOG_BODYRAWSIZE] = to_string(rawSize); + if (type == SLSClientManager::AuthType::ANONYMOUS) { + header[X_LOG_KEYPROVIDER] = MD5_SHA1_SALT_KEYPROVIDER; + } + + map parameterList; + string signature = GetUrlSignature(HTTP_POST, path, header, parameterList, body, accessKeySecret); + header[AUTHORIZATION] = LOG_HEADSIGNATURE_PREFIX + accessKeyId + ':' + signature; +} + +SLSResponse PostLogStoreLogs(const string& accessKeyId, + const string& accessKeySecret, + SLSClientManager::AuthType type, + const string& host, + bool httpsFlag, + const string& project, + const string& logstore, + const string& compressType, + RawDataType dataType, + const string& body, + size_t rawSize, + const string& shardHashKey) { + string path, query; + map header; + PreparePostLogStoreLogsRequest(accessKeyId, + accessKeySecret, + type, + host, + false, // sync request always uses vip + project, + logstore, + compressType, + dataType, + body, + rawSize, + shardHashKey, + nullopt, // sync request does not support exactly-once + path, + query, + header); + HttpResponse response; + SendHttpRequest( + make_unique(HTTP_POST, httpsFlag, host, httpsFlag ? 443 : 80, path, query, header, body), + response); + return ParseHttpResponse(response); +} + +SLSResponse PostMetricStoreLogs(const string& accessKeyId, + const string& accessKeySecret, + SLSClientManager::AuthType type, + const string& host, + bool httpsFlag, + const string& project, + const string& logstore, + const string& compressType, + const string& body, + size_t rawSize) { + string path; + map header; + PreparePostMetricStoreLogsRequest(accessKeyId, + accessKeySecret, + type, + host, + false, // sync request always uses vip + project, + logstore, + compressType, + body, + rawSize, + path, + header); + HttpResponse response; + SendHttpRequest(make_unique(HTTP_POST, httpsFlag, host, httpsFlag ? 443 : 80, path, "", header, body), + response); + return ParseHttpResponse(response); +} + +SLSResponse PutWebTracking(const string& host, + bool httpsFlag, + const string& logstore, + const string& compressType, + const string& body, + size_t rawSize) { + string path = LOGSTORES; + path.append("/").append(logstore).append("/track"); + + map header; + header[HOST] = host; + header[USER_AGENT] = SLSClientManager::GetInstance()->GetUserAgent(); + header[DATE] = GetDateString(); + header[CONTENT_LENGTH] = to_string(body.size()); + header[X_LOG_APIVERSION] = LOG_API_VERSION; + // header[X_LOG_SIGNATUREMETHOD] = HMAC_SHA1; + if (!compressType.empty()) { + header[X_LOG_COMPRESSTYPE] = compressType; + } + header[X_LOG_BODYRAWSIZE] = to_string(rawSize); + + HttpResponse response; + SendHttpRequest(make_unique(HTTP_POST, httpsFlag, host, httpsFlag ? 443 : 80, path, "", header, body), + response); + return ParseHttpResponse(response); } +// bool GetEndpointRealIp(const string& endpoint, string& ip) { +// static string body; +// if (body.empty()) { +// sls_logs::LogGroup logGroup; +// logGroup.set_source(LoongCollectorMonitor::mIpAddr); +// body = logGroup.SerializeAsString(); +// } + +// static string project = "logtail-real-ip-project"; +// static string logstore = "logtail-real-ip-logstore"; + +// SLSClientManager::AuthType type; +// string accessKeyId, accessKeySecret; +// SLSClientManager::GetInstance()->GetAccessKey("", type, accessKeyId, accessKeySecret); + +// string path, query; +// map header; +// string host = project + "." + endpoint; +// PreparePostLogStoreLogsRequest(accessKeyId, +// accessKeySecret, +// type, +// host, +// false, +// project, +// logstore, +// "", +// RawDataType::EVENT_GROUP, +// body, +// body.size(), +// "", +// nullopt, +// path, +// query, +// header); +// HttpResponse response; +// if (!SendHttpRequest(make_unique(HTTP_POST, false, host, 80, path, query, header, body), response)) +// { +// return false; +// } +// if (response.GetStatusCode() != 200) { +// auto it = response.GetHeader().find(X_LOG_HOSTIP); +// if (it != response.GetHeader().end()) { +// ip = it->second; +// } +// return true; +// } +// // should not happen +// return true; +// } + } // namespace logtail diff --git a/core/plugin/flusher/sls/SLSClientManager.h b/core/plugin/flusher/sls/SLSClientManager.h index c82a8a8cb1..39035f2272 100644 --- a/core/plugin/flusher/sls/SLSClientManager.h +++ b/core/plugin/flusher/sls/SLSClientManager.h @@ -16,25 +16,105 @@ #pragma once +#include + +#include +#include #include #include +#include #include +#include +#include #include #include #include #include #include +#include #include -#include "sdk/Client.h" +#include "common/http/HttpRequest.h" +#include "common/http/HttpResponse.h" +#include "pipeline/queue/SenderQueueItem.h" +#include "plugin/flusher/sls/SLSResponse.h" namespace logtail { +enum class EndpointMode { DEFAULT, ACCELERATE, CUSTOM }; +const std::string& EndpointModeToString(EndpointMode mode); + +struct CandidateEndpoints { + // currently only remote endpoints can be updated + EndpointMode mMode = EndpointMode::DEFAULT; + // when mMode = ACCELERATE, mLocalEndpoints should be empty + std::vector mLocalEndpoints; + std::vector mRemoteEndpoints; +}; + +class HostInfo { +public: + HostInfo(const std::string& hostname) : mHostname(hostname) {} + HostInfo(const HostInfo& rhs) : mHostname(rhs.mHostname) { mLatency = rhs.GetLatency(); } + + const std::string& GetHostname() const { return mHostname; } + std::chrono::milliseconds GetLatency() const; + void SetLatency(const std::chrono::milliseconds& latency); + void SetForbidden(); + bool IsForbidden() const; + +private: + // normally in the form of ., except for the real ip scene, which equals to endpoint + const std::string mHostname; + + mutable std::mutex mLatencyMux; + std::chrono::milliseconds mLatency = std::chrono::milliseconds::max(); +}; + +class CandidateHostsInfo { +public: + CandidateHostsInfo(const std::string& project, const std::string& region, EndpointMode mode) + : mProject(project), mRegion(region), mMode(mode) {} + + void UpdateHosts(const CandidateEndpoints& regionEndpoints); + bool UpdateHostInfo(const std::string& hostname, const std::chrono::milliseconds& latency); + void GetProbeHosts(std::vector& hosts) const; + void GetAllHosts(std::vector& hosts) const; + void SelectBestHost(); + std::string GetCurrentHost() const; + std::string GetFirstHost() const; + const std::string& GetProject() const { return mProject; } + const std::string& GetRegion() const { return mRegion; } + EndpointMode GetMode() const { return mMode; } + +private: + bool HasValidHost() const; + void SetCurrentHost(const std::string& host); + + // for real ip scene, mProject is empty + const std::string mProject; + const std::string mRegion; + const EndpointMode mMode; + + mutable std::mutex mCurrentHostMux; + std::string mCurrentHost; + + // mMode = DEFAULT: each mCandidateHosts element has exactly one element + // mMode = ACCELERATE: mCandidateHosts.size() == 1 && mCandidateHosts[0].size() == 2 + // mMode = CUSTOM: mCandidateHosts.size() == 1 && mCandidateHosts[0].size() == 1 + mutable std::mutex mCandidateHostsMux; + std::vector> mCandidateHosts; + +#ifdef APSARA_UNIT_TEST_MAIN + friend class CandidateHostsInfoUnittest; + friend class SLSClientManagerUnittest; +#endif +}; + class SLSClientManager { public: - enum class EndpointSourceType { LOCAL, REMOTE }; - enum class EndpointSwitchPolicy { DESIGNATED_FIRST, DESIGNATED_LOCKED }; enum class AuthType { ANONYMOUS, AK }; + // enum class RemoteEndpointUpdateAction { CREATE, OVERWRITE, APPEND }; virtual ~SLSClientManager() = default; SLSClientManager(const SLSClientManager&) = delete; @@ -42,131 +122,196 @@ class SLSClientManager { static SLSClientManager* GetInstance(); - void Init(); - void Stop(); + virtual void Init(); + virtual void Stop() {}; - EndpointSwitchPolicy GetServerSwitchPolicy() const { return mDataServerSwitchPolicy; } const std::string& GetUserAgent() const { return mUserAgent; } - void IncreaseAliuidReferenceCntForRegion(const std::string& region, const std::string& aliuid); - void DecreaseAliuidReferenceCntForRegion(const std::string& region, const std::string& aliuid); - - sdk::Client* GetClient(const std::string& region, const std::string& aliuid, bool createIfNotFound = true); - bool ResetClientEndpoint(const std::string& aliuid, const std::string& region, time_t curTime); - void CleanTimeoutClient(); virtual bool GetAccessKey(const std::string& aliuid, AuthType& type, std::string& accessKeyId, std::string& accessKeySecret); virtual void UpdateAccessKeyStatus(const std::string& aliuid, bool success) {} - void AddEndpointEntry(const std::string& region, - const std::string& endpoint, - bool isProxy, - const EndpointSourceType& endpointType); - void UpdateEndpointStatus(const std::string& region, - const std::string& endpoint, - bool status, - std::optional latency = std::optional()); + // // currently not support hot relead + // void UpdateLocalRegionEndpointsAndHttpsInfo(const std::string& region, const std::vector& endpoints); + // void UpdateRemoteRegionEndpoints(const std::string& region, + // const std::vector& endpoints, + // RemoteEndpointUpdateAction action = RemoteEndpointUpdateAction::OVERWRITE); + + // std::shared_ptr + // GetCandidateHostsInfo(const std::string& region, const std::string& project, EndpointMode mode); + // bool UpdateHostInfo(const std::string& project, + // EndpointMode mode, + // const std::string& host, + // const std::chrono::milliseconds& latency); + // // only for real ip + // bool UpdateHostInfo(const std::string& region, const std::string& host, const std::chrono::milliseconds& latency); + + // only for open source + std::shared_ptr GetCandidateHostsInfo(const std::string& project, const std::string& endpoint); - void ForceUpdateRealIp(const std::string& region); - void UpdateSendClientRealIp(sdk::Client* client, const std::string& region); + virtual bool UsingHttps(const std::string& region) const; + + // void UpdateOutdatedRealIpRegions(const std::string& region); + // std::string GetRealIp(const std::string& region) const; std::string GetRegionFromEndpoint(const std::string& endpoint); // for backward compatibility - bool HasNetworkAvailable(); // TODO: remove this function + +#ifdef APSARA_UNIT_TEST_MAIN + void Clear(); +#endif protected: SLSClientManager() = default; virtual std::string GetRunningEnvironment(); - bool TryCurlEndpoint(const std::string& endpoint); + bool PingEndpoint(const std::string& host, const std::string& path); std::string mUserAgent; private: - enum class EndpointStatus { STATUS_OK_WITH_IP, STATUS_OK_WITH_ENDPOINT, STATUS_ERROR }; - - struct EndpointInfo { - bool mValid = true; - std::optional mLatencyMs; - bool mProxy = false; - - EndpointInfo(bool valid, bool proxy) : mValid(valid), mProxy(proxy) {} - - void UpdateInfo(bool valid, std::optional latency) { - mValid = valid; - mLatencyMs = latency; - } - }; - - struct RegionEndpointsInfo { - std::unordered_map mEndpointInfoMap; - std::string mDefaultEndpoint; - EndpointSourceType mDefaultEndpointType; - - bool AddDefaultEndpoint(const std::string& endpoint, const EndpointSourceType& endpointType, bool& isDefault); - bool AddEndpoint(const std::string& endpoint, bool status, bool proxy = false); - void UpdateEndpointInfo(const std::string& endpoint, - bool status, - std::optional latency, - bool createFlag = true); - void RemoveEndpoint(const std::string& endpoint); - std::string GetAvailableEndpointWithTopPriority() const; - }; - - struct RealIpInfo { - std::string mRealIp; - time_t mLastUpdateTime = 0; - bool mForceFlushFlag = false; - - void SetRealIp(const std::string& realIp) { - mRealIp = realIp; - mLastUpdateTime = time(NULL); - mForceFlushFlag = false; - } - }; + // struct ProbeNetworkHttpRequest : public AsynHttpRequest { + // std::string mRegion; + // std::string mProject; + // EndpointMode mMode; + // std::string mHost; + + // ProbeNetworkHttpRequest(const std::string& region, + // const std::string& project, + // EndpointMode mode, + // const std::string& host, + // bool httpsFlag); + + // bool IsContextValid() const override { return true; }; + // void OnSendDone(HttpResponse& response) override; + // }; virtual void GenerateUserAgent(); - void InitEndpointSwitchPolicy(); - std::vector GetRegionAliuids(const std::string& region); - - void ResetClientPort(const std::string& region, sdk::Client* sendClient); - std::string GetAvailableEndpointWithTopPriority(const std::string& region) const; - void ProbeNetworkThread(); - bool TestEndpoint(const std::string& region, const std::string& endpoint); - - void UpdateRealIpThread(); - EndpointStatus UpdateRealIp(const std::string& region, const std::string& endpoint); - void SetRealIp(const std::string& region, const std::string& ip); - - mutable std::mutex mRegionAliuidRefCntMapLock; - std::unordered_map> mRegionAliuidRefCntMap; - - mutable std::mutex mClientMapMux; - std::unordered_map, time_t>> mClientMap; - // int32_t mLastCheckSendClientTime; - - mutable std::mutex mRegionEndpointEntryMapLock; - std::unordered_map mRegionEndpointEntryMap; - EndpointSwitchPolicy mDataServerSwitchPolicy = EndpointSwitchPolicy::DESIGNATED_FIRST; - std::unique_ptr mProbeNetworkClient; - - std::future mProbeNetworkThreadRes; - mutable std::mutex mProbeNetworkThreadRunningMux; - bool mIsProbeNetworkThreadRunning = true; - - mutable std::mutex mRegionRealIpLock; - std::unordered_map mRegionRealIpMap; - std::unique_ptr mUpdateRealIpClient; - - std::future mUpdateRealIpThreadRes; - mutable std::mutex mUpdateRealIpThreadRunningMux; - bool mIsUpdateRealIpThreadRunning = true; - - mutable std::condition_variable mStopCV; + // void UnInitializedHostProbeThread(); + // void DoProbeUnInitializedHost(); + // void HostProbeThread(); + // void DoProbeHost(); + // bool HasPartiallyInitializedCandidateHostsInfos() const; + // void ClearExpiredCandidateHostsInfos(); + + // void UpdateRealIpThread(); + // void DoUpdateRealIp(); + // void SetRealIp(const std::string& region, const std::string& ip); + // bool HasOutdatedRealIpRegions() const; + + // mutable std::mutex mRegionCandidateEndpointsMapMux; + // std::map mRegionCandidateEndpointsMap; + + // mutable std::mutex mHttpsRegionsMux; + // std::unordered_set mHttpsRegions; + + mutable std::mutex mCandidateHostsInfosMapMux; + std::map>> mRegionCandidateHostsInfosMap; + // for opensource, only custom mode is supported, and one project supports multiple custom endpoints + // for enterprise, only one info for each mode is supported + std::map>> mProjectCandidateHostsInfosMap; + +// CURLM* mUnInitializedHostProbeClient = nullptr; +// mutable std::mutex mUnInitializedCandidateHostsInfosMux; +// std::vector> mUnInitializedCandidateHostsInfos; + +// std::future mUnInitializedHostProbeThreadRes; +// mutable std::mutex mUnInitializedHostProbeThreadRunningMux; +// bool mIsUnInitializedHostProbeThreadRunning = true; + +// CURLM* mHostProbeClient = nullptr; +// mutable std::mutex mPartiallyInitializedCandidateHostsInfosMux; +// std::vector> mPartiallyInitializedCandidateHostsInfos; + +// std::future mHostProbeThreadRes; +// mutable std::mutex mHostProbeThreadRunningMux; +// bool mIsHostProbeThreadRunning = true; +// #ifdef APSARA_UNIT_TEST_MAIN +// HttpResponse (*mDoProbeNetwork)(const std::unique_ptr& req) = nullptr; +// #endif + +// mutable std::mutex mRegionRealIpMux; +// std::map mRegionRealIpMap; +// mutable std::mutex mOutdatedRealIpRegionsMux; +// std::vector mOutdatedRealIpRegions; +// mutable std::mutex mRegionRealIpCandidateHostsInfosMapMux; +// std::map mRegionRealIpCandidateHostsInfosMap; +// bool (*mGetEndpointRealIp)(const std::string& endpoint, std::string& ip) = nullptr; + +// std::future mUpdateRealIpThreadRes; +// mutable std::mutex mUpdateRealIpThreadRunningMux; +// bool mIsUpdateRealIpThreadRunning = true; + +// mutable std::condition_variable mStopCV; #ifdef APSARA_UNIT_TEST_MAIN friend class FlusherSLSUnittest; + friend class SLSClientManagerUnittest; + friend class ProbeNetworkMock; + friend class GetRealIpMock; #endif }; +void PreparePostLogStoreLogsRequest(const std::string& accessKeyId, + const std::string& accessKeySecret, + SLSClientManager::AuthType type, + const std::string& host, + bool isHostIp, + const std::string& project, + const std::string& logstore, + const std::string& compressType, + RawDataType dataType, + const std::string& body, + size_t rawSize, + const std::string& shardHashKey, + std::optional seqId, + std::string& path, + std::string& query, + std::map& header); +void PreparePostMetricStoreLogsRequest(const std::string& accessKeyId, + const std::string& accessKeySecret, + SLSClientManager::AuthType type, + const std::string& host, + bool isHostIp, + const std::string& project, + const std::string& logstore, + const std::string& compressType, + const std::string& body, + size_t rawSize, + std::string& path, + std::map& header); +SLSResponse PostLogStoreLogs(const std::string& accessKeyId, + const std::string& accessKeySecret, + SLSClientManager::AuthType type, + const std::string& host, + bool httpsFlag, + const std::string& project, + const std::string& logstore, + const std::string& compressType, + RawDataType dataType, + const std::string& body, + size_t rawSize, + const std::string& shardHashKey); +SLSResponse PostMetricStoreLogs(const std::string& accessKeyId, + const std::string& accessKeySecret, + SLSClientManager::AuthType type, + const std::string& host, + bool httpsFlag, + const std::string& project, + const std::string& logstore, + const std::string& compressType, + const std::string& body, + size_t rawSize); +SLSResponse PutWebTracking(const std::string& host, + bool httpsFlag, + const std::string& logstore, + const std::string& compressType, + const std::string& body, + size_t rawSize); +// bool GetEndpointRealIp(const std::string& endpoint, std::string& ip); + +// #ifdef APSARA_UNIT_TEST_MAIN +// extern HttpResponse (*DoGetRealIp)(const std::unique_ptr& req); +// #endif + } // namespace logtail diff --git a/core/plugin/flusher/sls/SLSConstant.cpp b/core/plugin/flusher/sls/SLSConstant.cpp new file mode 100644 index 0000000000..f6c340ddc4 --- /dev/null +++ b/core/plugin/flusher/sls/SLSConstant.cpp @@ -0,0 +1,96 @@ +// Copyright 2024 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "plugin/flusher/sls/SLSConstant.h" + +using namespace std; + +namespace logtail { + +const string LOGSTORES = "/logstores"; +const string METRICSTORES = "/prometheus"; +const string HEALTH = "/health"; + +const string CONTENT_MD5 = "Content-MD5"; + +const string LOG_HEADER_PREFIX = "x-log-"; +const string LOG_OLD_HEADER_PREFIX = "x-sls-"; +const string ACS_HEADER_PREFIX = "x-acs-"; +const string X_LOG_KEYPROVIDER = "x-log-keyprovider"; +const string X_LOG_APIVERSION = "x-log-apiversion"; +const string X_LOG_COMPRESSTYPE = "x-log-compresstype"; +const string X_LOG_BODYRAWSIZE = "x-log-bodyrawsize"; +const string X_LOG_SIGNATUREMETHOD = "x-log-signaturemethod"; +const string X_LOG_MODE = "x-log-mode"; +const string X_LOG_HOSTIP = "x-log-hostip"; +const string X_LOG_REQUEST_ID = "x-log-requestid"; +const string HMAC_SHA1 = "hmac-sha1"; +const string LOG_HEADSIGNATURE_PREFIX = "LOG "; +const string LOG_API_VERSION = "0.6.0"; +const string LOG_MODE_BATCH_GROUP = "batch_group"; + +const string MD5_SHA1_SALT_KEYPROVIDER = "md5-sha1-salt"; + +const string LOGE_REQUEST_ERROR = "RequestError"; +const string LOGE_INVALID_HOST = "InvalidHost"; +const string LOGE_UNKNOWN_ERROR = "UnknownError"; +const string LOGE_NOT_IMPLEMENTED = "NotImplemented"; +const string LOGE_SERVER_BUSY = "ServerBusy"; +const string LOGE_INTERNAL_SERVER_ERROR = "InternalServerError"; +const string LOGE_RESPONSE_SIG_ERROR = "ResponseSignatureError"; +const string LOGE_PARAMETER_INVALID = "ParameterInvalid"; +const string LOGE_MISSING_PARAMETER = "MissingParameter"; +const string LOGE_INVALID_METHOD = "InvalidMethod"; +const string LOGE_BAD_RESPONSE = "BadResponse"; +const string LOGE_UNAUTHORIZED = "Unauthorized"; +const string LOGE_QUOTA_EXCEED = "ExceedQuota"; +const string LOGE_REQUEST_TIMEOUT = "RequestTimeout"; +const string LOGE_CLIENT_OPERATION_TIMEOUT = "ClientOpertaionTimeout"; +const string LOGE_CLIENT_NETWORK_ERROR = "ClientNetworkError"; +const string LOGE_USER_NOT_EXIST = "UserNotExist"; +const string LOGE_CATEGORY_NOT_EXIST = "CategoryNotExist"; +const string LOGE_TOPIC_NOT_EXIST = "TopicNotExist"; +const string LOGE_POST_BODY_INVALID = "PostBodyInvalid"; +const string LOGE_INVALID_CONTENTTYPE = "InvalidContentType"; +const string LOGE_INVALID_CONTENLENGTH = "InvalidContentLength"; +const string LOGE_INVALID_APIVERSION = "InvalidAPIVersion"; +const string LOGE_PROJECT_NOT_EXIST = "ProjectNotExist"; +const string LOGE_MACHINEGROUP_NOT_EXIST = "MachineGroupNotExist"; +const string LOGE_MACHINEGROUP_ALREADY_EXIST = "MachineGroupAlreadyExist"; +const string LOGE_CONFIG_NOT_EXIST = "ConfigNotExist"; +const string LOGE_CONFIG_ALREADY_EXIST = "ConfigAlreadyExist"; +const string LOGE_LOGSTORE_NOT_EXIST = "LogStoreNotExist"; +const string LOGE_INVALID_ACCESSKEYID = "InvalidAccessKeyId"; +const string LOGE_SIGNATURE_NOT_MATCH = "SignatureNotMatch"; +const string LOGE_PROJECT_FORBIDDEN = "ProjectForbidden"; +const string LOGE_WRITE_QUOTA_EXCEED = "WriteQuotaExceed"; +const string LOGE_READ_QUOTA_EXCEED = "ReadQuotaExceed"; +const string LOGE_REQUEST_TIME_EXPIRED = "RequestTimeExpired"; +const string LOGE_INVALID_REQUEST_TIME = "InvalidRequestTime"; +const string LOGE_POST_BODY_TOO_LARGE = "PostBodyTooLarge"; +const string LOGE_INVALID_TIME_RANGE = "InvalidTimeRange"; +const string LOGE_INVALID_REVERSE = "InvalidReverse"; +const string LOGE_LOGSTORE_WITHOUT_SHARD = "LogStoreWithoutShard"; +const string LOGE_SHARD_WRITE_QUOTA_EXCEED = "ShardWriteQuotaExceed"; +const string LOGE_SHARD_READ_QUOTA_EXCEED = "ShardReadQuotaExceed"; +const string LOGE_INVALID_SEQUENCE_ID = "InvalidSequenceId"; +const string LOGE_NOT_SUPPORTED_ACCEPT_CONTENT_TYPE = "InvalidAcceptContentType"; +const string LOGE_NOT_SUPPORTED_ACCEPT_ENCODING = "InvalidAcceptEncoding"; +const string LOGE_SHARD_NOT_EXIST = "ShardNotExist"; +const string LOGE_INVALID_CURSOR = "InvalidCursor"; + +const string LOG_ERROR_CODE = "errorCode"; +const string LOG_ERROR_MESSAGE = "errorMessage"; + +} // namespace logtail diff --git a/core/plugin/flusher/sls/SLSConstant.h b/core/plugin/flusher/sls/SLSConstant.h new file mode 100644 index 0000000000..fbf5956908 --- /dev/null +++ b/core/plugin/flusher/sls/SLSConstant.h @@ -0,0 +1,99 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +namespace logtail { + +extern const std::string LOGSTORES; +extern const std::string METRICSTORES; +extern const std::string HEALTH; + +extern const std::string CONTENT_MD5; + +extern const std::string LOG_HEADER_PREFIX; +extern const std::string LOG_OLD_HEADER_PREFIX; +extern const std::string ACS_HEADER_PREFIX; +extern const std::string X_LOG_KEYPROVIDER; +extern const std::string X_LOG_APIVERSION; +extern const std::string X_LOG_COMPRESSTYPE; +extern const std::string X_LOG_BODYRAWSIZE; +extern const std::string X_LOG_SIGNATUREMETHOD; +extern const std::string X_LOG_MODE; +extern const std::string X_LOG_HOSTIP; +extern const std::string X_LOG_REQUEST_ID; + +extern const std::string LOG_HEADSIGNATURE_PREFIX; +extern const std::string LOG_API_VERSION; +extern const std::string LOG_MODE_BATCH_GROUP; +extern const std::string HMAC_SHA1; +extern const std::string MD5_SHA1_SALT_KEYPROVIDER; + +extern const std::string LOGE_REQUEST_ERROR; +extern const std::string LOGE_INVALID_HOST; +extern const std::string LOGE_UNKNOWN_ERROR; +extern const std::string LOGE_NOT_IMPLEMENTED; +extern const std::string LOGE_SERVER_BUSY; +extern const std::string LOGE_INTERNAL_SERVER_ERROR; +extern const std::string LOGE_RESPONSE_SIG_ERROR; +extern const std::string LOGE_PARAMETER_INVALID; +extern const std::string LOGE_MISSING_PARAMETER; +extern const std::string LOGE_INVALID_METHOD; +extern const std::string LOGE_INVALID_CONTENTTYPE; +extern const std::string LOGE_INVALID_CONTENTLENGTH; +extern const std::string LOGE_BAD_RESPONSE; +extern const std::string LOGE_UNAUTHORIZED; +extern const std::string LOGE_QUOTA_EXCEED; +extern const std::string LOGE_REQUEST_TIMEOUT; +extern const std::string LOGE_CLIENT_OPERATION_TIMEOUT; +extern const std::string LOGE_CLIENT_NETWORK_ERROR; +extern const std::string LOGE_USER_NOT_EXIST; +extern const std::string LOGE_CATEGORY_NOT_EXIST; +extern const std::string LOGE_TOPIC_NOT_EXIST; +extern const std::string LOGE_POST_BODY_INVALID; +extern const std::string LOGE_INVALID_HOST; +extern const std::string LOGE_INVALID_APIVERSION; +extern const std::string LOGE_PROJECT_NOT_EXIST; +extern const std::string LOGE_MACHINEGROUP_NOT_EXIST; +extern const std::string LOGE_MACHINEGROUP_ALREADY_EXIST; +extern const std::string LOGE_CONFIG_NOT_EXIST; +extern const std::string LOGE_CONFIG_ALREADY_EXIST; +extern const std::string LOGE_LOGSTORE_NOT_EXIST; +extern const std::string LOGE_INVALID_ACCESSKEYID; +extern const std::string LOGE_SIGNATURE_NOT_MATCH; +extern const std::string LOGE_PROJECT_FORBIDDEN; +extern const std::string LOGE_WRITE_QUOTA_EXCEED; +extern const std::string LOGE_READ_QUOTA_EXCEED; +extern const std::string LOGE_REQUEST_TIME_EXPIRED; +extern const std::string LOGE_INVALID_REQUEST_TIME; +extern const std::string LOGE_POST_BODY_TOO_LARGE; +extern const std::string LOGE_INVALID_TIME_RANGE; +extern const std::string LOGE_INVALID_REVERSE; +extern const std::string LOGE_LOGSTORE_WITHOUT_SHARD; +extern const std::string LOGE_INVALID_SEQUENCE_ID; +extern const std::string LOGE_NOT_SUPPORTED_ACCEPT_CONTENT_TYPE; +extern const std::string LOGE_NOT_SUPPORTED_ACCEPT_ENCODING; +extern const std::string LOGE_SHARD_NOT_EXIST; +extern const std::string LOGE_INVALID_CURSOR; +extern const std::string LOGE_SHARD_WRITE_QUOTA_EXCEED; +extern const std::string LOGE_SHARD_READ_QUOTA_EXCEED; + +extern const std::string LOG_ERROR_CODE; +extern const std::string LOG_ERROR_MESSAGE; + +} // namespace logtail diff --git a/core/plugin/flusher/sls/SLSResponse.cpp b/core/plugin/flusher/sls/SLSResponse.cpp index d9fa405479..e13847618a 100644 --- a/core/plugin/flusher/sls/SLSResponse.cpp +++ b/core/plugin/flusher/sls/SLSResponse.cpp @@ -14,31 +14,80 @@ #include "plugin/flusher/sls/SLSResponse.h" +#include + +#include "app_config/AppConfig.h" #include "common/ErrorUtil.h" #include "common/StringTools.h" #include "common/TimeUtil.h" #include "logger/Logger.h" -#include "sdk/Common.h" -#include "sdk/Exception.h" -#include "sdk/Result.h" +#include "plugin/flusher/sls/SLSConstant.h" +#include "Exception.h" using namespace std; +using namespace logtail::sdk; namespace logtail { +void ExtractJsonResult(const string& response, rapidjson::Document& document) { + document.Parse(response.c_str()); + if (document.HasParseError()) { + throw JsonException("ParseException", "Fail to parse from json string"); + } +} + +void JsonMemberCheck(const rapidjson::Value& value, const char* name) { + if (!value.IsObject()) { + throw JsonException("InvalidObjectException", "response is not valid JSON object"); + } + if (!value.HasMember(name)) { + throw JsonException("NoMemberException", string("Member ") + name + " does not exist"); + } +} + +void ExtractJsonResult(const rapidjson::Value& value, const char* name, string& dst) { + JsonMemberCheck(value, name); + if (value[name].IsString()) { + dst = value[name].GetString(); + } else { + throw JsonException("ValueTypeException", string("Member ") + name + " is not string type"); + } +} + +void ErrorCheck(const string& response, const string& requestId, const int32_t httpCode) { + rapidjson::Document document; + try { + ExtractJsonResult(response, document); + + string errorCode; + ExtractJsonResult(document, LOG_ERROR_CODE.c_str(), errorCode); + + string errorMessage; + ExtractJsonResult(document, LOG_ERROR_MESSAGE.c_str(), errorMessage); + + throw LOGException(errorCode, errorMessage, requestId, httpCode); + } catch (JsonException& e) { + if (httpCode >= 500) { + throw LOGException(LOGE_INTERNAL_SERVER_ERROR, response, requestId, httpCode); + } else { + throw LOGException(LOGE_BAD_RESPONSE, string("Unextractable error:") + response, requestId, httpCode); + } + } +} + bool SLSResponse::Parse(const HttpResponse& response) { - const auto iter = response.GetHeader().find(sdk::X_LOG_REQUEST_ID); + const auto iter = response.GetHeader().find(X_LOG_REQUEST_ID); if (iter != response.GetHeader().end()) { mRequestId = iter->second; } mStatusCode = response.GetStatusCode(); if (mStatusCode == 0) { - mErrorCode = sdk::LOGE_REQUEST_TIMEOUT; + mErrorCode = LOGE_REQUEST_TIMEOUT; mErrorMsg = "Request timeout"; } else if (mStatusCode != 200) { try { - sdk::ErrorCheck(*response.GetBody(), mRequestId, response.GetStatusCode()); + ErrorCheck(*response.GetBody(), mRequestId, response.GetStatusCode()); } catch (sdk::LOGException& e) { mErrorCode = e.GetErrorCode(); mErrorMsg = e.GetMessage_(); @@ -47,8 +96,30 @@ bool SLSResponse::Parse(const HttpResponse& response) { return true; } +SLSResponse ParseHttpResponse(const HttpResponse& response) { + SLSResponse slsResponse; + if (AppConfig::GetInstance()->IsResponseVerificationEnabled() && !IsSLSResponse(response)) { + slsResponse.mStatusCode = 0; + slsResponse.mErrorCode = LOGE_REQUEST_ERROR; + slsResponse.mErrorMsg = "invalid response body"; + } else { + slsResponse.Parse(response); + + if (AppConfig::GetInstance()->EnableLogTimeAutoAdjust()) { + static uint32_t sCount = 0; + if (sCount++ % 10000 == 0 || slsResponse.mErrorCode == LOGE_REQUEST_TIME_EXPIRED) { + time_t serverTime = GetServerTime(response); + if (serverTime > 0) { + UpdateTimeDelta(serverTime); + } + } + } + } + return slsResponse; +} + bool IsSLSResponse(const HttpResponse& response) { - const auto iter = response.GetHeader().find(sdk::X_LOG_REQUEST_ID); + const auto iter = response.GetHeader().find(X_LOG_REQUEST_ID); if (iter == response.GetHeader().end()) { return false; } diff --git a/core/plugin/flusher/sls/SLSResponse.h b/core/plugin/flusher/sls/SLSResponse.h index 1388c7ef56..6e52a9c2cc 100644 --- a/core/plugin/flusher/sls/SLSResponse.h +++ b/core/plugin/flusher/sls/SLSResponse.h @@ -32,6 +32,7 @@ struct SLSResponse { bool Parse(const HttpResponse& response); }; +SLSResponse ParseHttpResponse(const HttpResponse& response); bool IsSLSResponse(const HttpResponse& response); time_t GetServerTime(const HttpResponse& response); diff --git a/core/plugin/flusher/sls/SLSUtil.cpp b/core/plugin/flusher/sls/SLSUtil.cpp new file mode 100644 index 0000000000..21e30d4ce6 --- /dev/null +++ b/core/plugin/flusher/sls/SLSUtil.cpp @@ -0,0 +1,307 @@ +// Copyright 2024 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "plugin/flusher/sls/SLSUtil.h" + +#include "app_config/AppConfig.h" +#include "common/EncodingUtil.h" +#include "common/HashUtil.h" +#include "common/TimeUtil.h" +#include "common/http/Constant.h" +#include "plugin/flusher/sls/SLSConstant.h" + +using namespace std; + +namespace logtail { + +static string DATE_FORMAT_RFC822 = "%a, %d %b %Y %H:%M:%S GMT"; + +#define BIT_COUNT_WORDS 2 +#define BIT_COUNT_BYTES (BIT_COUNT_WORDS * sizeof(uint32_t)) + +/* + * define the rotate left (circular left shift) operation + */ +#define rotl(v, b) (((v) << (b)) | ((v) >> (32 - (b)))) + +/* + * Define the basic SHA-1 functions F1 ~ F4. Note that the exclusive-OR + * operation (^) in F1 and F3 may be replaced by a bitwise OR operation + * (|), which produce identical results. + * + * F1 is used in ROUND 0~19, F2 is used in ROUND 20~39 + * F3 is used in ROUND 40~59, F4 is used in ROUND 60~79 + */ +#define F1(B, C, D) (((B) & (C)) ^ (~(B) & (D))) +#define F2(B, C, D) ((B) ^ (C) ^ (D)) +#define F3(B, C, D) (((B) & (C)) ^ ((B) & (D)) ^ ((C) & (D))) +#define F4(B, C, D) ((B) ^ (C) ^ (D)) + +/* + * Use different K in different ROUND + */ +#define K00_19 0x5A827999 +#define K20_39 0x6ED9EBA1 +#define K40_59 0x8F1BBCDC +#define K60_79 0xCA62C1D6 + +/* + * Another implementation of the ROUND transformation: + * (here the T is a temp variable) + * For t=0 to 79: + * { + * T=rotl(A,5)+Func(B,C,D)+K+W[t]+E; + * E=D; D=C; C=rotl(B,30); B=A; A=T; + * } + */ +#define ROUND(t, A, B, C, D, E, Func, K) \ + E += rotl(A, 5) + Func(B, C, D) + W[t] + K; \ + B = rotl(B, 30); + +#define ROUND5(t, Func, K) \ + ROUND(t, A, B, C, D, E, Func, K); \ + ROUND(t + 1, E, A, B, C, D, Func, K); \ + ROUND(t + 2, D, E, A, B, C, Func, K); \ + ROUND(t + 3, C, D, E, A, B, Func, K); \ + ROUND(t + 4, B, C, D, E, A, Func, K) + +#define ROUND20(t, Func, K) \ + ROUND5(t, Func, K); \ + ROUND5(t + 5, Func, K); \ + ROUND5(t + 10, Func, K); \ + ROUND5(t + 15, Func, K) + +/* + * Define constant of the initial vector + */ +const uint32_t SHA1::IV[SHA1_DIGEST_WORDS] = {0x67452301, 0xEFCDAB89, 0x98BADCFE, 0x10325476, 0xC3D2E1F0}; + +/* + * the message must be the big-endian32 (or left-most word) + * before calling the transform() function. + */ +const static uint32_t iii = 1; +const static bool littleEndian = *(uint8_t*)&iii != 0; + +inline uint32_t littleEndianToBig(uint32_t d) { + uint8_t* data = (uint8_t*)&d; + return data[0] << 24 | data[1] << 16 | data[2] << 8 | data[3]; +} + +inline void make_big_endian32(uint32_t* data, unsigned n) { + if (!littleEndian) { + return; + } + for (; n > 0; ++data, --n) { + *data = littleEndianToBig(*data); + } +} + +inline size_t min(size_t a, size_t b) { + return a < b ? a : b; +} + +void SHA1::transform() { + uint32_t W[80]; + memcpy(W, M, SHA1_INPUT_BYTES); + memset((uint8_t*)W + SHA1_INPUT_BYTES, 0, sizeof(W) - SHA1_INPUT_BYTES); + for (unsigned t = 16; t < 80; t++) { + W[t] = rotl(W[t - 16] ^ W[t - 14] ^ W[t - 8] ^ W[t - 3], 1); + } + + uint32_t A = H[0]; + uint32_t B = H[1]; + uint32_t C = H[2]; + uint32_t D = H[3]; + uint32_t E = H[4]; + + ROUND20(0, F1, K00_19); + ROUND20(20, F2, K20_39); + ROUND20(40, F3, K40_59); + ROUND20(60, F4, K60_79); + + H[0] += A; + H[1] += B; + H[2] += C; + H[3] += D; + H[4] += E; +} + +void SHA1::add(const uint8_t* data, size_t data_len) { + unsigned mlen = (unsigned)((bits >> 3) % SHA1_INPUT_BYTES); + bits += (uint64_t)data_len << 3; + unsigned use = (unsigned)min((size_t)(SHA1_INPUT_BYTES - mlen), data_len); + memcpy(M + mlen, data, use); + mlen += use; + + while (mlen == SHA1_INPUT_BYTES) { + data_len -= use; + data += use; + make_big_endian32((uint32_t*)M, SHA1_INPUT_WORDS); + transform(); + use = (unsigned)min((size_t)SHA1_INPUT_BYTES, data_len); + memcpy(M, data, use); + mlen = use; + } +} + +uint8_t* SHA1::result() { + unsigned mlen = (unsigned)((bits >> 3) % SHA1_INPUT_BYTES), padding = SHA1_INPUT_BYTES - mlen; + M[mlen++] = 0x80; + if (padding > BIT_COUNT_BYTES) { + memset(M + mlen, 0x00, padding - BIT_COUNT_BYTES); + make_big_endian32((uint32_t*)M, SHA1_INPUT_WORDS - BIT_COUNT_WORDS); + } else { + memset(M + mlen, 0x00, SHA1_INPUT_BYTES - mlen); + make_big_endian32((uint32_t*)M, SHA1_INPUT_WORDS); + transform(); + memset(M, 0x00, SHA1_INPUT_BYTES - BIT_COUNT_BYTES); + } + + uint64_t temp = littleEndian ? bits << 32 | bits >> 32 : bits; + memcpy(M + SHA1_INPUT_BYTES - BIT_COUNT_BYTES, &temp, BIT_COUNT_BYTES); + transform(); + make_big_endian32(H, SHA1_DIGEST_WORDS); + return (uint8_t*)H; +} + +template +inline void axor(T* p1, const T* p2, size_t len) { + for (; len != 0; --len) + *p1++ ^= *p2++; +} + +HMAC::HMAC(const uint8_t* key, size_t lkey) { + init(key, lkey); +} + +void HMAC::init(const uint8_t* key, size_t lkey) { + in.init(); + out.init(); + + uint8_t ipad[SHA1_INPUT_BYTES]; + uint8_t opad[SHA1_INPUT_BYTES]; + memset(ipad, 0x36, sizeof(ipad)); + memset(opad, 0x5c, sizeof(opad)); + + if (lkey <= SHA1_INPUT_BYTES) { + axor(ipad, key, lkey); + axor(opad, key, lkey); + } else { + SHA1 tmp; + tmp.add(key, lkey); + const uint8_t* key2 = tmp.result(); + axor(ipad, key2, SHA1_DIGEST_BYTES); + axor(opad, key2, SHA1_DIGEST_BYTES); + } + + in.add((uint8_t*)ipad, sizeof(ipad)); + out.add((uint8_t*)opad, sizeof(opad)); +} + +string GetDateString() { + time_t now_time; + time(&now_time); + if (AppConfig::GetInstance()->EnableLogTimeAutoAdjust()) { + now_time += GetTimeDelta(); + } + char buffer[128] = {'\0'}; + tm timeInfo; +#if defined(__linux__) + gmtime_r(&now_time, &timeInfo); +#elif defined(_MSC_VER) + gmtime_s(&timeInfo, &now_time); +#endif + strftime(buffer, 128, DATE_FORMAT_RFC822.c_str(), &timeInfo); + return string(buffer); +} + +static bool StartWith(const std::string& input, const std::string& pattern) { + if (input.length() < pattern.length()) { + return false; + } + + size_t i = 0; + while (i < pattern.length() && input[i] == pattern[i]) { + i++; + } + + return i == pattern.length(); +} + +static std::string CalcSHA1(const std::string& message, const std::string& key) { + HMAC hmac(reinterpret_cast(key.data()), key.size()); + hmac.add(reinterpret_cast(message.data()), message.size()); + return string(reinterpret_cast(hmac.result()), SHA1_DIGEST_BYTES); +} + +string GetUrlSignature(const string& httpMethod, + const string& operationType, + map& httpHeader, + const map& parameterList, + const string& content, + const string& signKey) { + string contentMd5; + string signature; + string osstream; + if (!content.empty()) { + contentMd5 = CalcMD5(content); + } + string contentType; + map::iterator iter = httpHeader.find(CONTENT_TYPE); + if (iter != httpHeader.end()) { + contentType = iter->second; + } + map endingMap; + osstream.append(httpMethod); + osstream.append("\n"); + osstream.append(contentMd5); + osstream.append("\n"); + osstream.append(contentType); + osstream.append("\n"); + osstream.append(httpHeader[DATE]); + osstream.append("\n"); + for (map::const_iterator iter = httpHeader.begin(); iter != httpHeader.end(); ++iter) { + if (StartWith(iter->first, LOG_OLD_HEADER_PREFIX)) { + string key = iter->first; + endingMap.insert(make_pair(key.replace(0, LOG_OLD_HEADER_PREFIX.size(), LOG_HEADER_PREFIX), iter->second)); + } else if (StartWith(iter->first, LOG_HEADER_PREFIX) || StartWith(iter->first, ACS_HEADER_PREFIX)) { + endingMap.insert(make_pair(iter->first, iter->second)); + } + } + for (map::const_iterator it = endingMap.begin(); it != endingMap.end(); ++it) { + osstream.append(it->first); + osstream.append(":"); + osstream.append(it->second); + osstream.append("\n"); + } + osstream.append(operationType); + if (parameterList.size() > 0) { + osstream.append("?"); + for (map::const_iterator iter = parameterList.begin(); iter != parameterList.end(); ++iter) { + if (iter != parameterList.begin()) { + osstream.append("&"); + } + osstream.append(iter->first); + osstream.append("="); + osstream.append(iter->second); + } + } + + signature = Base64Enconde(CalcSHA1(osstream, signKey)); + + return signature; +} + +} // namespace logtail diff --git a/core/plugin/flusher/sls/SLSUtil.h b/core/plugin/flusher/sls/SLSUtil.h new file mode 100644 index 0000000000..a30288c90a --- /dev/null +++ b/core/plugin/flusher/sls/SLSUtil.h @@ -0,0 +1,82 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include + +namespace logtail { + +#define SHA1_INPUT_WORDS 16 +#define SHA1_DIGEST_WORDS 5 +#define SHA1_INPUT_BYTES (SHA1_INPUT_WORDS * sizeof(uint32_t)) +#define SHA1_DIGEST_BYTES (SHA1_DIGEST_WORDS * sizeof(uint32_t)) + +class SHA1 { +public: + SHA1() : bits(0) { memcpy(H, IV, sizeof(H)); } + SHA1(const SHA1& s) { + bits = s.bits; + memcpy(H, s.H, sizeof(H)); + memcpy(M, s.M, sizeof(M)); + } + void init() { + bits = 0; + memcpy(H, IV, sizeof(H)); + } + void add(const uint8_t* data, size_t len); + uint8_t* result(); + +private: + uint64_t bits; + uint32_t H[SHA1_DIGEST_WORDS]; + uint8_t M[SHA1_INPUT_BYTES]; + + static const uint32_t IV[SHA1_DIGEST_WORDS]; + void transform(); +}; + +class HMAC { +public: + HMAC(const uint8_t* key, size_t lkey); + HMAC(const HMAC& hm) : in(hm.in), out(hm.out) {} + + void init(const uint8_t* key, size_t lkey); + + void add(const uint8_t* data, size_t len) { in.add(data, len); } + + uint8_t* result() { + out.add(in.result(), SHA1_DIGEST_BYTES); + return out.result(); + } + +private: + SHA1 in, out; +}; + +std::string GetDateString(); + +std::string GetUrlSignature(const std::string& httpMethod, + const std::string& operationType, + std::map& httpHeader, + const std::map& parameterList, + const std::string& content, + const std::string& signKey); + +} // namespace logtail diff --git a/core/plugin/flusher/sls/SendResult.cpp b/core/plugin/flusher/sls/SendResult.cpp index 955bff6a52..1f168ab261 100644 --- a/core/plugin/flusher/sls/SendResult.cpp +++ b/core/plugin/flusher/sls/SendResult.cpp @@ -14,23 +14,23 @@ #include "plugin/flusher/sls/SendResult.h" -#include "sdk/Common.h" +#include "plugin/flusher/sls/SLSConstant.h" namespace logtail { SendResult ConvertErrorCode(const std::string& errorCode) { - if (errorCode == sdk::LOGE_REQUEST_ERROR || errorCode == sdk::LOGE_CLIENT_OPERATION_TIMEOUT - || errorCode == sdk::LOGE_REQUEST_TIMEOUT) { + if (errorCode == LOGE_REQUEST_ERROR || errorCode == LOGE_CLIENT_OPERATION_TIMEOUT + || errorCode == LOGE_REQUEST_TIMEOUT) { return SEND_NETWORK_ERROR; - } else if (errorCode == sdk::LOGE_SERVER_BUSY || errorCode == sdk::LOGE_INTERNAL_SERVER_ERROR) { + } else if (errorCode == LOGE_SERVER_BUSY || errorCode == LOGE_INTERNAL_SERVER_ERROR) { return SEND_SERVER_ERROR; - } else if (errorCode == sdk::LOGE_WRITE_QUOTA_EXCEED || errorCode == sdk::LOGE_SHARD_WRITE_QUOTA_EXCEED) { + } else if (errorCode == LOGE_WRITE_QUOTA_EXCEED || errorCode == LOGE_SHARD_WRITE_QUOTA_EXCEED) { return SEND_QUOTA_EXCEED; - } else if (errorCode == sdk::LOGE_UNAUTHORIZED) { + } else if (errorCode == LOGE_UNAUTHORIZED) { return SEND_UNAUTHORIZED; - } else if (errorCode == sdk::LOGE_INVALID_SEQUENCE_ID) { + } else if (errorCode == LOGE_INVALID_SEQUENCE_ID) { return SEND_INVALID_SEQUENCE_ID; - } else if (errorCode == sdk::LOGE_PARAMETER_INVALID) { + } else if (errorCode == LOGE_PARAMETER_INVALID) { return SEND_PARAMETER_INVALID; } else { return SEND_DISCARD_ERROR; diff --git a/core/plugin/processor/ProcessorDesensitizeNative.cpp b/core/plugin/processor/ProcessorDesensitizeNative.cpp index 5cef5e64c9..47c94bfcf8 100644 --- a/core/plugin/processor/ProcessorDesensitizeNative.cpp +++ b/core/plugin/processor/ProcessorDesensitizeNative.cpp @@ -20,7 +20,7 @@ #include "models/LogEvent.h" #include "monitor/metric_constants/MetricConstants.h" #include "pipeline/plugin/instance/ProcessorInstance.h" -#include "sdk/Common.h" +#include "common/HashUtil.h" namespace logtail { @@ -230,7 +230,7 @@ void ProcessorDesensitizeNative::CastOneSensitiveWord(std::string* value) { // add : xxxx, psw destStr.append(pVal->substr(beginPos, beginOffset - beginPos)); // md5: 123abc - destStr.append(sdk::CalcMD5(pVal->substr(beginOffset, endOffset - beginOffset))); + destStr.append(CalcMD5(pVal->substr(beginOffset, endOffset - beginOffset))); beginPos = endOffset; // refine for : xxxx. psw=123abc if (endOffset >= maxSize) { diff --git a/core/prometheus/PrometheusInputRunner.cpp b/core/prometheus/PrometheusInputRunner.cpp index 052959bb15..cdb4ba46bf 100644 --- a/core/prometheus/PrometheusInputRunner.cpp +++ b/core/prometheus/PrometheusInputRunner.cpp @@ -26,14 +26,14 @@ #include "common/StringTools.h" #include "common/TimeUtil.h" #include "common/http/AsynCurlRunner.h" +#include "common/http/Constant.h" +#include "common/http/Curl.h" #include "common/timer/Timer.h" #include "logger/Logger.h" #include "monitor/metric_constants/MetricConstants.h" #include "plugin/flusher/sls/FlusherSLS.h" #include "prometheus/Constants.h" #include "prometheus/Utils.h" -#include "sdk/Common.h" -#include "sdk/Exception.h" using namespace std; @@ -49,7 +49,6 @@ PrometheusInputRunner::PrometheusInputRunner() mPodName(STRING_FLAG(_pod_name_)), mEventPool(true), mUnRegisterMs(0) { - mClient = std::make_unique(); mTimer = std::make_shared(); // self monitor @@ -151,17 +150,18 @@ void PrometheusInputRunner::Init() { int retry = 0; while (mIsThreadRunning.load()) { ++retry; - sdk::HttpMessage httpResponse = SendRegisterMessage(prometheus::REGISTER_COLLECTOR_PATH); - if (httpResponse.statusCode != 200) { + auto httpResponse = SendRegisterMessage(prometheus::REGISTER_COLLECTOR_PATH); + if (httpResponse.GetStatusCode() != 200) { mPromRegisterRetryTotal->Add(1); if (retry % 10 == 0) { - LOG_INFO(sLogger, ("register failed, retried", retry)("statusCode", httpResponse.statusCode)); + LOG_INFO(sLogger, + ("register failed, retried", retry)("statusCode", httpResponse.GetStatusCode())); } } else { // register success // response will be { "unRegisterMs": 30000 } - if (!httpResponse.content.empty()) { - string responseStr = httpResponse.content; + if (!httpResponse.GetBody()->empty()) { + string responseStr = *httpResponse.GetBody(); string errMsg; Json::Value responseJson; if (!ParseJsonTable(responseStr, responseJson, errMsg)) { @@ -222,9 +222,9 @@ void PrometheusInputRunner::Stop() { auto res = std::async(launch::async, [this]() { std::lock_guard lock(mRegisterMutex); for (int retry = 0; retry < 3; ++retry) { - sdk::HttpMessage httpResponse = SendRegisterMessage(prometheus::UNREGISTER_COLLECTOR_PATH); - if (httpResponse.statusCode != 200) { - LOG_ERROR(sLogger, ("unregister failed, statusCode", httpResponse.statusCode)); + auto httpResponse = SendRegisterMessage(prometheus::UNREGISTER_COLLECTOR_PATH); + if (httpResponse.GetStatusCode() != 200) { + LOG_ERROR(sLogger, ("unregister failed, statusCode", httpResponse.GetStatusCode())); } else { LOG_INFO(sLogger, ("Unregister Success", mPodName)); mPromRegisterState->Set(0); @@ -242,29 +242,18 @@ bool PrometheusInputRunner::HasRegisteredPlugins() const { return !mTargetSubscriberSchedulerMap.empty(); } -sdk::HttpMessage PrometheusInputRunner::SendRegisterMessage(const string& url) const { - map httpHeader; - httpHeader[sdk::X_LOG_REQUEST_ID] = prometheus::PROMETHEUS_PREFIX + mPodName; - sdk::HttpMessage httpResponse; - httpResponse.header[sdk::X_LOG_REQUEST_ID] = prometheus::PROMETHEUS_PREFIX + mPodName; +HttpResponse PrometheusInputRunner::SendRegisterMessage(const string& url) const { + HttpResponse httpResponse; #ifdef APSARA_UNIT_TEST_MAIN - httpResponse.statusCode = 200; + httpResponse.SetStatusCode(200); return httpResponse; #endif - try { - mClient->Send(sdk::HTTP_GET, - mServiceHost, - mServicePort, - url, - "pod_name=" + mPodName, - httpHeader, - "", - 10, - httpResponse, - "", - false); - } catch (const sdk::LOGException& e) { - LOG_ERROR(sLogger, ("curl error", e.what())("url", url)("pod_name", mPodName)); + map httpHeader; + if (!SendHttpRequest( + make_unique( + HTTP_GET, false, mServiceHost, mServicePort, url, "pod_name=" + mPodName, httpHeader, "", 10), + httpResponse)) { + LOG_ERROR(sLogger, ("curl error", "")("url", url)("pod_name", mPodName)); } return httpResponse; } diff --git a/core/prometheus/PrometheusInputRunner.h b/core/prometheus/PrometheusInputRunner.h index 996caf163e..b8e7719171 100644 --- a/core/prometheus/PrometheusInputRunner.h +++ b/core/prometheus/PrometheusInputRunner.h @@ -21,12 +21,11 @@ #include #include "common/Lock.h" +#include "common/http/HttpResponse.h" #include "common/timer/Timer.h" #include "monitor/metric_models/MetricTypes.h" #include "prometheus/schedulers/TargetSubscriberScheduler.h" #include "runner/InputRunner.h" -#include "sdk/Common.h" -#include "sdk/CurlImp.h" namespace logtail { @@ -56,7 +55,7 @@ class PrometheusInputRunner : public InputRunner { private: PrometheusInputRunner(); - sdk::HttpMessage SendRegisterMessage(const std::string& url) const; + HttpResponse SendRegisterMessage(const std::string& url) const; void CancelAllTargetSubscriber(); void SubscribeOnce(); @@ -74,7 +73,6 @@ class PrometheusInputRunner : public InputRunner { int32_t mServicePort; std::string mPodName; - std::unique_ptr mClient; std::shared_ptr mTimer; EventPool mEventPool; diff --git a/core/prometheus/schedulers/ScrapeConfig.cpp b/core/prometheus/schedulers/ScrapeConfig.cpp index 3946bf54a2..3bc05e9f65 100644 --- a/core/prometheus/schedulers/ScrapeConfig.cpp +++ b/core/prometheus/schedulers/ScrapeConfig.cpp @@ -10,7 +10,7 @@ #include "logger/Logger.h" #include "prometheus/Constants.h" #include "prometheus/Utils.h" -#include "sdk/Common.h" +#include "common/EncodingUtil.h" using namespace std; @@ -223,7 +223,7 @@ bool ScrapeConfig::InitBasicAuth(const Json::Value& basicAuth) { } auto token = username + ":" + password; - auto token64 = sdk::Base64Enconde(token); + auto token64 = Base64Enconde(token); mRequestHeaders[prometheus::A_UTHORIZATION] = prometheus::BASIC_PREFIX + token64; return true; } diff --git a/core/prometheus/schedulers/ScrapeScheduler.cpp b/core/prometheus/schedulers/ScrapeScheduler.cpp index 07c3291311..308dc9f713 100644 --- a/core/prometheus/schedulers/ScrapeScheduler.cpp +++ b/core/prometheus/schedulers/ScrapeScheduler.cpp @@ -23,6 +23,7 @@ #include "common/StringTools.h" #include "common/TimeUtil.h" +#include "common/http/Constant.h" #include "common/timer/HttpRequestTimerEvent.h" #include "logger/Logger.h" #include "pipeline/queue/ProcessQueueItem.h" @@ -31,7 +32,6 @@ #include "prometheus/Constants.h" #include "prometheus/async/PromFuture.h" #include "prometheus/async/PromHttpRequest.h" -#include "sdk/Common.h" using namespace std; @@ -204,7 +204,7 @@ std::unique_ptr ScrapeScheduler::BuildScrapeTimerEvent(std::chrono:: retry -= 1; } auto request = std::make_unique( - sdk::HTTP_GET, + HTTP_GET, mScrapeConfigPtr->mScheme == prometheus::HTTPS, mHost, mPort, diff --git a/core/prometheus/schedulers/TargetSubscriberScheduler.cpp b/core/prometheus/schedulers/TargetSubscriberScheduler.cpp index 7745f6dda1..ce400f77d9 100644 --- a/core/prometheus/schedulers/TargetSubscriberScheduler.cpp +++ b/core/prometheus/schedulers/TargetSubscriberScheduler.cpp @@ -20,10 +20,10 @@ #include #include -#include "Common.h" -#include "TimeUtil.h" #include "common/JsonUtil.h" #include "common/StringTools.h" +#include "common/TimeUtil.h" +#include "common/http/Constant.h" #include "common/timer/HttpRequestTimerEvent.h" #include "common/timer/Timer.h" #include "logger/Logger.h" @@ -295,7 +295,7 @@ TargetSubscriberScheduler::BuildSubscriberTimerEvent(std::chrono::steady_clock:: if (!mETag.empty()) { httpHeader[prometheus::IF_NONE_MATCH] = mETag; } - auto request = std::make_unique(sdk::HTTP_GET, + auto request = std::make_unique(HTTP_GET, false, mServiceHost, mServicePort, diff --git a/core/protobuf/sls/logtail_buffer_meta.proto b/core/protobuf/sls/logtail_buffer_meta.proto index dc2639e997..131e4099d7 100644 --- a/core/protobuf/sls/logtail_buffer_meta.proto +++ b/core/protobuf/sls/logtail_buffer_meta.proto @@ -17,10 +17,17 @@ package sls_logs; import "sls_logs.proto"; +enum EndpointMode +{ + DEFAULT = 0; + ACCELERATE = 1; + CUSTOM = 2; +} + message LogtailBufferMeta { required string project = 1; - required string endpoint = 2; + required string region = 2; required string aliuid = 3; optional string logstore = 4; optional int32 datatype = 5; @@ -28,4 +35,6 @@ message LogtailBufferMeta optional string shardhashkey = 7; optional SlsCompressType compresstype = 8; optional SlsTelemetryType telemetrytype = 9; + optional EndpointMode endpointmode = 10; + optional string endpoint = 11; } diff --git a/core/runner/FlusherRunner.cpp b/core/runner/FlusherRunner.cpp index 38b09d78de..6dbc2de68e 100644 --- a/core/runner/FlusherRunner.cpp +++ b/core/runner/FlusherRunner.cpp @@ -27,12 +27,8 @@ #include "pipeline/queue/SenderQueueManager.h" #include "plugin/flusher/sls/DiskBufferWriter.h" #include "runner/sink/http/HttpSink.h" -// TODO: temporarily used here -#include "plugin/flusher/sls/PackIdManager.h" -#include "plugin/flusher/sls/SLSClientManager.h" DEFINE_FLAG_INT32(flusher_runner_exit_timeout_secs, "", 60); -DEFINE_FLAG_INT32(check_send_client_timeout_interval, "", 600); DECLARE_FLAG_INT32(discard_send_fail_interval); @@ -189,13 +185,6 @@ void FlusherRunner::Run() { mTotalDelayMs->Add(chrono::system_clock::now() - curTime); } - // TODO: move the following logic to scheduler - if ((time(NULL) - mLastCheckSendClientTime) > INT32_FLAG(check_send_client_timeout_interval)) { - SLSClientManager::GetInstance()->CleanTimeoutClient(); - PackIdManager::GetInstance()->CleanTimeoutEntry(); - mLastCheckSendClientTime = time(NULL); - } - if (mIsFlush && SenderQueueManager::GetInstance()->IsAllQueueEmpty()) { break; } diff --git a/core/runner/sink/http/HttpSink.cpp b/core/runner/sink/http/HttpSink.cpp index 2bd2e77cb9..951096addf 100644 --- a/core/runner/sink/http/HttpSink.cpp +++ b/core/runner/sink/http/HttpSink.cpp @@ -239,17 +239,18 @@ void HttpSink::HandleCompletedRequests(int& runningHandlers) { HttpSinkRequest* request = nullptr; curl_easy_getinfo(handler, CURLINFO_PRIVATE, &request); auto responseTime = chrono::system_clock::now() - request->mLastSendTime; - auto responseTimeMs = chrono::duration_cast(responseTime).count(); + auto responseTimeMs = chrono::duration_cast(responseTime); switch (msg->data.result) { case CURLE_OK: { long statusCode = 0; curl_easy_getinfo(handler, CURLINFO_RESPONSE_CODE, &statusCode); request->mResponse.SetStatusCode(statusCode); + request->mResponse.SetResponseTime(responseTimeMs); LOG_DEBUG( sLogger, ("send http request succeeded, item address", request->mItem)( "config-flusher-dst", QueueKeyManager::GetInstance()->GetName(request->mItem->mQueueKey))( - "response time", ToString(responseTimeMs) + "ms")("try cnt", ToString(request->mTryCnt))( + "response time", ToString(responseTimeMs.count()) + "ms")("try cnt", ToString(request->mTryCnt))( "sending cnt", ToString(FlusherRunner::GetInstance()->GetSendingBufferCount()))); static_cast(request->mItem->mFlusher)->OnSendDone(request->mResponse, request->mItem); FlusherRunner::GetInstance()->DecreaseHttpSendingCnt(); @@ -281,7 +282,7 @@ void HttpSink::HandleCompletedRequests(int& runningHandlers) { ("failed to send http request", "abort")("item address", request->mItem)( "config-flusher-dst", QueueKeyManager::GetInstance()->GetName(request->mItem->mQueueKey))( - "response time", ToString(responseTimeMs) + "ms")("try cnt", + "response time", ToString(responseTimeMs.count()) + "ms")("try cnt", ToString(request->mTryCnt))( "sending cnt", ToString(FlusherRunner::GetInstance()->GetSendingBufferCount()))); static_cast(request->mItem->mFlusher) diff --git a/core/sdk/Client.cpp b/core/sdk/Client.cpp deleted file mode 100644 index c99d4dd71f..0000000000 --- a/core/sdk/Client.cpp +++ /dev/null @@ -1,455 +0,0 @@ -// Copyright 2022 iLogtail Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "Client.h" - -#include "Common.h" -#include "CurlImp.h" -#include "Exception.h" -#include "Result.h" -#include "logger/Logger.h" -#include "plugin/flusher/sls/SLSClientManager.h" -#include "app_config/AppConfig.h" -#include "monitor/Monitor.h" - -namespace logtail { -namespace sdk { - - using namespace std; - - Client::Client(const string& aliuid, const string& slsHost, int32_t timeout) - : mTimeout(timeout), mHostFieldSuffix(""), mIsHostRawIp(false), mPort(80), mUsingHTTPS(false), mAliuid(aliuid) { - mClient = new CurlClient(); - mSlsHostUpdateTime = 0; - mSlsRealIpUpdateTime = 0; - SetSlsHost(slsHost); - if (mTimeout <= 0) { - mTimeout = LOG_REQUEST_TIMEOUT; - } - } - - Client::~Client() throw() { - if (mClient != NULL) { - delete mClient; - } - } - - void Client::SetPort(int32_t port) { - mPort = port; - mUsingHTTPS = (443 == mPort); - } - - string Client::GetSlsHost() { - mSpinLock.lock(); - string slsHost = mSlsHost; - mSpinLock.unlock(); - return slsHost; - } - - string Client::GetRawSlsHost() { - mSpinLock.lock(); - string rawSlsHost = mRawSlsHost; - mSpinLock.unlock(); - return rawSlsHost; - } - - string Client::GetHostFieldSuffix() { - mSpinLock.lock(); - string hostFieldSuffix = mHostFieldSuffix; - mSpinLock.unlock(); - return hostFieldSuffix; - } - - bool Client::GetRawSlsHostFlag() { - return mIsHostRawIp; - } - - void Client::SetSlsHost(const string& slsHost) { - mSpinLock.lock(); - if (slsHost == mRawSlsHost) { - mSpinLock.unlock(); - return; - } - mRawSlsHost = slsHost; - size_t bpos = slsHost.find("://"); - if (bpos == string::npos) - bpos = 0; - else - bpos += 3; - string tmpstr = slsHost.substr(bpos); - size_t epos = tmpstr.find_first_of("/"); - if (epos == string::npos) - epos = tmpstr.length(); - string host = tmpstr.substr(0, epos); - - mSlsHost = host; - - mHostFieldSuffix = "." + host; - size_t i = 0; - for (; i < host.length(); ++i) { - if ((host[i] >= 'a' && host[i] <= 'z') || (host[i] >= 'A' && host[i] <= 'Z')) - break; - } - if (i == host.length()) - mIsHostRawIp = true; - else - mIsHostRawIp = false; - mSpinLock.unlock(); - } - - - string Client::GetHost(const string& project) { - if (mIsHostRawIp || project.empty()) { - return GetSlsHost(); - } else { - return project + GetHostFieldSuffix(); - } - } - - GetRealIpResponse Client::GetRealIp() { - static string project = "logtail-real-ip-project"; - static string logstore = "logtail-real-ip-logstore"; - GetRealIpResponse rsp; - try { - PingSLSServer(project, logstore, &rsp.realIp); - } catch (const LOGException&) { - } - return rsp; - } - - bool Client::TestNetwork() { - static string project = "logtail-test-network-project"; - static string logstore = "logtail-test-network-logstore"; - PingSLSServer(project, logstore); - return true; - } - - PostLogStoreLogsResponse Client::PostLogStoreLogs(const std::string& project, - const std::string& logstore, - sls_logs::SlsCompressType compressType, - const std::string& compressedLogGroup, - uint32_t rawSize, - const std::string& hashKey, - bool isTimeSeries) { - map httpHeader; - httpHeader[CONTENT_TYPE] = TYPE_LOG_PROTOBUF; - httpHeader[X_LOG_BODYRAWSIZE] = std::to_string(rawSize); - httpHeader[X_LOG_COMPRESSTYPE] = Client::GetCompressTypeString(compressType); - if (isTimeSeries) { - return SynPostMetricStoreLogs(project, logstore, compressedLogGroup, httpHeader); - } else { - return SynPostLogStoreLogs(project, logstore, compressedLogGroup, httpHeader, hashKey); - } - } - - PostLogStoreLogsResponse Client::PostLogStoreLogPackageList(const std::string& project, - const std::string& logstore, - sls_logs::SlsCompressType compressType, - const std::string& packageListData, - const std::string& hashKey) { - map httpHeader; - httpHeader[CONTENT_TYPE] = TYPE_LOG_PROTOBUF; - httpHeader[X_LOG_MODE] = LOG_MODE_BATCH_GROUP; - httpHeader[X_LOG_BODYRAWSIZE] = std::to_string(packageListData.size()); - httpHeader[X_LOG_COMPRESSTYPE] = Client::GetCompressTypeString(compressType); - return SynPostLogStoreLogs(project, logstore, packageListData, httpHeader, hashKey); - } - - unique_ptr Client::CreatePostLogStoreLogsRequest(const std::string& project, - const std::string& logstore, - sls_logs::SlsCompressType compressType, - const std::string& compressedLogGroup, - uint32_t rawSize, - SenderQueueItem* item, - const std::string& hashKey, - int64_t hashKeySeqID, - bool isTimeSeries) { - map httpHeader; - httpHeader[CONTENT_TYPE] = TYPE_LOG_PROTOBUF; - httpHeader[X_LOG_BODYRAWSIZE] = std::to_string(rawSize); - httpHeader[X_LOG_COMPRESSTYPE] = Client::GetCompressTypeString(compressType); - if (isTimeSeries) { - return CreateAsynPostMetricStoreLogsRequest(project, logstore, compressedLogGroup, httpHeader, item); - } else { - return CreateAsynPostLogStoreLogsRequest( - project, logstore, compressedLogGroup, httpHeader, hashKey, hashKeySeqID, item); - } - } - - - unique_ptr Client::CreatePostLogStoreLogPackageListRequest(const std::string& project, - const std::string& logstore, - sls_logs::SlsCompressType compressType, - const std::string& packageListData, - SenderQueueItem* item, - const std::string& hashKey) { - map httpHeader; - httpHeader[CONTENT_TYPE] = TYPE_LOG_PROTOBUF; - httpHeader[X_LOG_MODE] = LOG_MODE_BATCH_GROUP; - httpHeader[X_LOG_BODYRAWSIZE] = std::to_string(packageListData.size()); - httpHeader[X_LOG_COMPRESSTYPE] = Client::GetCompressTypeString(compressType); - return CreateAsynPostLogStoreLogsRequest( - project, logstore, packageListData, httpHeader, hashKey, kInvalidHashKeySeqID, item); - } - - void Client::SendRequest(const std::string& project, - const std::string& httpMethod, - const std::string& url, - const std::string& body, - const std::map& parameterList, - std::map& header, - HttpMessage& httpMessage, - std::string* realIpPtr) { - SLSClientManager::AuthType type; - string accessKeyId, accessKeySecret; - if (!SLSClientManager::GetInstance()->GetAccessKey(mAliuid, type, accessKeyId, accessKeySecret)) { - throw LOGException(LOGE_UNAUTHORIZED, ""); - } - if (type == SLSClientManager::AuthType::ANONYMOUS) { - header[X_LOG_KEYPROVIDER] = MD5_SHA1_SALT_KEYPROVIDER; - } - - string host = GetHost(project); - SetCommonHeader(header, (int32_t)(body.length()), project); - string signature = GetUrlSignature(httpMethod, url, header, parameterList, body, accessKeySecret); - header[AUTHORIZATION] = LOG_HEADSIGNATURE_PREFIX + accessKeyId + ':' + signature; - - string queryString; - GetQueryString(parameterList, queryString); - - int32_t port = mPort; - if (mPort == 80 && mUsingHTTPS) { - port = 443; - } - mClient->Send( - httpMethod, host, port, url, queryString, header, body, mTimeout, httpMessage, AppConfig::GetInstance()->GetBindInterface(), mUsingHTTPS); - - if (httpMessage.statusCode != 200) { - if (realIpPtr != NULL) { - *realIpPtr = httpMessage.header[X_LOG_HOSTIP]; - } - ErrorCheck(httpMessage.content, httpMessage.header[X_LOG_REQUEST_ID], httpMessage.statusCode); - } - } - - std::unique_ptr - Client::CreateAsynPostMetricStoreLogsRequest(const std::string& project, - const std::string& logstore, - const std::string& body, - std::map& httpHeader, - SenderQueueItem* item) { - SLSClientManager::AuthType type; - string accessKeyId, accessKeySecret; - if (!SLSClientManager::GetInstance()->GetAccessKey(mAliuid, type, accessKeyId, accessKeySecret)) { - return nullptr; - } - if (type == SLSClientManager::AuthType::ANONYMOUS) { - httpHeader[X_LOG_KEYPROVIDER] = MD5_SHA1_SALT_KEYPROVIDER; - } - - string operation = METRICSTORES; - operation.append("/").append(project).append("/").append(logstore).append("/api/v1/write"); - httpHeader[CONTENT_MD5] = CalcMD5(body); - map parameterList; - string host = GetSlsHost(); - SetCommonHeader(httpHeader, (int32_t)(body.length()), ""); - string signature = GetUrlSignature(HTTP_POST, operation, httpHeader, parameterList, body, accessKeySecret); - httpHeader[AUTHORIZATION] = LOG_HEADSIGNATURE_PREFIX + accessKeyId + ':' + signature; - return make_unique(HTTP_POST, mUsingHTTPS, host, mPort, operation, "", httpHeader, body, item); - } - - unique_ptr - Client::CreateAsynPostLogStoreLogsRequest(const std::string& project, - const std::string& logstore, - const std::string& body, - std::map& httpHeader, - const std::string& hashKey, - int64_t hashKeySeqID, - SenderQueueItem* item) { - SLSClientManager::AuthType type; - string accessKeyId, accessKeySecret; - if (!SLSClientManager::GetInstance()->GetAccessKey(mAliuid, type, accessKeyId, accessKeySecret)) { - return nullptr; - } - if (type == SLSClientManager::AuthType::ANONYMOUS) { - httpHeader[X_LOG_KEYPROVIDER] = MD5_SHA1_SALT_KEYPROVIDER; - } - - string operation = LOGSTORES; - operation.append("/").append(logstore); - if (hashKey.empty()) - operation.append("/shards/lb"); - else - operation.append("/shards/route"); - - httpHeader[CONTENT_MD5] = CalcMD5(body); - - map parameterList; - if (!hashKey.empty()) { - parameterList["key"] = hashKey; - if (hashKeySeqID != kInvalidHashKeySeqID) { - parameterList["seqid"] = std::to_string(hashKeySeqID); - } - } - - string host = GetHost(project); - SetCommonHeader(httpHeader, (int32_t)(body.length()), project); - string signature = GetUrlSignature(HTTP_POST, operation, httpHeader, parameterList, body, accessKeySecret); - httpHeader[AUTHORIZATION] = LOG_HEADSIGNATURE_PREFIX + accessKeyId + ':' + signature; - - string queryString; - GetQueryString(parameterList, queryString); - - return make_unique( - HTTP_POST, mUsingHTTPS, host, mPort, operation, queryString, httpHeader, body, item); - } - - PostLogStoreLogsResponse - Client::PingSLSServer(const std::string& project, const std::string& logstore, std::string* realIpPtr) { - sls_logs::LogGroup logGroup; - logGroup.set_source(LoongCollectorMonitor::mIpAddr); - auto serializeData = logGroup.SerializeAsString(); - - std::map httpHeader; - httpHeader[CONTENT_TYPE] = TYPE_LOG_PROTOBUF; - httpHeader[X_LOG_BODYRAWSIZE] = std::to_string(serializeData.size()); - return SynPostLogStoreLogs(project, logstore, serializeData, httpHeader, "", realIpPtr); - } - - PostLogStoreLogsResponse Client::SynPostLogStoreLogs(const std::string& project, - const std::string& logstore, - const std::string& body, - std::map& httpHeader, - const std::string& hashKey, - std::string* realIpPtr) { - string operation = LOGSTORES; - operation.append("/").append(logstore); - if (hashKey.empty()) - operation.append("/shards/lb"); - else - operation.append("/shards/route"); - - httpHeader[CONTENT_MD5] = CalcMD5(body); - - map parameterList; - if (!hashKey.empty()) - parameterList["key"] = hashKey; - - HttpMessage httpResponse; - SendRequest(project, HTTP_POST, operation, body, parameterList, httpHeader, httpResponse, realIpPtr); - - PostLogStoreLogsResponse ret; - ret.bodyBytes = (int32_t)body.size(); - ret.statusCode = httpResponse.statusCode; - ret.requestId = httpResponse.header[X_LOG_REQUEST_ID]; - return ret; - } - - PostLogStoreLogsResponse Client::SynPostMetricStoreLogs(const std::string& project, - const std::string& logstore, - const std::string& body, - std::map& httpHeader, - std::string* realIpPtr) { - string operation = METRICSTORES; - operation.append("/").append(project).append("/").append(logstore).append("/api/v1/write"); - httpHeader[CONTENT_MD5] = CalcMD5(body); - map parameterList; - HttpMessage httpResponse; - SendRequest(project, HTTP_POST, operation, body, parameterList, httpHeader, httpResponse, realIpPtr); - PostLogStoreLogsResponse ret; - ret.bodyBytes = (int32_t)body.size(); - ret.statusCode = httpResponse.statusCode; - ret.requestId = httpResponse.header[X_LOG_REQUEST_ID]; - return ret; - } - - PostLogStoreLogsResponse Client::PostLogUsingWebTracking(const std::string& project, - const std::string& logstore, - sls_logs::SlsCompressType compressType, - const std::string& compressedLog, - uint32_t rawSize) { - map httpHeader; - httpHeader[X_LOG_COMPRESSTYPE] = Client::GetCompressTypeString(compressType); - httpHeader[X_LOG_BODYRAWSIZE] = std::to_string(rawSize); - SetCommonHeader(httpHeader, (int32_t)(compressedLog.length()), project); - - string operation = LOGSTORES; - operation.append("/").append(logstore).append("/track"); - - string host = GetHost(project); - int32_t port = mPort; - if (mPort == 80 && mUsingHTTPS) { - port = 443; - } - - HttpMessage httpResponse; - mClient->Send(HTTP_POST, - host, - port, - operation, - "", - httpHeader, - compressedLog, - mTimeout, - httpResponse, - AppConfig::GetInstance()->GetBindInterface(), - mUsingHTTPS); - - PostLogStoreLogsResponse ret; - ret.bodyBytes = (int32_t)compressedLog.length(); - ret.statusCode = httpResponse.statusCode; - ret.requestId = httpResponse.header[X_LOG_REQUEST_ID]; - return ret; - } - - void Client::SetCommonHeader(map& httpHeader, int32_t contentLength, const string& project) { - if (!project.empty()) { - httpHeader[HOST] = project + GetHostFieldSuffix(); - } else { - httpHeader[HOST] = GetSlsHost(); - } - - httpHeader[USER_AGENT] = SLSClientManager::GetInstance()->GetUserAgent(); - httpHeader[X_LOG_APIVERSION] = LOG_API_VERSION; - httpHeader[X_LOG_SIGNATUREMETHOD] = HMAC_SHA1; - httpHeader[DATE] = GetDateString(); - httpHeader[CONTENT_LENGTH] = std::to_string(contentLength); - } - - std::string Client::GetCompressTypeString(sls_logs::SlsCompressType compressType) { - switch (compressType) { - case sls_logs::SLS_CMP_NONE: - return ""; - case sls_logs::SLS_CMP_LZ4: - return LOG_LZ4; - case sls_logs::SLS_CMP_ZSTD: - return LOG_ZSTD; - default: - return LOG_LZ4; - } - } - - sls_logs::SlsCompressType Client::GetCompressType(std::string compressTypeString, - sls_logs::SlsCompressType defaultType) { - if (compressTypeString == "none") { - return sls_logs::SLS_CMP_NONE; - } else if (compressTypeString == LOG_LZ4) { - return sls_logs::SLS_CMP_LZ4; - } else if (compressTypeString == LOG_ZSTD) { - return sls_logs::SLS_CMP_ZSTD; - } - return defaultType; - } -} // namespace sdk -} // namespace logtail diff --git a/core/sdk/Client.h b/core/sdk/Client.h deleted file mode 100644 index 073f39b657..0000000000 --- a/core/sdk/Client.h +++ /dev/null @@ -1,223 +0,0 @@ -/* - * Copyright 2022 iLogtail Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#include -#include - -#include "Common.h" -#include "CurlImp.h" -#include "protobuf/sls/sls_logs.pb.h" -#include "runner/sink/http/HttpSinkRequest.h" - -namespace logtail { -namespace sdk { - - class Client { - public: - /** Constructor needs at least three parameters. - * @param LOGHost LOG service address, for example:http://cn-hangzhou.log.aliyuncs.com. - * @param timeout Timeout time of one operation. - */ - Client(const std::string& aliuid, - const std::string& slsHost, - int32_t timeout = LOG_REQUEST_TIMEOUT); - ~Client() throw(); - - void SetPort(int32_t port); - - GetRealIpResponse GetRealIp(); - bool TestNetwork(); - - std::string GetHost(const std::string& project); - - void SetSlsHost(const std::string& slsHost); - std::string GetSlsHost(); - std::string GetRawSlsHost(); - std::string GetHostFieldSuffix(); - bool GetRawSlsHostFlag(); - - void SetSlsHostUpdateTime(int32_t uptime) { mSlsHostUpdateTime = uptime; } - int32_t GetSlsHostUpdateTime() { return mSlsHostUpdateTime; } - - void SetSlsRealIpUpdateTime(int32_t uptime) { mSlsRealIpUpdateTime = uptime; } - int32_t GetSlsRealIpUpdateTime() { return mSlsRealIpUpdateTime; } - bool IsUsingHTTPS() { return mUsingHTTPS; } - - /////////////////////////////////////Internal Interface For Logtail//////////////////////////////////////// - /** Sync Put data to LOG service. Unsuccessful opertaion will cause an LOGException. - * @param project The project name - * @param logstore The logstore name - * @param compressedLogGroup serialized data of logGroup, LZ4 or ZSTD comressed - * @param rawSize before compress - * @param compressType compression type - * @return request_id. - */ - PostLogStoreLogsResponse PostLogStoreLogs(const std::string& project, - const std::string& logstore, - sls_logs::SlsCompressType compressType, - const std::string& compressedLogGroup, - uint32_t rawSize, - const std::string& hashKey = "", - bool isTimeSeries = false); - - PostLogStoreLogsResponse PostMetricStoreLogs(const std::string& project, - const std::string& logstore, - sls_logs::SlsCompressType compressType, - const std::string& compressedLogGroup, - uint32_t rawSize) { - return PostLogStoreLogs(project, logstore, compressType, compressedLogGroup, rawSize, "", true); - } - - - /** Sync Put data to LOG service. Unsuccessful opertaion will cause an LOGException. - * @param project The project name - * @param logstore The logstore name - * @param packageListData data of logPackageList, consist of several LogGroup - * @return request_id. - */ - PostLogStoreLogsResponse PostLogStoreLogPackageList(const std::string& project, - const std::string& logstore, - sls_logs::SlsCompressType compressType, - const std::string& packageListData, - const std::string& hashKey = ""); - /** Async Put data to LOG service. Unsuccessful opertaion will cause an LOGException. - * @param project The project name - * @param logstore The logstore name - * @param compressedLogGroup data of logGroup, LZ4 comressed - * @param rawSize before compress - * @param compressType compression type - * @return request_id. - */ - std::unique_ptr CreatePostLogStoreLogsRequest(const std::string& project, - const std::string& logstore, - sls_logs::SlsCompressType compressType, - const std::string& compressedLogGroup, - uint32_t rawSize, - SenderQueueItem* item, - const std::string& hashKey = "", - int64_t hashKeySeqID = kInvalidHashKeySeqID, - bool isTimeSeries = false); - /** Async Put metrics data to SLS metricstore. Unsuccessful opertaion will cause an LOGException. - * @param project The project name - * @param logstore The logstore name - * @param compressedLogGroup data of logGroup, LZ4 comressed - * @param rawSize before compress - * @param compressType compression type - * @return request_id. - */ - std::unique_ptr CreatePostMetricStoreLogsRequest(const std::string& project, - const std::string& logstore, - sls_logs::SlsCompressType compressType, - const std::string& compressedLogGroup, - uint32_t rawSize, - SenderQueueItem* item) { - return CreatePostLogStoreLogsRequest( - project, logstore, compressType, compressedLogGroup, rawSize, item, "", kInvalidHashKeySeqID, true); - } - - - /** Async Put data to LOG service. Unsuccessful opertaion will cause an LOGException. - * @param project The project name - * @param logstore The logstore name - * @param packageListData data of logPackageList, consist of several LogGroup - * @return request_id. - */ - std::unique_ptr CreatePostLogStoreLogPackageListRequest(const std::string& project, - const std::string& logstore, - sls_logs::SlsCompressType compressType, - const std::string& packageListData, - SenderQueueItem* item, - const std::string& hashKey = ""); - - PostLogStoreLogsResponse PostLogUsingWebTracking(const std::string& project, - const std::string& logstore, - sls_logs::SlsCompressType compressType, - const std::string& compressedLogGroup, - uint32_t rawSize); - ///////////////////////////////////////////////////////////////////////////////////////////////// - - static std::string GetCompressTypeString(sls_logs::SlsCompressType compressType); - static sls_logs::SlsCompressType GetCompressType(std::string compressTypeString, - sls_logs::SlsCompressType defaultType = sls_logs::SLS_CMP_LZ4); - - protected: - void SendRequest(const std::string& project, - const std::string& httpMethod, - const std::string& url, - const std::string& body, - const std::map& parameterList, - std::map& header, - HttpMessage& httpMessage, - std::string* realIpPtr = NULL); - - std::unique_ptr - CreateAsynPostLogStoreLogsRequest(const std::string& project, - const std::string& logstore, - const std::string& body, - std::map& httpHeader, - const std::string& hashKey, - int64_t hashKeySeqID, - SenderQueueItem* item); - - std::unique_ptr - CreateAsynPostMetricStoreLogsRequest(const std::string& project, - const std::string& logstore, - const std::string& body, - std::map& httpHeader, - SenderQueueItem* item); - - // PingSLSServer sends a trivial data packet to SLS for some inner purposes. - PostLogStoreLogsResponse - PingSLSServer(const std::string& project, const std::string& logstore, std::string* realIpPtr = NULL); - - PostLogStoreLogsResponse SynPostLogStoreLogs(const std::string& project, - const std::string& logstore, - const std::string& body, - std::map& httpHeader, - const std::string& hashKey, - std::string* realIpPtr = NULL); - - PostLogStoreLogsResponse SynPostMetricStoreLogs(const std::string& project, - const std::string& logstore, - const std::string& body, - std::map& httpHeader, - std::string* realIpPtr = NULL); - - void SetCommonHeader(std::map& httpHeader, - int32_t contentLength, - const std::string& project = ""); - - protected: - int32_t mSlsHostUpdateTime; - int32_t mSlsRealIpUpdateTime; - std::string mRawSlsHost; - std::string mSlsHost; - int32_t mTimeout; - std::string mHostFieldSuffix; - bool mIsHostRawIp; - int32_t mPort; - bool mUsingHTTPS; - std::string mAliuid; - - SpinLock mSpinLock; - - CurlClient* mClient; - }; - -} // namespace sdk - -} // namespace logtail diff --git a/core/sdk/Common.cpp b/core/sdk/Common.cpp deleted file mode 100644 index 8f3a1a7890..0000000000 --- a/core/sdk/Common.cpp +++ /dev/null @@ -1,883 +0,0 @@ -// Copyright 2022 iLogtail Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "Common.h" -#include "app_config/AppConfig.h" -#include "common/TimeUtil.h" -#include "common/StringTools.h" -#include "common/ErrorUtil.h" -#include "logger/Logger.h" - -using namespace std; -using namespace logtail::sdk; - -namespace logtail { -namespace sdk { - const char* const LOG_HEADSIGNATURE_PREFIX = "LOG "; - const char* const LOGE_REQUEST_ERROR = "RequestError"; - const char* const LOGE_INVALID_HOST = "InvalidHost"; - const char* const LOGE_UNKNOWN_ERROR = "UnknownError"; - const char* const LOGE_NOT_IMPLEMENTED = "NotImplemented"; - const char* const LOGE_SERVER_BUSY = "ServerBusy"; - const char* const LOGE_INTERNAL_SERVER_ERROR = "InternalServerError"; - const char* const LOGE_RESPONSE_SIG_ERROR = "ResponseSignatureError"; - const char* const LOGE_PARAMETER_INVALID = "ParameterInvalid"; - const char* const LOGE_MISSING_PARAMETER = "MissingParameter"; - const char* const LOGE_INVALID_METHOD = "InvalidMethod"; - const char* const LOGE_BAD_RESPONSE = "BadResponse"; - const char* const LOGE_UNAUTHORIZED = "Unauthorized"; - const char* const LOGE_QUOTA_EXCEED = "ExceedQuota"; - const char* const LOGE_REQUEST_TIMEOUT = "RequestTimeout"; - const char* const LOGE_CLIENT_OPERATION_TIMEOUT = "ClientOpertaionTimeout"; - const char* const LOGE_CLIENT_NETWORK_ERROR = "ClientNetworkError"; - const char* const LOGE_USER_NOT_EXIST = "UserNotExist"; - const char* const LOGE_CATEGORY_NOT_EXIST = "CategoryNotExist"; - const char* const LOGE_TOPIC_NOT_EXIST = "TopicNotExist"; - const char* const LOGE_POST_BODY_INVALID = "PostBodyInvalid"; - const char* const LOGE_INVALID_CONTENTTYPE = "InvalidContentType"; - const char* const LOGE_INVALID_CONTENLENGTH = "InvalidContentLength"; - const char* const LOGE_INVALID_APIVERSION = "InvalidAPIVersion"; - const char* const LOGE_PROJECT_NOT_EXIST = "ProjectNotExist"; - const char* const LOGE_MACHINEGROUP_NOT_EXIST = "MachineGroupNotExist"; - const char* const LOGE_MACHINEGROUP_ALREADY_EXIST = "MachineGroupAlreadyExist"; - const char* const LOGE_CONFIG_NOT_EXIST = "ConfigNotExist"; - const char* const LOGE_CONFIG_ALREADY_EXIST = "ConfigAlreadyExist"; - const char* const LOGE_LOGSTORE_NOT_EXIST = "LogStoreNotExist"; - const char* const LOGE_INVALID_ACCESSKEYID = "InvalidAccessKeyId"; - const char* const LOGE_SIGNATURE_NOT_MATCH = "SignatureNotMatch"; - const char* const LOGE_PROJECT_FORBIDDEN = "ProjectForbidden"; - const char* const LOGE_WRITE_QUOTA_EXCEED = "WriteQuotaExceed"; - const char* const LOGE_READ_QUOTA_EXCEED = "ReadQuotaExceed"; - const char* const LOGE_REQUEST_TIME_EXPIRED = "RequestTimeExpired"; - const char* const LOGE_INVALID_REQUEST_TIME = "InvalidRequestTime"; - const char* const LOGE_POST_BODY_TOO_LARGE = "PostBodyTooLarge"; - const char* const LOGE_INVALID_TIME_RANGE = "InvalidTimeRange"; - const char* const LOGE_INVALID_REVERSE = "InvalidReverse"; - const char* const LOGE_LOGSTORE_WITHOUT_SHARD = "LogStoreWithoutShard"; - const char* const LOGE_SHARD_WRITE_QUOTA_EXCEED = "ShardWriteQuotaExceed"; - const char* const LOGE_SHARD_READ_QUOTA_EXCEED = "ShardReadQuotaExceed"; - const char* const LOGE_INVALID_SEQUENCE_ID = "InvalidSequenceId"; - - const char* const LOGSTORES = "/logstores"; - const char* const METRICSTORES = "/prometheus"; - const char* const SHARDS = "/shards"; - const char* const INDEX = "/index"; - const char* const CONFIGS = "/configs"; - const char* const MACHINES = "/machines"; - const char* const MACHINEGROUPS = "/machinegroups"; - const char* const ACLS = "/acls"; - const char* const CONFIGSERVERAGENT = "/Agent"; - - const char* const HTTP_GET = "GET"; - const char* const HTTP_POST = "POST"; - const char* const HTTP_PUT = "PUT"; - const char* const HTTP_DELETE = "DELETE"; - - const char* const HOST = "Host"; - const char* const DATE = "Date"; - const char* const USER_AGENT = "User-Agent"; - const char* const LOG_OLD_HEADER_PREFIX = "x-sls-"; - const char* const LOG_HEADER_PREFIX = "x-log-"; - const char* const ACS_HEADER_PREFIX = "x-acs-"; - const char* const X_LOG_KEYPROVIDER = "x-log-keyprovider"; - const char* const X_LOG_APIVERSION = "x-log-apiversion"; - const char* const X_LOG_COMPRESSTYPE = "x-log-compresstype"; - const char* const X_LOG_BODYRAWSIZE = "x-log-bodyrawsize"; - const char* const X_LOG_SIGNATUREMETHOD = "x-log-signaturemethod"; - const char* const X_ACS_SECURITY_TOKEN = "x-acs-security-token"; - const char* const X_LOG_CURSOR = "x-log-cursor"; - const char* const X_LOG_REQUEST_ID = "x-log-requestid"; - const char* const X_LOG_MODE = "x-log-mode"; - - const char* const X_LOG_PROGRESS = "x-log-progress"; - const char* const X_LOG_COUNT = "x-log-count"; - const char* const X_LOG_HOSTIP = "x-log-hostip"; - - const char* const HTTP_ACCEPT = "accept"; - const char* const DEFLATE = "deflate"; - const char* const HMAC_SHA1 = "hmac-sha1"; - const char* const CONTENT_TYPE = "Content-Type"; - const char* const CONTENT_LENGTH = "Content-Length"; - const char* const CONTENT_MD5 = "Content-MD5"; - const char* const AUTHORIZATION = "Authorization"; - const char* const SIGNATURE = "Signature"; - const char* const ACCEPT_ENCODING = "Accept-Encoding"; - const char* const ENCONDING_GZIP = "gzip"; - const char* const TYPE_LOG_PROTOBUF = "application/x-protobuf"; - const char* const TYPE_LOG_JSON = "application/json"; - const char* const LOG_MODE_BATCH_GROUP = "batch_group"; - const char* const LOGITEM_TIME_STAMP_LABEL = "__time__"; - const char* const LOGITEM_SOURCE_LABEL = "__source__"; - const char* const LOG_API_VERSION = "0.6.0"; - const char* const LOGTAIL_USER_AGENT = "ali-log-logtail"; - const char* const MD5_SHA1_SALT_KEYPROVIDER = "md5-sha1-salt"; - const char* const LOG_TYPE_CURSOR = "cursor"; - const char* const LOG_TYPE = "type"; - const char* const LOGE_NOT_SUPPORTED_ACCEPT_CONTENT_TYPE = "InvalidAcceptContentType"; - const char* const LOGE_NOT_SUPPORTED_ACCEPT_ENCODING = "InvalidAcceptEncoding"; - const char* const LOGE_SHARD_NOT_EXIST = "ShardNotExist"; - const char* const LOGE_INVALID_CURSOR = "InvalidCursor"; - const char* const LOG_LZ4 = "lz4"; - const char* const LOG_DEFLATE = "deflate"; - const char* const LOG_ZSTD = "zstd"; - const char* const LOG_ERROR_CODE = "errorCode"; - const char* const LOG_ERROR_MESSAGE = "errorMessage"; - - const char* const LOG_SHARD_STATUS_READWRITE = "readwrite"; - const char* const LOG_SHARD_STATUS_READONLY = "readonly"; - - bool caseInsensitiveComp(const char lhs, const char rhs) { - return tolower(lhs) < tolower(rhs); - } - - bool compareHeader(const std::string& lhs, const std::string& rhs) { - return std::lexicographical_compare(lhs.begin(), lhs.end(), rhs.begin(), rhs.end(), caseInsensitiveComp); - } - - bool HttpMessage::IsLogServiceResponse() const { - if (!AppConfig::GetInstance()->IsResponseVerificationEnabled()) { - return true; - } - - const auto iter = header.find(X_LOG_REQUEST_ID); - if (iter == header.end()) { - return false; - } - return !iter->second.empty(); - } - - static unsigned char ToHex(unsigned char x) { - return x > 9 ? x + 55 : x + 48; - } - - static unsigned char FromHex(unsigned char x) { - unsigned char y = '\0'; - if (x >= 'A' && x <= 'Z') - y = x - 'A' + 10; - else if (x >= 'a' && x <= 'z') - y = x - 'a' + 10; - else if (x >= '0' && x <= '9') - y = x - '0'; - else - assert(0); - return y; - } - - - static std::string HexToString(const uint8_t md5[16]) { - static const char* table = "0123456789ABCDEF"; - std::string ss(32, 'a'); - for (int i = 0; i < 16; ++i) { - ss[i * 2] = table[md5[i] >> 4]; - ss[i * 2 + 1] = table[md5[i] & 0x0F]; - } - return ss; - } - - std::string CalcMD5(const std::string& message) { - uint8_t md5[MD5_BYTES]; - DoMd5((const uint8_t*)message.data(), message.length(), md5); - return HexToString(md5); - } - - std::string CalcSHA1(const std::string& message, const std::string& key) { - HMAC hmac(reinterpret_cast(key.data()), key.size()); - hmac.add(reinterpret_cast(message.data()), message.size()); - return string(reinterpret_cast(hmac.result()), SHA1_DIGEST_BYTES); - } - - - void Base64Encoding(std::istream& is, std::ostream& os, char makeupChar, const char* alphabet) { - int out[4]; - int remain = 0; - while (!is.eof()) { - int byte1 = is.get(); - if (byte1 < 0) { - break; - } - int byte2 = is.get(); - int byte3; - if (byte2 < 0) { - byte2 = 0; - byte3 = 0; - remain = 1; - } else { - byte3 = is.get(); - if (byte3 < 0) { - byte3 = 0; - remain = 2; - } - } - out[0] = static_cast(byte1) >> 2; - out[1] = ((byte1 & 0x03) << 4) + (static_cast(byte2) >> 4); - out[2] = ((byte2 & 0x0F) << 2) + (static_cast(byte3) >> 6); - out[3] = byte3 & 0x3F; - - if (remain == 1) { - os.put(out[0] = alphabet[out[0]]); - os.put(out[1] = alphabet[out[1]]); - os.put(makeupChar); - os.put(makeupChar); - } else if (remain == 2) { - os.put(out[0] = alphabet[out[0]]); - os.put(out[1] = alphabet[out[1]]); - os.put(out[2] = alphabet[out[2]]); - os.put(makeupChar); - } else { - os.put(out[0] = alphabet[out[0]]); - os.put(out[1] = alphabet[out[1]]); - os.put(out[2] = alphabet[out[2]]); - os.put(out[3] = alphabet[out[3]]); - } - } - } - - - std::string Base64Enconde(const std::string& message) { - std::istringstream iss(message); - std::ostringstream oss; - Base64Encoding(iss, oss); - return oss.str(); - } - - - std::string UrlEncode(const std::string& str) { - std::string strTemp; - size_t length = str.length(); - for (size_t i = 0; i < length; i++) { - if (isalnum((unsigned char)str[i]) || (str[i] == '-') || (str[i] == '_') || (str[i] == '.') - || (str[i] == '~')) - strTemp += str[i]; - else if (str[i] == ' ') - strTemp += "+"; - else { - strTemp += '%'; - strTemp += ToHex((unsigned char)str[i] >> 4); - strTemp += ToHex((unsigned char)str[i] % 16); - } - } - return strTemp; - } - - - std::string UrlDecode(const std::string& str) { - std::string strTemp = ""; - size_t length = str.length(); - for (size_t i = 0; i < length; i++) { - if (str[i] == '+') - strTemp += ' '; - else if (str[i] == '%') { - assert(i + 2 < length); - unsigned char high = FromHex((unsigned char)str[++i]); - unsigned char low = FromHex((unsigned char)str[++i]); - strTemp += high * 16 + low; - } else - strTemp += str[i]; - } - return strTemp; - } - - std::string GetDateString(const std::string& dateFormat) { - time_t now_time; - time(&now_time); - if (AppConfig::GetInstance()->EnableLogTimeAutoAdjust()) { - now_time += GetTimeDelta(); - } - char buffer[128] = {'\0'}; - tm timeInfo; -#if defined(__linux__) - gmtime_r(&now_time, &timeInfo); -#elif defined(_MSC_VER) - gmtime_s(&timeInfo, &now_time); -#endif - strftime(buffer, 128, dateFormat.c_str(), &timeInfo); - return string(buffer); - } - - std::string GetDateString() { - return GetDateString(DATE_FORMAT_RFC822); - } - - time_t DecodeDateString(const std::string dateString, const std::string& dateFormat) { - return 0; - // struct tm t; - // memset(&t, 0, sizeof(t)); - // t.tm_sec = -1; - // strptime(dateString.c_str(), dateFormat.c_str(),&t); - // if(t.tm_sec == -1) - // { - // throw LOGException(LOGE_PARAMETER_INVALID, string("Invalid date string:") + dateString + ",format:" + - // dateFormat); - // } - // struct timezone tz; - // struct timeval tv; - // gettimeofday(&tv, &tz); - // return mktime(&t)-tz.tz_minuteswest*60; - } - - bool StartWith(const std::string& input, const std::string& pattern) { - if (input.length() < pattern.length()) { - return false; - } - - size_t i = 0; - while (i < pattern.length() && input[i] == pattern[i]) { - i++; - } - - return i == pattern.length(); - } - - void GetQueryString(const map& parameterList, string& queryString) { - queryString.clear(); - for (map::const_iterator iter = parameterList.begin(); iter != parameterList.end(); ++iter) { - if (iter != parameterList.begin()) { - queryString.append("&"); - } - queryString.append(iter->first); - queryString.append("="); - queryString.append(UrlEncode(iter->second)); - } - } - - string GetUrlSignature(const string& httpMethod, - const string& operationType, - map& httpHeader, - const map& parameterList, - const string& content, - const string& signKey) { - string contentMd5; - string signature; - string osstream; - if (!content.empty()) { - contentMd5 = CalcMD5(content); - } - string contentType; - map::iterator iter = httpHeader.find(CONTENT_TYPE); - if (iter != httpHeader.end()) { - contentType = iter->second; - } - std::map endingMap; - osstream.append(httpMethod); - osstream.append("\n"); - osstream.append(contentMd5); - osstream.append("\n"); - osstream.append(contentType); - osstream.append("\n"); - osstream.append(httpHeader[DATE]); - osstream.append("\n"); - for (map::const_iterator iter = httpHeader.begin(); iter != httpHeader.end(); ++iter) { - if (StartWith(iter->first, LOG_OLD_HEADER_PREFIX)) { - std::string key = iter->first; - endingMap.insert(std::make_pair(key.replace(0, std::strlen(LOG_OLD_HEADER_PREFIX), LOG_HEADER_PREFIX), - iter->second)); - } else if (StartWith(iter->first, LOG_HEADER_PREFIX) || StartWith(iter->first, ACS_HEADER_PREFIX)) { - endingMap.insert(std::make_pair(iter->first, iter->second)); - } - } - for (map::const_iterator it = endingMap.begin(); it != endingMap.end(); ++it) { - osstream.append(it->first); - osstream.append(":"); - osstream.append(it->second); - osstream.append("\n"); - } - osstream.append(operationType); - if (parameterList.size() > 0) { - osstream.append("?"); - for (map::const_iterator iter = parameterList.begin(); iter != parameterList.end(); - ++iter) { - if (iter != parameterList.begin()) { - osstream.append("&"); - } - osstream.append(iter->first); - osstream.append("="); - osstream.append(iter->second); - } - } - - signature = Base64Enconde(CalcSHA1(osstream, signKey)); - - return signature; - } - - -/////////////////////////////////////////////// MACRO ////////////////////////////////////////////////// -#define SHIFT_LEFT(a, b) ((a) << (b) | (a) >> (32 - b)) - -/** - * each operation - */ -#define F(b, c, d) (((b) & (c)) | ((~(b)) & (d))) -#define G(b, c, d) (((d) & (b)) | ((~(d)) & (c))) -#define H(b, c, d) ((b) ^ (c) ^ (d)) -#define I(b, c, d) ((c) ^ ((b) | (~(d)))) - -/** - * each round - */ -#define FF(a, b, c, d, word, shift, k) \ - { \ - (a) += F((b), (c), (d)) + (word) + (k); \ - (a) = SHIFT_LEFT((a), (shift)); \ - (a) += (b); \ - } -#define GG(a, b, c, d, word, shift, k) \ - { \ - (a) += G((b), (c), (d)) + (word) + (k); \ - (a) = SHIFT_LEFT((a), (shift)); \ - (a) += (b); \ - } -#define HH(a, b, c, d, word, shift, k) \ - { \ - (a) += H((b), (c), (d)) + (word) + (k); \ - (a) = SHIFT_LEFT((a), (shift)); \ - (a) += (b); \ - } -#define II(a, b, c, d, word, shift, k) \ - { \ - (a) += I((b), (c), (d)) + (word) + (k); \ - (a) = SHIFT_LEFT((a), (shift)); \ - (a) += (b); \ - } - ////////////////////////////////////////////////////////// GLOBAL VARIABLE - //////////////////////////////////////////////////////////// - const uint8_t gPadding[64] = {0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}; - - - //////////////////////////////////////////////////////// LOCAL DECLEARATION - ///////////////////////////////////////////////////////////// - struct Md5Block { - uint32_t word[16]; - }; - /** - * copy a pool into a block, using little endian - */ - void CopyBytesToBlock(const uint8_t* poolIn, struct Md5Block& block) { - uint32_t j = 0; - for (uint32_t i = 0; i < 16; ++i, j += 4) { - block.word[i] = ((uint32_t)poolIn[j]) | (((uint32_t)poolIn[j + 1]) << 8) | (((uint32_t)poolIn[j + 2]) << 16) - | (((uint32_t)poolIn[j + 3]) << 24); - } - } - - /** - * calculate md5 hash value from a block - */ - void CalMd5(struct Md5Block block, uint32_t h[4]) { - uint32_t a = h[0]; - uint32_t b = h[1]; - uint32_t c = h[2]; - uint32_t d = h[3]; - - // Round 1 - FF(a, b, c, d, block.word[0], 7, 0xd76aa478); - FF(d, a, b, c, block.word[1], 12, 0xe8c7b756); - FF(c, d, a, b, block.word[2], 17, 0x242070db); - FF(b, c, d, a, block.word[3], 22, 0xc1bdceee); - FF(a, b, c, d, block.word[4], 7, 0xf57c0faf); - FF(d, a, b, c, block.word[5], 12, 0x4787c62a); - FF(c, d, a, b, block.word[6], 17, 0xa8304613); - FF(b, c, d, a, block.word[7], 22, 0xfd469501); - FF(a, b, c, d, block.word[8], 7, 0x698098d8); - FF(d, a, b, c, block.word[9], 12, 0x8b44f7af); - FF(c, d, a, b, block.word[10], 17, 0xffff5bb1); - FF(b, c, d, a, block.word[11], 22, 0x895cd7be); - FF(a, b, c, d, block.word[12], 7, 0x6b901122); - FF(d, a, b, c, block.word[13], 12, 0xfd987193); - FF(c, d, a, b, block.word[14], 17, 0xa679438e); - FF(b, c, d, a, block.word[15], 22, 0x49b40821); - - // Round 2 - GG(a, b, c, d, block.word[1], 5, 0xf61e2562); - GG(d, a, b, c, block.word[6], 9, 0xc040b340); - GG(c, d, a, b, block.word[11], 14, 0x265e5a51); - GG(b, c, d, a, block.word[0], 20, 0xe9b6c7aa); - GG(a, b, c, d, block.word[5], 5, 0xd62f105d); - GG(d, a, b, c, block.word[10], 9, 0x2441453); - GG(c, d, a, b, block.word[15], 14, 0xd8a1e681); - GG(b, c, d, a, block.word[4], 20, 0xe7d3fbc8); - GG(a, b, c, d, block.word[9], 5, 0x21e1cde6); - GG(d, a, b, c, block.word[14], 9, 0xc33707d6); - GG(c, d, a, b, block.word[3], 14, 0xf4d50d87); - GG(b, c, d, a, block.word[8], 20, 0x455a14ed); - GG(a, b, c, d, block.word[13], 5, 0xa9e3e905); - GG(d, a, b, c, block.word[2], 9, 0xfcefa3f8); - GG(c, d, a, b, block.word[7], 14, 0x676f02d9); - GG(b, c, d, a, block.word[12], 20, 0x8d2a4c8a); - - // Round 3 - HH(a, b, c, d, block.word[5], 4, 0xfffa3942); - HH(d, a, b, c, block.word[8], 11, 0x8771f681); - HH(c, d, a, b, block.word[11], 16, 0x6d9d6122); - HH(b, c, d, a, block.word[14], 23, 0xfde5380c); - HH(a, b, c, d, block.word[1], 4, 0xa4beea44); - HH(d, a, b, c, block.word[4], 11, 0x4bdecfa9); - HH(c, d, a, b, block.word[7], 16, 0xf6bb4b60); - HH(b, c, d, a, block.word[10], 23, 0xbebfbc70); - HH(a, b, c, d, block.word[13], 4, 0x289b7ec6); - HH(d, a, b, c, block.word[0], 11, 0xeaa127fa); - HH(c, d, a, b, block.word[3], 16, 0xd4ef3085); - HH(b, c, d, a, block.word[6], 23, 0x4881d05); - HH(a, b, c, d, block.word[9], 4, 0xd9d4d039); - HH(d, a, b, c, block.word[12], 11, 0xe6db99e5); - HH(c, d, a, b, block.word[15], 16, 0x1fa27cf8); - HH(b, c, d, a, block.word[2], 23, 0xc4ac5665); - - // Round 4 - II(a, b, c, d, block.word[0], 6, 0xf4292244); - II(d, a, b, c, block.word[7], 10, 0x432aff97); - II(c, d, a, b, block.word[14], 15, 0xab9423a7); - II(b, c, d, a, block.word[5], 21, 0xfc93a039); - II(a, b, c, d, block.word[12], 6, 0x655b59c3); - II(d, a, b, c, block.word[3], 10, 0x8f0ccc92); - II(c, d, a, b, block.word[10], 15, 0xffeff47d); - II(b, c, d, a, block.word[1], 21, 0x85845dd1); - II(a, b, c, d, block.word[8], 6, 0x6fa87e4f); - II(d, a, b, c, block.word[15], 10, 0xfe2ce6e0); - II(c, d, a, b, block.word[6], 15, 0xa3014314); - II(b, c, d, a, block.word[13], 21, 0x4e0811a1); - II(a, b, c, d, block.word[4], 6, 0xf7537e82); - II(d, a, b, c, block.word[11], 10, 0xbd3af235); - II(c, d, a, b, block.word[2], 15, 0x2ad7d2bb); - II(b, c, d, a, block.word[9], 21, 0xeb86d391); - - // Add to hash value - h[0] += a; - h[1] += b; - h[2] += c; - h[3] += d; - } - - - void DoMd5Little(const uint8_t* poolIn, const uint64_t inputBytesNum, uint8_t hash[16]) { - struct Md5Block block; - - /// initialize hash value - uint32_t h[4]; - h[0] = 0x67452301; - h[1] = 0xEFCDAB89; - h[2] = 0x98BADCFE; - h[3] = 0x10325476; - - /// padding and divide input data into blocks - uint64_t fullLen = (inputBytesNum >> 6) << 6; /// complete blocked length - uint64_t partLen = inputBytesNum & 0x3F; /// length remained - - uint32_t i; - for (i = 0; i < fullLen; i += 64) { - /// copy input data into block - memcpy(block.word, &(poolIn[i]), 64); - - /// calculate Md5 - CalMd5(block, h); - } - - - if (partLen > 55) /// append two more blocks - { - /// copy input data into block and pad - memcpy(block.word, &(poolIn[i]), partLen); - memcpy(((uint8_t*)&(block.word[partLen >> 2])) + (partLen & 0x3), gPadding, (64 - partLen)); - - /// calculate Md5 - CalMd5(block, h); - - /// set rest byte to 0x0 - memset(block.word, 0x0, 64); - } else /// append one more block - { - /// copy input data into block and pad - memcpy(block.word, &(poolIn[i]), partLen); - memcpy(((uint8_t*)&(block.word[partLen >> 2])) + (partLen & 0x3), gPadding, (64 - partLen)); - } - - /// append length (bits) - uint64_t bitsNum = inputBytesNum * 8; - memcpy(&(block.word[14]), &bitsNum, 8); - - /// calculate Md5 - CalMd5(block, h); - - /// clear sensitive information - memset(block.word, 0, 64); - - /// fill hash value - memcpy(&(hash[0]), &(h[0]), 16); - } /// DoMd5Little - - void DoMd5Big(const uint8_t* poolIn, const uint64_t inputBytesNum, uint8_t hash[16]) { - struct Md5Block block; - uint8_t tempBlock[64]; - - /// initialize hash value - uint32_t h[4]; - h[0] = 0x67452301; - h[1] = 0xEFCDAB89; - h[2] = 0x98BADCFE; - h[3] = 0x10325476; - - /// padding and divide input data into blocks - uint64_t fullLen = (inputBytesNum >> 6) << 6; - uint64_t partLen = inputBytesNum & 0x3F; - - uint32_t i; - for (i = 0; i < fullLen; i += 64) { - /// copy input data into block, in little endian - CopyBytesToBlock(&(poolIn[i]), block); - - /// calculate Md5 - CalMd5(block, h); - } - - /// append two more blocks - if (partLen > 55) { - /// put input data into a temporary block - memcpy(tempBlock, &(poolIn[i]), partLen); - memcpy(&(tempBlock[partLen]), gPadding, (64 - partLen)); - - /// copy temporary data into block, in little endian - CopyBytesToBlock(tempBlock, block); - - /// calculate Md5 - CalMd5(block, h); - - memset(tempBlock, 0x0, 64); - } - /// append one more block - else { - memcpy(tempBlock, &(poolIn[i]), partLen); - memcpy(&(tempBlock[partLen]), gPadding, (64 - partLen)); - } - /// append length (bits) - uint64_t bitsNum = inputBytesNum * 8; - memcpy(&(tempBlock[56]), &bitsNum, 8); - - /// copy temporary data into block, in little endian - CopyBytesToBlock(tempBlock, block); - - /// calculate Md5 - CalMd5(block, h); - - /// clear sensitive information - memset(block.word, 0, 64); - memset(tempBlock, 0, 64); - - /// fill hash value - memcpy(&(hash[0]), &(h[0]), 16); - } /// DoMd5Big - - void DoMd5(const uint8_t* poolIn, const uint64_t inputBytesNum, uint8_t md5[16]) { - /// detect big or little endian - union { - uint32_t a; - uint8_t b; - } symbol; - - symbol.a = 1; - - /// for little endian - if (symbol.b == 1) { - DoMd5Little(poolIn, inputBytesNum, md5); - } - /// for big endian - else { - DoMd5Big(poolIn, inputBytesNum, md5); - } - } /// DoMd5 - -/* - * define the rotate left (circular left shift) operation - */ -#define rotl(v, b) (((v) << (b)) | ((v) >> (32 - (b)))) - -/* - * Define the basic SHA-1 functions F1 ~ F4. Note that the exclusive-OR - * operation (^) in F1 and F3 may be replaced by a bitwise OR operation - * (|), which produce identical results. - * - * F1 is used in ROUND 0~19, F2 is used in ROUND 20~39 - * F3 is used in ROUND 40~59, F4 is used in ROUND 60~79 - */ -#define F1(B, C, D) (((B) & (C)) ^ (~(B) & (D))) -#define F2(B, C, D) ((B) ^ (C) ^ (D)) -#define F3(B, C, D) (((B) & (C)) ^ ((B) & (D)) ^ ((C) & (D))) -#define F4(B, C, D) ((B) ^ (C) ^ (D)) - -/* - * Use different K in different ROUND - */ -#define K00_19 0x5A827999 -#define K20_39 0x6ED9EBA1 -#define K40_59 0x8F1BBCDC -#define K60_79 0xCA62C1D6 - -/* - * Another implementation of the ROUND transformation: - * (here the T is a temp variable) - * For t=0 to 79: - * { - * T=rotl(A,5)+Func(B,C,D)+K+W[t]+E; - * E=D; D=C; C=rotl(B,30); B=A; A=T; - * } - */ -#define ROUND(t, A, B, C, D, E, Func, K) \ - E += rotl(A, 5) + Func(B, C, D) + W[t] + K; \ - B = rotl(B, 30); - -#define ROUND5(t, Func, K) \ - ROUND(t, A, B, C, D, E, Func, K); \ - ROUND(t + 1, E, A, B, C, D, Func, K); \ - ROUND(t + 2, D, E, A, B, C, Func, K); \ - ROUND(t + 3, C, D, E, A, B, Func, K); \ - ROUND(t + 4, B, C, D, E, A, Func, K) - -#define ROUND20(t, Func, K) \ - ROUND5(t, Func, K); \ - ROUND5(t + 5, Func, K); \ - ROUND5(t + 10, Func, K); \ - ROUND5(t + 15, Func, K) - - /* - * Define constant of the initial vector - */ - const uint32_t SHA1::IV[SHA1_DIGEST_WORDS] = {0x67452301, 0xEFCDAB89, 0x98BADCFE, 0x10325476, 0xC3D2E1F0}; - - /* - * the message must be the big-endian32 (or left-most word) - * before calling the transform() function. - */ - const static uint32_t iii = 1; - const static bool littleEndian = *(uint8_t*)&iii != 0; - - inline uint32_t littleEndianToBig(uint32_t d) { - uint8_t* data = (uint8_t*)&d; - return data[0] << 24 | data[1] << 16 | data[2] << 8 | data[3]; - } - - inline void make_big_endian32(uint32_t* data, unsigned n) { - if (!littleEndian) { - return; - } - for (; n > 0; ++data, --n) { - *data = littleEndianToBig(*data); - } - } - - inline size_t min(size_t a, size_t b) { - return a < b ? a : b; - } - - void SHA1::transform() { - uint32_t W[80]; - memcpy(W, M, SHA1_INPUT_BYTES); - memset((uint8_t*)W + SHA1_INPUT_BYTES, 0, sizeof(W) - SHA1_INPUT_BYTES); - for (unsigned t = 16; t < 80; t++) { - W[t] = rotl(W[t - 16] ^ W[t - 14] ^ W[t - 8] ^ W[t - 3], 1); - } - - uint32_t A = H[0]; - uint32_t B = H[1]; - uint32_t C = H[2]; - uint32_t D = H[3]; - uint32_t E = H[4]; - - ROUND20(0, F1, K00_19); - ROUND20(20, F2, K20_39); - ROUND20(40, F3, K40_59); - ROUND20(60, F4, K60_79); - - H[0] += A; - H[1] += B; - H[2] += C; - H[3] += D; - H[4] += E; - } - - void SHA1::add(const uint8_t* data, size_t data_len) { - unsigned mlen = (unsigned)((bits >> 3) % SHA1_INPUT_BYTES); - bits += (uint64_t)data_len << 3; - unsigned use = (unsigned)min((size_t)(SHA1_INPUT_BYTES - mlen), data_len); - memcpy(M + mlen, data, use); - mlen += use; - - while (mlen == SHA1_INPUT_BYTES) { - data_len -= use; - data += use; - make_big_endian32((uint32_t*)M, SHA1_INPUT_WORDS); - transform(); - use = (unsigned)min((size_t)SHA1_INPUT_BYTES, data_len); - memcpy(M, data, use); - mlen = use; - } - } - - uint8_t* SHA1::result() { - unsigned mlen = (unsigned)((bits >> 3) % SHA1_INPUT_BYTES), padding = SHA1_INPUT_BYTES - mlen; - M[mlen++] = 0x80; - if (padding > BIT_COUNT_BYTES) { - memset(M + mlen, 0x00, padding - BIT_COUNT_BYTES); - make_big_endian32((uint32_t*)M, SHA1_INPUT_WORDS - BIT_COUNT_WORDS); - } else { - memset(M + mlen, 0x00, SHA1_INPUT_BYTES - mlen); - make_big_endian32((uint32_t*)M, SHA1_INPUT_WORDS); - transform(); - memset(M, 0x00, SHA1_INPUT_BYTES - BIT_COUNT_BYTES); - } - - uint64_t temp = littleEndian ? bits << 32 | bits >> 32 : bits; - memcpy(M + SHA1_INPUT_BYTES - BIT_COUNT_BYTES, &temp, BIT_COUNT_BYTES); - transform(); - make_big_endian32(H, SHA1_DIGEST_WORDS); - return (uint8_t*)H; - } - - template - inline void axor(T* p1, const T* p2, size_t len) { - for (; len != 0; --len) - *p1++ ^= *p2++; - } - - HMAC::HMAC(const uint8_t* key, size_t lkey) { - init(key, lkey); - } - - void HMAC::init(const uint8_t* key, size_t lkey) { - in.init(); - out.init(); - - uint8_t ipad[SHA1_INPUT_BYTES]; - uint8_t opad[SHA1_INPUT_BYTES]; - memset(ipad, 0x36, sizeof(ipad)); - memset(opad, 0x5c, sizeof(opad)); - - if (lkey <= SHA1_INPUT_BYTES) { - axor(ipad, key, lkey); - axor(opad, key, lkey); - } else { - SHA1 tmp; - tmp.add(key, lkey); - const uint8_t* key2 = tmp.result(); - axor(ipad, key2, SHA1_DIGEST_BYTES); - axor(opad, key2, SHA1_DIGEST_BYTES); - } - - in.add((uint8_t*)ipad, sizeof(ipad)); - out.add((uint8_t*)opad, sizeof(opad)); - } - -} // namespace sdk -} // namespace logtail diff --git a/core/sdk/Common.h b/core/sdk/Common.h deleted file mode 100644 index 56ce512560..0000000000 --- a/core/sdk/Common.h +++ /dev/null @@ -1,323 +0,0 @@ -/* - * Copyright 2022 iLogtail Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace logtail { -namespace sdk { - - const int64_t kInvalidHashKeySeqID = 0; - const int64_t kFirstHashKeySeqID = 1; - - const uint32_t LOG_REQUEST_TIMEOUT = 20; - const uint32_t MD5_BYTES = 16; -#define DATE_FORMAT_RFC822 "%a, %d %b %Y %H:%M:%S GMT" ///< RFC822 date formate, GMT time. - - extern const char* const LOG_HEADSIGNATURE_PREFIX; ///< ""; - - extern const char* const LOGE_REQUEST_ERROR; //= "RequestError"; - extern const char* const LOGE_INVALID_HOST; //= "InvalidHost" - extern const char* const LOGE_UNKNOWN_ERROR; //= "UnknownError"; - extern const char* const LOGE_NOT_IMPLEMENTED; //= "NotImplemented"; - extern const char* const LOGE_SERVER_BUSY; //= "ServerBusy"; - extern const char* const LOGE_INTERNAL_SERVER_ERROR; //= "InternalServerError"; - extern const char* const LOGE_RESPONSE_SIG_ERROR; //= "ResponseSignatureError"; - extern const char* const LOGE_PARAMETER_INVALID; //= "ParameterInvalid"; - extern const char* const LOGE_MISSING_PARAMETER; //= "MissingParameter"; - extern const char* const LOGE_INVALID_METHOD; //= "InvalidMethod"; - extern const char* const LOGE_INVALID_CONTENTTYPE; //= "InvalidContentType"; - extern const char* const LOGE_INVALID_CONTENTLENGTH; //= "InvalidContentLength"; - extern const char* const LOGE_BAD_RESPONSE; //= "BadResponse"; - extern const char* const LOGE_UNAUTHORIZED; //= "Unauthorized"; - extern const char* const LOGE_QUOTA_EXCEED; //= "ExceedQuota"; - extern const char* const LOGE_REQUEST_TIMEOUT; //= "RequestTimeout"; - extern const char* const LOGE_CLIENT_OPERATION_TIMEOUT; //= "ClientOpertaionTimeout"; - extern const char* const LOGE_CLIENT_NETWORK_ERROR; //= "ClientNetworkError"; - extern const char* const LOGE_USER_NOT_EXIST; //= "UserNotExist"; - extern const char* const LOGE_CATEGORY_NOT_EXIST; //= "CategoryNotExist"; - extern const char* const LOGE_TOPIC_NOT_EXIST; //= "TopicNotExist"; - extern const char* const LOGE_POST_BODY_INVALID; //= "PostBodyInvalid"; - extern const char* const LOGE_INVALID_HOST; //= "InvalidHost"; - extern const char* const LOGE_INVALID_APIVERSION; //="InvalidAPIVersion"; - extern const char* const LOGE_PROJECT_NOT_EXIST; //="ProjectNotExist"; - extern const char* const LOGE_MACHINEGROUP_NOT_EXIST; //="MachineGroupNotExist"; - extern const char* const LOGE_MACHINEGROUP_ALREADY_EXIST; //="MachineGroupAlreadyExist"; - extern const char* const LOGE_CONFIG_NOT_EXIST; //="ConfigNotExist"; - extern const char* const LOGE_CONFIG_ALREADY_EXIST; //="ConfigAlreadyExist"; - extern const char* const LOGE_LOGSTORE_NOT_EXIST; //="LogStoreNotExist"; - extern const char* const LOGE_INVALID_ACCESSKEYID; //="InvalidAccessKeyId"; - extern const char* const LOGE_SIGNATURE_NOT_MATCH; //="SignatureNotMatch"; - extern const char* const LOGE_PROJECT_FORBIDDEN; //="ProjectForbidden"; - extern const char* const LOGE_WRITE_QUOTA_EXCEED; //="WriteQuotaExceed"; - extern const char* const LOGE_READ_QUOTA_EXCEED; //="ReadQuotaExceed"; - extern const char* const LOGE_REQUEST_TIME_EXPIRED; //="RequestTimeExpired"; - extern const char* const LOGE_INVALID_REQUEST_TIME; //="InvalidRequestTime"; - extern const char* const LOGE_POST_BODY_TOO_LARGE; //="PostBodyTooLarge"; - extern const char* const LOGE_INVALID_TIME_RANGE; //="InvalidTimeRange"; - extern const char* const LOGE_INVALID_REVERSE; //="InvalidReverse"; - extern const char* const LOGE_LOGSTORE_WITHOUT_SHARD; //="LogStoreWithoutShard"; - extern const char* const LOGE_INVALID_SEQUENCE_ID; //="InvalidSequenceId"; - - extern const char* const LOGSTORES; //= "/logstores" - extern const char* const METRICSTORES; //= "/prometheus" - extern const char* const SHARDS; //= "/shards" - extern const char* const INDEX; //= "/index" - extern const char* const CONFIGS; //= "/configs" - extern const char* const MACHINES; //= "/machines" - extern const char* const MACHINEGROUPS; //= "/machinegroups" - extern const char* const ACLS; //= "/acls" - extern const char* const CONFIGSERVERAGENT; //= "/Agent" - - extern const char* const HTTP_GET; //= "GET"; - extern const char* const HTTP_POST; //= "POST"; - extern const char* const HTTP_PUT; //= "PUT"; - extern const char* const HTTP_DELETE; //= "DELETE"; - - extern const char* const HOST; //= "Host"; - extern const char* const DATE; //= "Date"; - extern const char* const USER_AGENT; //= "User-Agent"; - extern const char* const LOG_HEADER_PREFIX; //= "x-log-"; - extern const char* const LOG_OLD_HEADER_PREFIX; //= "x-sls-"; - extern const char* const X_LOG_KEYPROVIDER; // = "x-log-keyprovider"; - extern const char* const X_LOG_APIVERSION; // = "x-log-apiversion"; - extern const char* const X_LOG_COMPRESSTYPE; // = "x-log-compresstype"; - extern const char* const X_LOG_BODYRAWSIZE; // = "x-log-bodyrawsize"; - extern const char* const X_LOG_SIGNATUREMETHOD; // = "x-log-signaturemethod"; - extern const char* const X_ACS_SECURITY_TOKEN; // = "x-acs-security-token"; - extern const char* const X_LOG_CURSOR; // = "cursor"; - extern const char* const X_LOG_REQUEST_ID; // = "x-log-requestid"; - extern const char* const X_LOG_MODE; // = "x-log-mode"; - - extern const char* const X_LOG_PROGRESS; // = "x-log-progress"; - extern const char* const X_LOG_COUNT; // = "x-log-count"; - extern const char* const X_LOG_HOSTIP; // = "x-log-hostip" - - extern const char* const HTTP_ACCEPT; // = "accept"; - extern const char* const DEFLATE; //= "deflate"; - extern const char* const HMAC_SHA1; //= "hmac-sha1"; - extern const char* const CONTENT_LENGTH; //= "Content-Length"; - extern const char* const CONTENT_TYPE; //= "Content-Type"; - extern const char* const CONTENT_MD5; //= "Content-MD5"; - extern const char* const AUTHORIZATION; //= "Authorization"; - extern const char* const SIGNATURE; //= "Signature"; - extern const char* const ACCEPT_ENCODING; // = "Accept-Encoding"; - extern const char* const ENCONDING_GZIP; // = "gzip"; - extern const char* const TYPE_LOG_PROTOBUF; //="application/x-protobuf"; - extern const char* const TYPE_LOG_JSON; //="application/json"; - extern const char* const LOG_MODE_BATCH_GROUP; //="batch_group"; - extern const char* const LOGITEM_TIME_STAMP_LABEL; //="__time__"; - extern const char* const LOGITEM_SOURCE_LABEL; //="__source__"; - extern const char* const LOG_API_VERSION; // = "0.6.0"; - extern const char* const LOGTAIL_USER_AGENT; // = "ali-sls-logtail"; - extern const char* const MD5_SHA1_SALT_KEYPROVIDER; // = "md5-shal-salt"; - extern const char* const LOG_TYPE_CURSOR; // = "cursor"; - extern const char* const LOG_TYPE; // = "type"; - extern const char* const LOGE_NOT_SUPPORTED_ACCEPT_CONTENT_TYPE; - extern const char* const LOGE_NOT_SUPPORTED_ACCEPT_ENCODING; - extern const char* const LOGE_SHARD_NOT_EXIST; - extern const char* const LOGE_INVALID_CURSOR; - extern const char* const LOGE_SHARD_WRITE_QUOTA_EXCEED; - extern const char* const LOGE_SHARD_READ_QUOTA_EXCEED; - extern const char* const LOG_LZ4; //= "lz4"; - extern const char* const LOG_DEFLATE; //= "deflate"; - extern const char* const LOG_ZSTD; //= "zstd"; - - extern const char* const LOG_ERROR_CODE; //= "errorCode"; - extern const char* const LOG_ERROR_MESSAGE; //= "errorMessage"; - - extern const char* const LOG_SHARD_STATUS_READWRITE; // "readwrite"; - extern const char* const LOG_SHARD_STATUS_READONLY; // "readonly"; - - bool caseInsensitiveComp(const char lhs, const char rhs); - - bool compareHeader(const std::string& lhs, const std::string& rhs); - - /** - * HTTP message structure includes three parts: http status code, http header, and http content. - */ - struct HttpMessage { - int32_t statusCode = 0; ///< Http status code - std::map - header; ///< Only contains the header lines which have key:value pair - std::string content; ///< Http content - /** Constructor with no parameter. - * @param void None. - * @return The objcect pointer. - */ - HttpMessage() : header(compareHeader) {} - /** Constructor with header and content. - * @param para_header A map structure which contains the key:value pair of http header lines. - Those header lines which do not contains key:value pair are not included. - * @param para_content A string which contains the content of http request. - * @return The objcect pointer. - */ - HttpMessage(const std::map& para_header, const std::string& para_content) - : header(para_header.begin(), para_header.end(), compareHeader), content(para_content) {} - /** Constructor with status code, header and content. - * @param para_statusCode Http status code. - * @param para_header A map structure which contains the key:value pair of http header lines. - Those header lines which do not contains key:value pair are not included. - * @param para_content A string which contains the http content of http content. - * @return The objcect pointer. - */ - HttpMessage(const int32_t para_statusCode, - const std::map& para_header, - const std::string& para_content) - : statusCode(para_statusCode), - header(para_header.begin(), para_header.end(), compareHeader), - content(para_content) {} - - bool IsLogServiceResponse() const; - }; - - /* - * Responses - */ - struct Response { - int32_t statusCode = 0; ///< Http status code - std::string requestId; - - virtual ~Response() {} - }; - - struct PostLogStoreLogsResponse : public Response { - int32_t bodyBytes; - }; - - struct GetRealIpResponse : public Response { - std::string realIp; - }; - -#define SHA1_INPUT_WORDS 16 -#define SHA1_DIGEST_WORDS 5 -#define SHA1_INPUT_BYTES (SHA1_INPUT_WORDS * sizeof(uint32_t)) -#define SHA1_DIGEST_BYTES (SHA1_DIGEST_WORDS * sizeof(uint32_t)) - -#define BIT_COUNT_WORDS 2 -#define BIT_COUNT_BYTES (BIT_COUNT_WORDS * sizeof(uint32_t)) - - /** Calculate Md5 for a byte stream, - * result is stored in md5[16] - * - * @param poolIn Input data - * @param inputBytesNum Length of input data - * @param md5[16] A 128-bit pool for storing md5 - */ - void DoMd5(const uint8_t* poolIn, const uint64_t inputBytesNum, uint8_t md5[16]); - void Base64Encoding(std::istream&, - std::ostream&, - char makeupChar = '=', - const char* alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"); - - std::string CalcMD5(const std::string& message); - std::string CalcSHA1(const std::string& message, const std::string& key); - std::string Base64Enconde(const std::string& message); - - std::string UrlEncode(const std::string& str); - std::string UrlDecode(const std::string& str); - - std::string GetDateString(const std::string& dateFormat); - std::string GetDateString(); - time_t DecodeDateString(const std::string dateString, const std::string& dateFormat = DATE_FORMAT_RFC822); - - bool StartWith(const std::string& input, const std::string& pattern); - - void GetQueryString(const std::map& parameterList, std::string& queryString); - - std::string GetUrlSignature(const std::string& httpMethod, - const std::string& operationType, - std::map& httpHeader, - const std::map& parameterList, - const std::string& content, - const std::string& signKey); - - class SHA1 { - public: - SHA1() : bits(0) { memcpy(H, IV, sizeof(H)); } - SHA1(const SHA1& s) { - bits = s.bits; - memcpy(H, s.H, sizeof(H)); - memcpy(M, s.M, sizeof(M)); - } - void init() { - bits = 0; - memcpy(H, IV, sizeof(H)); - } - void add(const uint8_t* data, size_t len); - uint8_t* result(); - - private: - uint64_t bits; - uint32_t H[SHA1_DIGEST_WORDS]; - uint8_t M[SHA1_INPUT_BYTES]; - - static const uint32_t IV[SHA1_DIGEST_WORDS]; - void transform(); - }; - - class HMAC { - public: - HMAC(const uint8_t* key, size_t lkey); - HMAC(const HMAC& hm) : in(hm.in), out(hm.out) {} - - void init(const uint8_t* key, size_t lkey); - - void add(const uint8_t* data, size_t len) { in.add(data, len); } - - uint8_t* result() { - out.add(in.result(), SHA1_DIGEST_BYTES); - return out.result(); - } - - private: - SHA1 in, out; - }; - - class SpinLock { - std::atomic_flag locked = ATOMIC_FLAG_INIT; - - SpinLock(const SpinLock&) = delete; - SpinLock& operator=(const SpinLock&) = delete; - - public: - SpinLock() {} - - void lock() { - while (locked.test_and_set(std::memory_order_acquire)) { - ; - } - } - - void unlock() { locked.clear(std::memory_order_release); } - }; - - using ScopedSpinLock = std::lock_guard; - -} // namespace sdk -} // namespace logtail diff --git a/core/sdk/CurlImp.cpp b/core/sdk/CurlImp.cpp deleted file mode 100644 index 9cea4bb9d6..0000000000 --- a/core/sdk/CurlImp.cpp +++ /dev/null @@ -1,192 +0,0 @@ -// Copyright 2022 iLogtail Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "CurlImp.h" - -#include - -#include "DNSCache.h" -#include "Exception.h" -#include "app_config/AppConfig.h" -#include "common/http/Curl.h" - -using namespace std; - -namespace logtail { -namespace sdk { - - static CURLcode globalInitCode = curl_global_init(CURL_GLOBAL_ALL); - - // callback function to store the response - static size_t data_write_callback(char* buffer, size_t size, size_t nmemb, string* write_buf) { - unsigned long sizes = size * nmemb; - - if (buffer == NULL) { - return 0; - } - - write_buf->append(buffer, sizes); - return sizes; - } - - static size_t header_write_callback(char* buffer, - size_t size, - size_t nmemb, - map* write_buf) { - unsigned long sizes = size * nmemb; - - if (buffer == NULL) { - return 0; - } - unsigned long colonIndex; - for (colonIndex = 1; colonIndex < sizes - 2; colonIndex++) { - if (buffer[colonIndex] == ':') - break; - } - if (colonIndex < sizes - 2) { - unsigned long leftSpaceNum, rightSpaceNum; - for (leftSpaceNum = 0; leftSpaceNum < colonIndex - 1; leftSpaceNum++) { - if (buffer[colonIndex - leftSpaceNum - 1] != ' ') - break; - } - for (rightSpaceNum = 0; rightSpaceNum < sizes - colonIndex - 1 - 2; rightSpaceNum++) { - if (buffer[colonIndex + rightSpaceNum + 1] != ' ') - break; - } - (*write_buf)[string(buffer, 0, colonIndex - leftSpaceNum)] - = string(buffer, colonIndex + 1 + rightSpaceNum, sizes - colonIndex - 1 - 2 - rightSpaceNum); - } - return sizes; - } - - CURL* PackCurlRequest(const std::string& httpMethod, - const std::string& host, - const int32_t port, - const std::string& url, - const std::string& queryString, - const std::map& header, - const std::string& body, - const int32_t timeout, - HttpMessage& httpMessage, - const std::string& intf, - const bool httpsFlag, - curl_slist*& headers) { - static DnsCache* dnsCache = DnsCache::GetInstance(); - - CURL* curl = curl_easy_init(); - if (curl == NULL) - return NULL; - - string totalUrl = httpsFlag ? "https://" : "http://"; - std::string hostIP; - if (AppConfig::GetInstance()->IsHostIPReplacePolicyEnabled() && dnsCache->GetIPFromDnsCache(host, hostIP)) { - totalUrl.append(hostIP); - } else { - totalUrl.append(host); - } - totalUrl.append(url); - if (!queryString.empty()) { - totalUrl.append("?").append(queryString); - } - curl_easy_setopt(curl, CURLOPT_URL, totalUrl.c_str()); - for (const auto& iter : header) { - headers = curl_slist_append(headers, (iter.first + ":" + iter.second).c_str()); - } - if (headers != NULL) { - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - } - curl_easy_setopt(curl, CURLOPT_PORT, port); - curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, httpMethod.c_str()); - if (!body.empty()) { - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, (void*)body.c_str()); - curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, body.size()); - } - - curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); - curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1); - curl_easy_setopt(curl, CURLOPT_TCP_NODELAY, 1); - curl_easy_setopt(curl, CURLOPT_NETRC, CURL_NETRC_IGNORED); - - if (httpsFlag) { - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L); - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L); - } - curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout); - if (!intf.empty()) { - curl_easy_setopt(curl, CURLOPT_INTERFACE, intf.c_str()); - } - // curl_easy_setopt(curl, CURLOPT_VERBOSE, 1); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &(httpMessage.content)); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, data_write_callback); - curl_easy_setopt(curl, CURLOPT_HEADERDATA, &(httpMessage.header)); - curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_write_callback); - return curl; - } - - void CurlClient::Send(const std::string& httpMethod, - const std::string& host, - const int32_t port, - const std::string& url, - const std::string& queryString, - const std::map& header, - const std::string& body, - const int32_t timeout, - HttpMessage& httpMessage, - const std::string& intf, - const bool httpsFlag) { - curl_slist* headers = NULL; - CURL* curl = PackCurlRequest( - httpMethod, host, port, url, queryString, header, body, timeout, httpMessage, intf, httpsFlag, headers); - if (curl == NULL) { - throw LOGException(LOGE_UNKNOWN_ERROR, "Init curl instance error."); - } - - CURLcode res = curl_easy_perform(curl); - if (headers != NULL) { - curl_slist_free_all(headers); - } - - switch (res) { - case CURLE_OK: - break; - case CURLE_OPERATION_TIMEDOUT: - curl_easy_cleanup(curl); - throw LOGException(LOGE_CLIENT_OPERATION_TIMEOUT, "Request operation timeout."); - break; - case CURLE_COULDNT_CONNECT: - curl_easy_cleanup(curl); - throw LOGException(LOGE_REQUEST_TIMEOUT, "Can not connect to server."); - break; - default: - curl_easy_cleanup(curl); - throw LOGException(LOGE_REQUEST_ERROR, - string("Request operation failed, curl error code : ") + curl_easy_strerror(res)); - break; - } - - long http_code = 0; - if ((res = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code)) != CURLE_OK) { - curl_easy_cleanup(curl); - throw LOGException(LOGE_UNKNOWN_ERROR, - string("Get curl response code error, curl error code : ") + curl_easy_strerror(res)); - } - httpMessage.statusCode = (int32_t)http_code; - curl_easy_cleanup(curl); - if (!httpMessage.IsLogServiceResponse()) { - throw LOGException(LOGE_REQUEST_ERROR, "Get invalid response"); - } - } - -} // namespace sdk -} // namespace logtail diff --git a/core/sdk/CurlImp.h b/core/sdk/CurlImp.h deleted file mode 100644 index cf020e6669..0000000000 --- a/core/sdk/CurlImp.h +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright 2022 iLogtail Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#include -#include - -#include "Common.h" - -namespace logtail { -namespace sdk { - - class CurlClient { - public: - void Send(const std::string& httpMethod, - const std::string& host, - const int32_t port, - const std::string& url, - const std::string& queryString, - const std::map& header, - const std::string& body, - const int32_t timeout, - HttpMessage& httpMessage, - const std::string& intf, - const bool httpsFlag); - }; - -} // namespace sdk -} // namespace logtail diff --git a/core/sdk/Result.cpp b/core/sdk/Result.cpp deleted file mode 100644 index 4caf9fc4c5..0000000000 --- a/core/sdk/Result.cpp +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright 2022 iLogtail Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "Result.h" -#include "Exception.h" - -namespace logtail { - -namespace sdk { - - using namespace std; - - /************************ common method ***********************/ - /************************ json method *************************/ - void ExtractJsonResult(const string& response, rapidjson::Document& document) { - document.Parse(response.c_str()); - if (document.HasParseError()) { - throw JsonException("ParseException", "Fail to parse from json string"); - } - } - - void JsonMemberCheck(const rapidjson::Value& value, const char* name) { - if (!value.IsObject()) { - throw JsonException("InvalidObjectException", "response is not valid JSON object"); - } - if (!value.HasMember(name)) { - throw JsonException("NoMemberException", string("Member ") + name + " does not exist"); - } - } - - void ExtractJsonResult(const rapidjson::Value& value, const char* name, int32_t& number) { - JsonMemberCheck(value, name); - if (value[name].IsInt()) { - number = value[name].GetInt(); - } else { - throw JsonException("ValueTypeException", string("Member ") + name + " is not int type"); - } - } - - void ExtractJsonResult(const rapidjson::Value& value, const char* name, uint32_t& number) { - JsonMemberCheck(value, name); - if (value[name].IsUint()) { - number = value[name].GetUint(); - } else { - throw JsonException("ValueTypeException", string("Member ") + name + " is not uint type"); - } - } - - void ExtractJsonResult(const rapidjson::Value& value, const char* name, int64_t& number) { - JsonMemberCheck(value, name); - if (value[name].IsInt64()) { - number = value[name].GetInt64(); - } else { - throw JsonException("ValueTypeException", string("Member ") + name + " is not int type"); - } - } - - void ExtractJsonResult(const rapidjson::Value& value, const char* name, uint64_t& number) { - JsonMemberCheck(value, name); - if (value[name].IsUint64()) { - number = value[name].GetUint64(); - } else { - throw JsonException("ValueTypeException", string("Member ") + name + " is not uint type"); - } - } - - void ExtractJsonResult(const rapidjson::Value& value, const char* name, bool& boolean) { - JsonMemberCheck(value, name); - if (value[name].IsBool()) { - boolean = value[name].GetBool(); - } else { - throw JsonException("ValueTypeException", string("Member ") + name + " is not boolean type"); - } - } - - void ExtractJsonResult(const rapidjson::Value& value, const char* name, string& dst) { - JsonMemberCheck(value, name); - if (value[name].IsString()) { - dst = value[name].GetString(); - } else { - throw JsonException("ValueTypeException", string("Member ") + name + " is not string type"); - } - } - - const rapidjson::Value& GetJsonValue(const rapidjson::Value& value, const char* name) { - JsonMemberCheck(value, name); - if (value[name].IsObject() || value[name].IsArray()) { - return value[name]; - } else { - throw JsonException("ValueTypeException", string("Member ") + name + " is not json value type"); - } - } - - - void ErrorCheck(const string& response, const string& requestId, const int32_t httpCode) { - rapidjson::Document document; - try { - ExtractJsonResult(response, document); - - string errorCode; - ExtractJsonResult(document, LOG_ERROR_CODE, errorCode); - - string errorMessage; - ExtractJsonResult(document, LOG_ERROR_MESSAGE, errorMessage); - - throw LOGException(errorCode, errorMessage, requestId, httpCode); - } catch (JsonException& e) { - if (httpCode >= 500) { - throw LOGException(LOGE_INTERNAL_SERVER_ERROR, response, requestId, httpCode); - } else { - throw LOGException(LOGE_BAD_RESPONSE, string("Unextractable error:") + response, requestId, httpCode); - } - } - } - -} // namespace sdk -} // namespace logtail diff --git a/core/sdk/Result.h b/core/sdk/Result.h deleted file mode 100644 index d99b2fc135..0000000000 --- a/core/sdk/Result.h +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2022 iLogtail Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#include -#include -#include "Common.h" -#include "rapidjson/document.h" - -namespace logtail { -namespace sdk { - - void ExtractJsonResult(const std::string& response, rapidjson::Document& document); - - void JsonMemberCheck(const rapidjson::Value& value, const char* name); - - void ExtractJsonResult(const rapidjson::Value& value, const char* name, int32_t& number); - - void ExtractJsonResult(const rapidjson::Value& value, const char* name, uint32_t& number); - - void ExtractJsonResult(const rapidjson::Value& value, const char* name, int64_t& number); - - void ExtractJsonResult(const rapidjson::Value& value, const char* name, uint64_t& number); - - void ExtractJsonResult(const rapidjson::Value& value, const char* name, bool& boolean); - - void ExtractJsonResult(const rapidjson::Value& value, const char* name, std::string& dst); - - const rapidjson::Value& GetJsonValue(const rapidjson::Value& value, const char* name); - - void ErrorCheck(const std::string& response, const std::string& requestId, const int32_t httpCode); - -} // namespace sdk -} // namespace logtail diff --git a/core/unittest/CMakeLists.txt b/core/unittest/CMakeLists.txt index 41f1601069..c3acf4c478 100644 --- a/core/unittest/CMakeLists.txt +++ b/core/unittest/CMakeLists.txt @@ -52,7 +52,6 @@ macro(add_core_subdir) add_subdirectory(provider) add_subdirectory(queue) add_subdirectory(reader) - add_subdirectory(sdk) add_subdirectory(sender) add_subdirectory(serializer) add_subdirectory(prometheus) diff --git a/core/unittest/flusher/CMakeLists.txt b/core/unittest/flusher/CMakeLists.txt index 76ac1afd6d..5c63884d14 100644 --- a/core/unittest/flusher/CMakeLists.txt +++ b/core/unittest/flusher/CMakeLists.txt @@ -21,6 +21,9 @@ target_link_libraries(flusher_sls_unittest ${UT_BASE_TARGET}) add_executable(pack_id_manager_unittest PackIdManagerUnittest.cpp) target_link_libraries(pack_id_manager_unittest ${UT_BASE_TARGET}) +add_executable(sls_client_manager_unittest SLSClientManagerUnittest.cpp) +target_link_libraries(sls_client_manager_unittest ${UT_BASE_TARGET}) + if (ENABLE_ENTERPRISE) add_executable(enterprise_sls_client_manager_unittest EnterpriseSLSClientManagerUnittest.cpp) target_link_libraries(enterprise_sls_client_manager_unittest ${UT_BASE_TARGET}) @@ -31,6 +34,7 @@ endif () include(GoogleTest) gtest_discover_tests(flusher_sls_unittest) gtest_discover_tests(pack_id_manager_unittest) +gtest_discover_tests(sls_client_manager_unittest) if (ENABLE_ENTERPRISE) gtest_discover_tests(enterprise_sls_client_manager_unittest) gtest_discover_tests(enterprise_flusher_sls_monitor_unittest) diff --git a/core/unittest/flusher/FlusherSLSUnittest.cpp b/core/unittest/flusher/FlusherSLSUnittest.cpp index 73dc1e97e3..c8ad20ba19 100644 --- a/core/unittest/flusher/FlusherSLSUnittest.cpp +++ b/core/unittest/flusher/FlusherSLSUnittest.cpp @@ -101,16 +101,16 @@ void FlusherSLSUnittest::OnSuccessfulInit() { APSARA_TEST_EQUAL(STRING_FLAG(default_region_name), flusher->mRegion); #ifndef __ENTERPRISE__ APSARA_TEST_EQUAL("cn-hangzhou.log.aliyuncs.com", flusher->mEndpoint); -#else - APSARA_TEST_EQUAL("", flusher->mEndpoint); + APSARA_TEST_EQUAL("test_project.cn-hangzhou.log.aliyuncs.com", flusher->mCandidateHostsInfo->GetFirstHost()); #endif APSARA_TEST_EQUAL("", flusher->mAliuid); APSARA_TEST_EQUAL(sls_logs::SlsTelemetryType::SLS_TELEMETRY_TYPE_LOGS, flusher->mTelemetryType); APSARA_TEST_TRUE(flusher->mShardHashKeys.empty()); APSARA_TEST_EQUAL(static_cast(INT32_FLAG(merge_log_count_limit)), flusher->mBatcher.GetEventFlushStrategy().GetMinCnt()); - APSARA_TEST_EQUAL(static_cast(INT32_FLAG(max_send_log_group_size) / DOUBLE_FLAG(sls_serialize_size_expansion_ratio)), - flusher->mBatcher.GetEventFlushStrategy().GetMaxSizeBytes()); + APSARA_TEST_EQUAL( + static_cast(INT32_FLAG(max_send_log_group_size) / DOUBLE_FLAG(sls_serialize_size_expansion_ratio)), + flusher->mBatcher.GetEventFlushStrategy().GetMaxSizeBytes()); APSARA_TEST_EQUAL(static_cast(INT32_FLAG(batch_send_metric_size)), flusher->mBatcher.GetEventFlushStrategy().GetMinSizeBytes()); uint32_t timeout = static_cast(INT32_FLAG(batch_send_interval)) / 2; @@ -146,6 +146,9 @@ void FlusherSLSUnittest::OnSuccessfulInit() { } )"; APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); +#ifndef __ENTERPRISE__ + configJson["EndpointMode"] = "default"; +#endif flusher.reset(new FlusherSLS()); flusher->SetContext(ctx); flusher->SetMetricsRecordRef(FlusherSLS::sName, "1"); @@ -153,12 +156,14 @@ void FlusherSLSUnittest::OnSuccessfulInit() { APSARA_TEST_EQUAL("cn-hangzhou", flusher->mRegion); #ifdef __ENTERPRISE__ APSARA_TEST_EQUAL("123456789", flusher->mAliuid); + APSARA_TEST_EQUAL(EndpointMode::DEFAULT, flusher->mEndpointMode); #else - APSARA_TEST_EQUAL("cn-hangzhou.log.aliyuncs.com", flusher->mEndpoint); APSARA_TEST_EQUAL("", flusher->mAliuid); #endif + APSARA_TEST_EQUAL("cn-hangzhou.log.aliyuncs.com", flusher->mEndpoint); APSARA_TEST_EQUAL(sls_logs::SlsTelemetryType::SLS_TELEMETRY_TYPE_METRICS, flusher->mTelemetryType); APSARA_TEST_EQUAL(1U, flusher->mShardHashKeys.size()); + APSARA_TEST_EQUAL("__source__", flusher->mShardHashKeys[0]); SenderQueueManager::GetInstance()->Clear(); // invalid optional param @@ -175,6 +180,7 @@ void FlusherSLSUnittest::OnSuccessfulInit() { )"; APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); #ifdef __ENTERPRISE__ + configJson["EndpointMode"] = true; configJson["Endpoint"] = true; #else configJson["Endpoint"] = "cn-hangzhou.log.aliyuncs.com"; @@ -185,6 +191,7 @@ void FlusherSLSUnittest::OnSuccessfulInit() { APSARA_TEST_TRUE(flusher->Init(configJson, optionalGoPipeline)); APSARA_TEST_EQUAL(STRING_FLAG(default_region_name), flusher->mRegion); #ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL(EndpointMode::DEFAULT, flusher->mEndpointMode); APSARA_TEST_EQUAL("", flusher->mEndpoint); #endif APSARA_TEST_EQUAL("", flusher->mAliuid); @@ -214,6 +221,82 @@ void FlusherSLSUnittest::OnSuccessfulInit() { SenderQueueManager::GetInstance()->Clear(); #endif +#ifdef __ENTERPRISE__ + // EndpointMode && Endpoint + SLSClientManager::GetInstance()->Clear(); + // Endpoint ignored in acclerate mode + configStr = R"( + { + "Type": "flusher_sls", + "Project": "test_project", + "Logstore": "test_logstore", + "EndpointMode": "accelerate", + "Endpoint": " cn-hangzhou.log.aliyuncs.com " + } + )"; + APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); + flusher.reset(new FlusherSLS()); + flusher->SetContext(ctx); + flusher->SetMetricsRecordRef(FlusherSLS::sName, "1"); + APSARA_TEST_TRUE(flusher->Init(configJson, optionalGoPipeline)); + APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, flusher->mEndpointMode); + APSARA_TEST_TRUE( + SLSClientManager::GetInstance()->mRegionCandidateEndpointsMap["cn-hangzhou"].mRemoteEndpoints.empty()); + APSARA_TEST_EQUAL(flusher->mProject, flusher->mCandidateHostsInfo->GetProject()); + APSARA_TEST_EQUAL(flusher->mRegion, flusher->mCandidateHostsInfo->GetRegion()); + APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, flusher->mCandidateHostsInfo->GetMode()); + SenderQueueManager::GetInstance()->Clear(); + + // Endpoint should be added to region remote endpoints if not existed + configStr = R"( + { + "Type": "flusher_sls", + "Project": "test_project", + "Logstore": "test_logstore", + "EndpointMode": "unknown", + "Endpoint": " cn-hangzhou.log.aliyuncs.com " + } + )"; + APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); + flusher.reset(new FlusherSLS()); + flusher->SetContext(ctx); + flusher->SetMetricsRecordRef(FlusherSLS::sName, "1"); + APSARA_TEST_TRUE(flusher->Init(configJson, optionalGoPipeline)); + APSARA_TEST_EQUAL(EndpointMode::DEFAULT, flusher->mEndpointMode); + auto& endpoints = SLSClientManager::GetInstance()->mRegionCandidateEndpointsMap["cn-hangzhou"].mRemoteEndpoints; + APSARA_TEST_EQUAL(1U, endpoints.size()); + APSARA_TEST_EQUAL("cn-hangzhou.log.aliyuncs.com", endpoints[0]); + APSARA_TEST_EQUAL(flusher->mProject, flusher->mCandidateHostsInfo->GetProject()); + APSARA_TEST_EQUAL(flusher->mRegion, flusher->mCandidateHostsInfo->GetRegion()); + APSARA_TEST_EQUAL(EndpointMode::DEFAULT, flusher->mCandidateHostsInfo->GetMode()); + SenderQueueManager::GetInstance()->Clear(); + + // Endpoint should be ignored when region remote endpoints existed + configStr = R"( + { + "Type": "flusher_sls", + "Project": "test_project", + "Logstore": "test_logstore", + "EndpointMode": "default", + "Endpoint": " cn-hangzhou-intranet.log.aliyuncs.com " + } + )"; + APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); + flusher.reset(new FlusherSLS()); + flusher->SetContext(ctx); + flusher->SetMetricsRecordRef(FlusherSLS::sName, "1"); + APSARA_TEST_TRUE(flusher->Init(configJson, optionalGoPipeline)); + APSARA_TEST_EQUAL(EndpointMode::DEFAULT, flusher->mEndpointMode); + auto& endpoints = SLSClientManager::GetInstance()->mRegionCandidateEndpointsMap["cn-hangzhou"].mRemoteEndpoints; + APSARA_TEST_EQUAL(1U, endpoints.size()); + APSARA_TEST_EQUAL("cn-hangzhou.log.aliyuncs.com", endpoints[0]); + APSARA_TEST_EQUAL(flusher->mProject, flusher->mCandidateHostsInfo->GetProject()); + APSARA_TEST_EQUAL(flusher->mRegion, flusher->mCandidateHostsInfo->GetRegion()); + APSARA_TEST_EQUAL(EndpointMode::DEFAULT, flusher->mCandidateHostsInfo->GetMode()); + SenderQueueManager::GetInstance()->Clear(); +#endif + +#ifndef __ENTERPRISE__ // Endpoint configStr = R"( { @@ -229,12 +312,9 @@ void FlusherSLSUnittest::OnSuccessfulInit() { flusher->SetContext(ctx); flusher->SetMetricsRecordRef(FlusherSLS::sName, "1"); APSARA_TEST_TRUE(flusher->Init(configJson, optionalGoPipeline)); - APSARA_TEST_EQUAL("cn-hangzhou.log.aliyuncs.com", flusher->mEndpoint); - auto iter = SLSClientManager::GetInstance()->mRegionEndpointEntryMap.find("cn-hangzhou"); - APSARA_TEST_NOT_EQUAL(SLSClientManager::GetInstance()->mRegionEndpointEntryMap.end(), iter); - APSARA_TEST_NOT_EQUAL(iter->second.mEndpointInfoMap.end(), - iter->second.mEndpointInfoMap.find("http://cn-hangzhou.log.aliyuncs.com")); + APSARA_TEST_EQUAL("test_project.cn-hangzhou.log.aliyuncs.com", flusher->mCandidateHostsInfo->GetFirstHost()); SenderQueueManager::GetInstance()->Clear(); +#endif // TelemetryType configStr = R"( @@ -244,7 +324,7 @@ void FlusherSLSUnittest::OnSuccessfulInit() { "Logstore": "test_logstore", "Region": "cn-hangzhou", "Endpoint": "cn-hangzhou.log.aliyuncs.com", - "TelemetryType": "logs" + "TelemetryType": "metrics" } )"; APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); @@ -252,7 +332,7 @@ void FlusherSLSUnittest::OnSuccessfulInit() { flusher->SetContext(ctx); flusher->SetMetricsRecordRef(FlusherSLS::sName, "1"); APSARA_TEST_TRUE(flusher->Init(configJson, optionalGoPipeline)); - APSARA_TEST_EQUAL(sls_logs::SlsTelemetryType::SLS_TELEMETRY_TYPE_LOGS, flusher->mTelemetryType); + APSARA_TEST_EQUAL(sls_logs::SlsTelemetryType::SLS_TELEMETRY_TYPE_METRICS, flusher->mTelemetryType); SenderQueueManager::GetInstance()->Clear(); configStr = R"( @@ -486,8 +566,6 @@ void FlusherSLSUnittest::OnPipelineUpdate() { APSARA_TEST_TRUE(flusher1.Init(configJson, optionalGoPipeline)); APSARA_TEST_TRUE(flusher1.Start()); APSARA_TEST_EQUAL(1U, FlusherSLS::sProjectRefCntMap.size()); - APSARA_TEST_TRUE(FlusherSLS::IsRegionContainingConfig("cn-hangzhou")); - APSARA_TEST_EQUAL(1U, SLSClientManager::GetInstance()->GetRegionAliuids("cn-hangzhou").size()); { PipelineContext ctx2; @@ -510,14 +588,10 @@ void FlusherSLSUnittest::OnPipelineUpdate() { APSARA_TEST_TRUE(flusher1.Stop(false)); APSARA_TEST_TRUE(FlusherSLS::sProjectRefCntMap.empty()); - APSARA_TEST_FALSE(FlusherSLS::IsRegionContainingConfig("cn-hangzhou")); - APSARA_TEST_TRUE(SLSClientManager::GetInstance()->GetRegionAliuids("cn-hangzhou").empty()); APSARA_TEST_TRUE(SenderQueueManager::GetInstance()->IsQueueMarkedDeleted(flusher1.GetQueueKey())); APSARA_TEST_TRUE(flusher2.Start()); APSARA_TEST_EQUAL(1U, FlusherSLS::sProjectRefCntMap.size()); - APSARA_TEST_TRUE(FlusherSLS::IsRegionContainingConfig("cn-hangzhou")); - APSARA_TEST_EQUAL(1U, SLSClientManager::GetInstance()->GetRegionAliuids("cn-hangzhou").size()); APSARA_TEST_TRUE(SenderQueueManager::GetInstance()->IsQueueMarkedDeleted(flusher1.GetQueueKey())); APSARA_TEST_FALSE(SenderQueueManager::GetInstance()->IsQueueMarkedDeleted(flusher2.GetQueueKey())); flusher2.Stop(true); @@ -754,7 +828,7 @@ void FlusherSLSUnittest::TestSend() { APSARA_TEST_TRUE(item->mBufferOrNot); APSARA_TEST_EQUAL(&flusher, item->mFlusher); APSARA_TEST_EQUAL(flusher.mQueueKey, item->mQueueKey); - APSARA_TEST_EQUAL(sdk::CalcMD5("tag_value"), item->mShardHashKey); + APSARA_TEST_EQUAL(CalcMD5("tag_value"), item->mShardHashKey); APSARA_TEST_EQUAL(flusher.mLogstore, item->mLogstore); auto compressor @@ -1090,7 +1164,6 @@ UNIT_TEST_CASE(FlusherSLSUnittest, TestFlushAll) UNIT_TEST_CASE(FlusherSLSUnittest, TestAddPackId) UNIT_TEST_CASE(FlusherSLSUnittest, OnGoPipelineSend) - } // namespace logtail UNIT_TEST_MAIN diff --git a/core/unittest/flusher/SLSClientManagerUnittest.cpp b/core/unittest/flusher/SLSClientManagerUnittest.cpp new file mode 100644 index 0000000000..bd2ee85718 --- /dev/null +++ b/core/unittest/flusher/SLSClientManagerUnittest.cpp @@ -0,0 +1,1380 @@ +// Copyright 2024 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "app_config/AppConfig.h" +#include "plugin/flusher/sls/SLSClientManager.h" +#include "unittest/Unittest.h" + +DECLARE_FLAG_STRING(default_access_key_id); +DECLARE_FLAG_STRING(default_access_key); +DECLARE_FLAG_INT32(sls_hosts_probe_interval_sec); +DECLARE_FLAG_INT32(sls_all_hosts_probe_interval_sec); +DECLARE_FLAG_BOOL(send_prefer_real_ip); +DECLARE_FLAG_INT32(send_switch_real_ip_interval); + +using namespace std; + +namespace logtail { + +class HostInfoUnittest : public ::testing::Test { +public: + void TestHostname(); + void TestLatency(); + +private: + const std::string mHostname = "project.endpoint"; +}; + +void HostInfoUnittest::TestHostname() { + HostInfo hostInfo(mHostname); + APSARA_TEST_EQUAL(mHostname, hostInfo.GetHostname()); +} + +void HostInfoUnittest::TestLatency() { + HostInfo hostInfo(mHostname); + APSARA_TEST_TRUE(hostInfo.IsForbidden()); + + auto latency = chrono::milliseconds(100); + hostInfo.SetLatency(latency); + APSARA_TEST_EQUAL(latency, hostInfo.GetLatency()); + APSARA_TEST_FALSE(hostInfo.IsForbidden()); + + hostInfo.SetForbidden(); + APSARA_TEST_EQUAL(chrono::milliseconds::max(), hostInfo.GetLatency()); +} + +UNIT_TEST_CASE(HostInfoUnittest, TestHostname) +UNIT_TEST_CASE(HostInfoUnittest, TestLatency) + +class CandidateHostsInfoUnittest : public ::testing::Test { +public: + void TestBasicInfo(); + void TestHostsInfo(); + void TestUpdateHosts(); + void TestFirstHost(); + +private: + const string mProject = "project"; + const string mRegion = "region"; + const EndpointMode mMode = EndpointMode::DEFAULT; +}; + +void CandidateHostsInfoUnittest::TestBasicInfo() { + CandidateHostsInfo info(mProject, mRegion, mMode); + APSARA_TEST_EQUAL(mProject, info.GetProject()); + APSARA_TEST_EQUAL(mRegion, info.GetRegion()); + APSARA_TEST_EQUAL(mMode, info.GetMode()); +} + +void CandidateHostsInfoUnittest::TestHostsInfo() { + const string host1 = mProject + ".endpoint_1"; + const string host21 = mProject + ".endpoint_2_1"; + const string host22 = mProject + ".endpoint_2_2"; + const string host3 = mProject + ".endpoint_3"; + + CandidateHostsInfo info(mProject, mRegion, mMode); + info.mCandidateHosts.push_back({host1}); + info.mCandidateHosts.push_back({host21, host22}); + info.mCandidateHosts.push_back({host3}); + + // initialized + APSARA_TEST_TRUE(info.GetCurrentHost().empty()); + { + vector res; + info.GetAllHosts(res); + APSARA_TEST_EQUAL(4U, res.size()); + APSARA_TEST_EQUAL(host1, res[0]); + APSARA_TEST_EQUAL(host21, res[1]); + APSARA_TEST_EQUAL(host22, res[2]); + APSARA_TEST_EQUAL(host3, res[3]); + } + { + vector res; + info.GetProbeHosts(res); + APSARA_TEST_EQUAL(4U, res.size()); + APSARA_TEST_EQUAL(host1, res[0]); + APSARA_TEST_EQUAL(host21, res[1]); + APSARA_TEST_EQUAL(host22, res[2]); + APSARA_TEST_EQUAL(host3, res[3]); + } + + // no valid host + info.SelectBestHost(); + APSARA_TEST_TRUE(info.GetCurrentHost().empty()); + + // some hosts become valid + info.UpdateHostInfo(host21, chrono::milliseconds(100)); + info.SelectBestHost(); + + APSARA_TEST_EQUAL(host21, info.GetCurrentHost()); + { + vector res; + info.GetProbeHosts(res); + APSARA_TEST_EQUAL(1U, res.size()); + APSARA_TEST_EQUAL(host21, res[0]); + } + { + vector res; + info.GetAllHosts(res); + APSARA_TEST_EQUAL(4U, res.size()); + APSARA_TEST_EQUAL(host1, res[0]); + APSARA_TEST_EQUAL(host21, res[1]); + APSARA_TEST_EQUAL(host22, res[2]); + APSARA_TEST_EQUAL(host3, res[3]); + } + + // host with the same priority as the current one has lower latency + info.UpdateHostInfo(host22, chrono::milliseconds(50)); + info.SelectBestHost(); + + APSARA_TEST_EQUAL(host22, info.GetCurrentHost()); + { + vector res; + info.GetProbeHosts(res); + APSARA_TEST_EQUAL(2U, res.size()); + APSARA_TEST_EQUAL(host21, res[0]); + APSARA_TEST_EQUAL(host22, res[1]); + } + { + vector res; + info.GetAllHosts(res); + APSARA_TEST_EQUAL(4U, res.size()); + APSARA_TEST_EQUAL(host1, res[0]); + APSARA_TEST_EQUAL(host21, res[1]); + APSARA_TEST_EQUAL(host22, res[2]); + APSARA_TEST_EQUAL(host3, res[3]); + } + + // host with higher priority becomes valid + info.UpdateHostInfo(host1, chrono::milliseconds(200)); + info.SelectBestHost(); + + APSARA_TEST_EQUAL(host1, info.GetCurrentHost()); + { + vector res; + info.GetProbeHosts(res); + APSARA_TEST_EQUAL(3U, res.size()); + APSARA_TEST_EQUAL(host1, res[0]); + APSARA_TEST_EQUAL(host21, res[1]); + APSARA_TEST_EQUAL(host22, res[2]); + } + { + vector res; + info.GetAllHosts(res); + APSARA_TEST_EQUAL(4U, res.size()); + APSARA_TEST_EQUAL(host1, res[0]); + APSARA_TEST_EQUAL(host21, res[1]); + APSARA_TEST_EQUAL(host22, res[2]); + APSARA_TEST_EQUAL(host3, res[3]); + } + + // no change + info.SelectBestHost(); + + APSARA_TEST_EQUAL(host1, info.GetCurrentHost()); + { + vector res; + info.GetProbeHosts(res); + APSARA_TEST_EQUAL(3U, res.size()); + APSARA_TEST_EQUAL(host1, res[0]); + APSARA_TEST_EQUAL(host21, res[1]); + APSARA_TEST_EQUAL(host22, res[2]); + } + { + vector res; + info.GetAllHosts(res); + APSARA_TEST_EQUAL(4U, res.size()); + APSARA_TEST_EQUAL(host1, res[0]); + APSARA_TEST_EQUAL(host21, res[1]); + APSARA_TEST_EQUAL(host22, res[2]); + APSARA_TEST_EQUAL(host3, res[3]); + } + + // all hosts becomes invalid + info.UpdateHostInfo(host1, chrono::milliseconds::max()); + info.UpdateHostInfo(host21, chrono::milliseconds::max()); + info.UpdateHostInfo(host22, chrono::milliseconds::max()); + info.UpdateHostInfo(host3, chrono::milliseconds::max()); + info.SelectBestHost(); + + APSARA_TEST_TRUE(info.GetCurrentHost().empty()); + { + vector res; + info.GetProbeHosts(res); + APSARA_TEST_EQUAL(4U, res.size()); + APSARA_TEST_EQUAL(host1, res[0]); + APSARA_TEST_EQUAL(host21, res[1]); + APSARA_TEST_EQUAL(host22, res[2]); + APSARA_TEST_EQUAL(host3, res[3]); + } + { + vector res; + info.GetAllHosts(res); + APSARA_TEST_EQUAL(4U, res.size()); + APSARA_TEST_EQUAL(host1, res[0]); + APSARA_TEST_EQUAL(host21, res[1]); + APSARA_TEST_EQUAL(host22, res[2]); + APSARA_TEST_EQUAL(host3, res[3]); + } +} + +void CandidateHostsInfoUnittest::TestUpdateHosts() { + const string region = "region"; + const string publicEndpoint = region + ".log.aliyuncs.com"; + const string privateEndpoint = region + "-intranet.log.aliyuncs.com"; + const string internalEndpoint = region + "-internal.log.aliyuncs.com"; + const string globalEndpoint = "log.global.aliyuncs.com"; + const string customEndpoint = "custom.endpoint"; + const string publicHost = mProject + "." + publicEndpoint; + const string privateHost = mProject + "." + privateEndpoint; + const string internalHost = mProject + "." + internalEndpoint; + const string globalHost = mProject + "." + globalEndpoint; + const string customHost = mProject + "." + customEndpoint; + { + // default mode + CandidateHostsInfo info("project", region, EndpointMode::DEFAULT); + + // from no remote endpoints + CandidateEndpoints regionEndpoints{EndpointMode::DEFAULT, {publicEndpoint}, {}}; + info.UpdateHosts(regionEndpoints); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[0][0].GetLatency()); + + info.UpdateHostInfo(publicHost, chrono::milliseconds(100)); + + // to with remote endpoints + regionEndpoints.mRemoteEndpoints = {privateEndpoint, publicEndpoint}; + info.UpdateHosts(regionEndpoints); + APSARA_TEST_EQUAL(2U, info.mCandidateHosts.size()); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds(100), info.mCandidateHosts[0][0].GetLatency()); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts[1].size()); + APSARA_TEST_EQUAL(privateHost, info.mCandidateHosts[1][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[1][0].GetLatency()); + + info.UpdateHostInfo(publicHost, chrono::milliseconds(50)); + info.UpdateHostInfo(privateHost, chrono::milliseconds(150)); + + // to updated remote endpoints + regionEndpoints.mRemoteEndpoints = {internalEndpoint, privateEndpoint, publicEndpoint}; + info.UpdateHosts(regionEndpoints); + APSARA_TEST_EQUAL(3U, info.mCandidateHosts.size()); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds(50), info.mCandidateHosts[0][0].GetLatency()); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts[1].size()); + APSARA_TEST_EQUAL(internalHost, info.mCandidateHosts[1][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[1][0].GetLatency()); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts[2].size()); + APSARA_TEST_EQUAL(privateHost, info.mCandidateHosts[2][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds(150), info.mCandidateHosts[2][0].GetLatency()); + } + { + // accelerate mode + { + CandidateHostsInfo info("project", region, EndpointMode::ACCELERATE); + // from no remote endpoints + CandidateEndpoints regionEndpoints{EndpointMode::DEFAULT, {publicEndpoint}, {}}; + info.UpdateHosts(regionEndpoints); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[0][0].GetLatency()); + + info.UpdateHostInfo(globalHost, chrono::milliseconds(100)); + + // to with remote endpoints + regionEndpoints.mRemoteEndpoints = {privateEndpoint, publicEndpoint}; + info.UpdateHosts(regionEndpoints); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); + APSARA_TEST_EQUAL(2U, info.mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds(100), info.mCandidateHosts[0][0].GetLatency()); + APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][1].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[0][1].GetLatency()); + + info.UpdateHostInfo(globalHost, chrono::milliseconds(50)); + info.UpdateHostInfo(publicHost, chrono::milliseconds(150)); + + // to updated remote endpoints + regionEndpoints.mRemoteEndpoints = {internalEndpoint, privateEndpoint, publicEndpoint}; + info.UpdateHosts(regionEndpoints); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); + APSARA_TEST_EQUAL(2U, info.mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds(50), info.mCandidateHosts[0][0].GetLatency()); + APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][1].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds(150), info.mCandidateHosts[0][1].GetLatency()); + } + { + CandidateHostsInfo info("project", region, EndpointMode::ACCELERATE); + // from no remote endpoints + CandidateEndpoints regionEndpoints{EndpointMode::ACCELERATE, {globalEndpoint}, {}}; + info.UpdateHosts(regionEndpoints); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[0][0].GetLatency()); + + info.UpdateHostInfo(globalHost, chrono::milliseconds(100)); + + // to with remote endpoints + regionEndpoints.mRemoteEndpoints = {privateEndpoint, publicEndpoint}; + info.UpdateHosts(regionEndpoints); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); + APSARA_TEST_EQUAL(2U, info.mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds(100), info.mCandidateHosts[0][0].GetLatency()); + APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][1].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[0][1].GetLatency()); + + info.UpdateHostInfo(globalHost, chrono::milliseconds(50)); + info.UpdateHostInfo(publicHost, chrono::milliseconds(150)); + + // to updated remote endpoints + regionEndpoints.mRemoteEndpoints = {internalEndpoint, privateEndpoint, publicEndpoint}; + info.UpdateHosts(regionEndpoints); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); + APSARA_TEST_EQUAL(2U, info.mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds(50), info.mCandidateHosts[0][0].GetLatency()); + APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][1].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds(150), info.mCandidateHosts[0][1].GetLatency()); + } + { + CandidateHostsInfo info("project", region, EndpointMode::ACCELERATE); + // from no remote endpoints + CandidateEndpoints regionEndpoints{EndpointMode::CUSTOM, {customEndpoint}, {}}; + info.UpdateHosts(regionEndpoints); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[0][0].GetLatency()); + + info.UpdateHostInfo(globalHost, chrono::milliseconds(100)); + + // to with remote endpoints + regionEndpoints.mRemoteEndpoints = {privateEndpoint, publicEndpoint}; + info.UpdateHosts(regionEndpoints); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); + APSARA_TEST_EQUAL(2U, info.mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds(100), info.mCandidateHosts[0][0].GetLatency()); + APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][1].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[0][1].GetLatency()); + + info.UpdateHostInfo(globalHost, chrono::milliseconds(50)); + info.UpdateHostInfo(publicHost, chrono::milliseconds(150)); + + // to updated remote endpoints + regionEndpoints.mRemoteEndpoints = {internalEndpoint, privateEndpoint, publicEndpoint}; + info.UpdateHosts(regionEndpoints); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); + APSARA_TEST_EQUAL(2U, info.mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds(50), info.mCandidateHosts[0][0].GetLatency()); + APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][1].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds(150), info.mCandidateHosts[0][1].GetLatency()); + } + } + { + // custom mode + CandidateHostsInfo info("project", region, EndpointMode::CUSTOM); + + // from no remote endpoints + CandidateEndpoints regionEndpoints{EndpointMode::CUSTOM, {customEndpoint}, {}}; + info.UpdateHosts(regionEndpoints); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(customHost, info.mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[0][0].GetLatency()); + + info.UpdateHostInfo(customHost, chrono::milliseconds(100)); + + // to with remote endpoints + regionEndpoints.mRemoteEndpoints = {privateEndpoint, publicEndpoint}; + info.UpdateHosts(regionEndpoints); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); + APSARA_TEST_EQUAL(1U, info.mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(customHost, info.mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds(100), info.mCandidateHosts[0][0].GetLatency()); + } +} + +void CandidateHostsInfoUnittest::TestFirstHost() { + const string host1 = mProject + ".endpoint_1"; + const string host2 = mProject + ".endpoint_2"; + + CandidateHostsInfo info(mProject, mRegion, mMode); + APSARA_TEST_EQUAL("", info.GetFirstHost()); + + info.mCandidateHosts.push_back({host1, host2}); + APSARA_TEST_EQUAL(host1, info.GetFirstHost()); + + info.mCandidateHosts[0].clear(); + APSARA_TEST_EQUAL("", info.GetFirstHost()); +} + +UNIT_TEST_CASE(CandidateHostsInfoUnittest, TestBasicInfo) +UNIT_TEST_CASE(CandidateHostsInfoUnittest, TestHostsInfo) +UNIT_TEST_CASE(CandidateHostsInfoUnittest, TestUpdateHosts) +UNIT_TEST_CASE(CandidateHostsInfoUnittest, TestFirstHost) + +class ProbeNetworkMock { +public: + static HttpResponse DoProbeNetwork(const std::unique_ptr& req) { + HttpResponse response; + chrono::milliseconds latency = chrono::milliseconds::max(); + if (!enableUnavailableHosts) { + if (req->mProject == project1 && req->mHost == project1 + "." + publicEndpoint1) { + static uint32_t sec = 1; + latency = chrono::milliseconds(sec++); + } else if (req->mProject == project2) { + if (req->mHost == project2 + "." + publicEndpoint2) { + static uint32_t sec = 10; + latency = chrono::milliseconds(sec++); + } else if (req->mHost == project2 + "." + globalEndpoint) { + static uint32_t sec = 5; + latency = chrono::milliseconds(sec++); + } + } else if (req->mProject == project3 && req->mHost == project3 + "." + customEndpoint) { + static uint32_t sec = 20; + latency = chrono::milliseconds(sec++); + } + } else { + static uint32_t sec = 200; + latency = chrono::milliseconds(sec++); + } + + response.SetResponseTime(latency); + return response; + } + + static void Prepare() { + publicEndpoint1 = region1 + ".log.aliyuncs.com"; + privateEndpoint1 = region1 + "-intranet.log.aliyuncs.com"; + internalEndpoint1 = region1 + "-internal.log.aliyuncs.com"; + publicEndpoint2 = region2 + ".log.aliyuncs.com"; + privateEndpoint2 = region2 + "-intranet.log.aliyuncs.com"; + internalEndpoint2 = region2 + "-internal.log.aliyuncs.com"; + + // region_1: only public endpoint is available + // region_2: both accelerate and public endpoints are available + // region_3: only custom endpoint is available + // project_x belongs to region_x + SLSClientManager::GetInstance()->UpdateLocalRegionEndpointsAndHttpsInfo(region1, {publicEndpoint1}); + SLSClientManager::GetInstance()->UpdateLocalRegionEndpointsAndHttpsInfo(region2, {globalEndpoint}); + SLSClientManager::GetInstance()->UpdateLocalRegionEndpointsAndHttpsInfo(region3, {customEndpoint}); + SLSClientManager::GetInstance()->UpdateRemoteRegionEndpoints( + region1, {internalEndpoint1, privateEndpoint1, publicEndpoint1}); + SLSClientManager::GetInstance()->UpdateRemoteRegionEndpoints( + region2, {internalEndpoint2, privateEndpoint2, publicEndpoint2}); + } + + static bool enableUnavailableHosts; + + static string project1; + static string project2; + static string project3; + static string region1; + static string region2; + static string region3; + static string publicEndpoint1; + static string privateEndpoint1; + static string internalEndpoint1; + static string publicEndpoint2; + static string privateEndpoint2; + static string internalEndpoint2; + static string globalEndpoint; + static string customEndpoint; +}; + +bool ProbeNetworkMock::enableUnavailableHosts = false; +string ProbeNetworkMock::region1 = "region_1"; +string ProbeNetworkMock::region2 = "region_2"; +string ProbeNetworkMock::region3 = "region_3"; +string ProbeNetworkMock::project1 = "project_1"; +string ProbeNetworkMock::project2 = "project_2"; +string ProbeNetworkMock::project3 = "project_3"; +string ProbeNetworkMock::publicEndpoint1; +string ProbeNetworkMock::privateEndpoint1; +string ProbeNetworkMock::internalEndpoint1; +string ProbeNetworkMock::publicEndpoint2; +string ProbeNetworkMock::privateEndpoint2; +string ProbeNetworkMock::internalEndpoint2; +string ProbeNetworkMock::globalEndpoint = "log.global.aliyuncs.com"; +string ProbeNetworkMock::customEndpoint = "custom.endpoint"; + +class GetRealIpMock { +public: + static bool GetEndpointRealIpMock(const string& endpoint, string& ip) { + if (endpoint.find(normalRegion) != string::npos) { + static size_t i = 0; + ip = normalRegionIps[i]; + i = (i + 1) % normalRegionIps.size(); + return true; + } else if (endpoint.find(unavailableRegion) != string::npos) { + static size_t i = 0; + ip = unavailableRegionIps[i]; + i = (i + 1) % unavailableRegionIps.size(); + return true; + } else { + return false; + } + } + + static HttpResponse DoProbeNetwork(const std::unique_ptr& req) { + HttpResponse response; + response.SetResponseTime(chrono::milliseconds(100)); + return response; + } + + static string normalRegion; + static string unavailableRegion; + static vector normalRegionIps; + static vector unavailableRegionIps; +}; + +string GetRealIpMock::normalRegion = "region_1"; +string GetRealIpMock::unavailableRegion = "region_2"; +vector GetRealIpMock::normalRegionIps = {"192.168.0.1", "192.168.0.2"}; +vector GetRealIpMock::unavailableRegionIps = {"10.0.0.1", "10.0.0.2"}; + +class SLSClientManagerUnittest : public ::testing::Test { +public: + void TestLocalRegionEndpoints(); + void TestRemoteRegionEndpoints(); + void TestGetCandidateHostsInfo(); + void TestUpdateHostInfo(); + void TestUsingHttps(); + void TestProbeNetwork(); + void TestRealIp(); + void TestAccessKeyManagement(); + void TestGetCandidateHostsInfoOpen(); + +protected: + static void SetUpTestCase() { + SLSClientManager::GetInstance()->mDoProbeNetwork = ProbeNetworkMock::DoProbeNetwork; + SLSClientManager::GetInstance()->mGetEndpointRealIp = GetRealIpMock::GetEndpointRealIpMock; + } + + void TearDown() override { mManager->Clear(); } + +private: + SLSClientManager* mManager = SLSClientManager::GetInstance(); +}; + +void SLSClientManagerUnittest::TestLocalRegionEndpoints() { + const string region = "region"; + { + // default + const string endpoint = region + ".log.aliyuncs.com"; + mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint}); + + auto& item = mManager->mRegionCandidateEndpointsMap[region]; + APSARA_TEST_EQUAL(EndpointMode::DEFAULT, item.mMode); + APSARA_TEST_EQUAL(1U, item.mLocalEndpoints.size()); + APSARA_TEST_EQUAL(endpoint, item.mLocalEndpoints[0]); + APSARA_TEST_FALSE(mManager->UsingHttps(region)); + } + { + // accelerate (+ default) + const string endpoint1 = region + "-intranet.log.aliyuncs.com"; + const string endpoint2 = "log.global.aliyuncs.com"; + mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); + + auto& item = mManager->mRegionCandidateEndpointsMap[region]; + APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, item.mMode); + APSARA_TEST_EQUAL(0U, item.mLocalEndpoints.size()); + APSARA_TEST_FALSE(mManager->UsingHttps(region)); + } + { + // custom (+ default) + const string endpoint1 = region + "-intranet.log.aliyuncs.com"; + const string endpoint2 = "custom.endpoint"; + mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); + + auto& item = mManager->mRegionCandidateEndpointsMap[region]; + APSARA_TEST_EQUAL(EndpointMode::CUSTOM, item.mMode); + APSARA_TEST_EQUAL(1U, item.mLocalEndpoints.size()); + APSARA_TEST_EQUAL(endpoint2, item.mLocalEndpoints[0]); + APSARA_TEST_FALSE(mManager->UsingHttps(region)); + } + { + // accelerate + custom (+ default) + const string endpoint1 = "custom.endpoint"; + const string endpoint2 = region + "-intranet.log.aliyuncs.com"; + const string endpoint3 = "log.global.aliyuncs.com"; + mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2, endpoint3}); + + auto& item = mManager->mRegionCandidateEndpointsMap[region]; + APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, item.mMode); + APSARA_TEST_EQUAL(0U, item.mLocalEndpoints.size()); + APSARA_TEST_FALSE(mManager->UsingHttps(region)); + } + { + // http -> https -> https + const string endpoint1 = region + "-intranet.log.aliyuncs.com"; + const string endpoint2 = "https://log.global.aliyuncs.com"; + mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); + APSARA_TEST_TRUE(mManager->UsingHttps(region)); + + mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); + APSARA_TEST_TRUE(mManager->UsingHttps(region)); + } + { + // https -> http -> http + const string endpoint1 = region + "-intranet.log.aliyuncs.com"; + const string endpoint2 = "log.global.aliyuncs.com"; + mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); + APSARA_TEST_FALSE(mManager->UsingHttps(region)); + + mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); + APSARA_TEST_FALSE(mManager->UsingHttps(region)); + } +} + +void SLSClientManagerUnittest::TestRemoteRegionEndpoints() { + { + auto info1 = mManager->GetCandidateHostsInfo("region_1", "project_1", EndpointMode::DEFAULT); + auto info2 = mManager->GetCandidateHostsInfo("region_1", "project_2", EndpointMode::DEFAULT); + auto info3 = mManager->GetCandidateHostsInfo("region_2", "project_3", EndpointMode::DEFAULT); + APSARA_TEST_EQUAL(0U, info3->mCandidateHosts.size()); + + // create remote info + mManager->UpdateRemoteRegionEndpoints("region_1", {"endpoint_1", "endpoint_2"}); + APSARA_TEST_EQUAL(2U, info1->mCandidateHosts.size()); + APSARA_TEST_EQUAL(2U, info2->mCandidateHosts.size()); + APSARA_TEST_EQUAL(0U, info3->mCandidateHosts.size()); + APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[1].size()); + APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[1].size()); + APSARA_TEST_EQUAL("project_1.endpoint_1", info1->mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL("project_1.endpoint_2", info1->mCandidateHosts[1][0].GetHostname()); + APSARA_TEST_EQUAL("project_2.endpoint_1", info2->mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL("project_2.endpoint_2", info2->mCandidateHosts[1][0].GetHostname()); + + // update remote info with overwrite + mManager->UpdateRemoteRegionEndpoints("region_1", {"endpoint_2", "endpoint_3"}); + APSARA_TEST_EQUAL(2U, info1->mCandidateHosts.size()); + APSARA_TEST_EQUAL(2U, info2->mCandidateHosts.size()); + APSARA_TEST_EQUAL(0U, info3->mCandidateHosts.size()); + APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[1].size()); + APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[1].size()); + APSARA_TEST_EQUAL("project_1.endpoint_2", info1->mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL("project_1.endpoint_3", info1->mCandidateHosts[1][0].GetHostname()); + APSARA_TEST_EQUAL("project_2.endpoint_2", info2->mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL("project_2.endpoint_3", info2->mCandidateHosts[1][0].GetHostname()); + + // update remote info with create + mManager->UpdateRemoteRegionEndpoints( + "region_1", {"endpoint_4"}, SLSClientManager::RemoteEndpointUpdateAction::CREATE); + APSARA_TEST_EQUAL(2U, info1->mCandidateHosts.size()); + APSARA_TEST_EQUAL(2U, info2->mCandidateHosts.size()); + APSARA_TEST_EQUAL(0U, info3->mCandidateHosts.size()); + APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[1].size()); + APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[1].size()); + APSARA_TEST_EQUAL("project_1.endpoint_2", info1->mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL("project_1.endpoint_3", info1->mCandidateHosts[1][0].GetHostname()); + APSARA_TEST_EQUAL("project_2.endpoint_2", info2->mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL("project_2.endpoint_3", info2->mCandidateHosts[1][0].GetHostname()); + + // update remote info with append + mManager->UpdateRemoteRegionEndpoints( + "region_1", {"endpoint_3", "endpoint_1"}, SLSClientManager::RemoteEndpointUpdateAction::APPEND); + APSARA_TEST_EQUAL(3U, info1->mCandidateHosts.size()); + APSARA_TEST_EQUAL(3U, info2->mCandidateHosts.size()); + APSARA_TEST_EQUAL(0U, info3->mCandidateHosts.size()); + APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[1].size()); + APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[2].size()); + APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[1].size()); + APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[2].size()); + APSARA_TEST_EQUAL("project_1.endpoint_2", info1->mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL("project_1.endpoint_3", info1->mCandidateHosts[1][0].GetHostname()); + APSARA_TEST_EQUAL("project_1.endpoint_1", info1->mCandidateHosts[2][0].GetHostname()); + APSARA_TEST_EQUAL("project_2.endpoint_2", info2->mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL("project_2.endpoint_3", info2->mCandidateHosts[1][0].GetHostname()); + APSARA_TEST_EQUAL("project_2.endpoint_1", info2->mCandidateHosts[2][0].GetHostname()); + } + { + // get candidate host info after remote info is updated + auto info = mManager->GetCandidateHostsInfo("region_1", "project_1", EndpointMode::DEFAULT); + APSARA_TEST_EQUAL(3U, info->mCandidateHosts.size()); + APSARA_TEST_EQUAL(1U, info->mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(1U, info->mCandidateHosts[1].size()); + APSARA_TEST_EQUAL(1U, info->mCandidateHosts[2].size()); + APSARA_TEST_EQUAL("project_1.endpoint_2", info->mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL("project_1.endpoint_3", info->mCandidateHosts[1][0].GetHostname()); + APSARA_TEST_EQUAL("project_1.endpoint_1", info->mCandidateHosts[2][0].GetHostname()); + } +} + +void SLSClientManagerUnittest::TestGetCandidateHostsInfo() { + const string region = "region"; + const string project = "project"; + const string publicEndpoint = region + ".log.aliyuncs.com"; + const string privateEndpoint = region + "-intranet.log.aliyuncs.com"; + const string internalEndpoint = region + "-internal.log.aliyuncs.com"; + const string globalEndpoint = "log.global.aliyuncs.com"; + const string customEndpoint = "custom.endpoint"; + const string publicHost = project + "." + publicEndpoint; + const string privateHost = project + "." + privateEndpoint; + const string internalHost = project + "." + internalEndpoint; + const string globalHost = project + "." + globalEndpoint; + const string customHost = project + "." + customEndpoint; + { + // no candidate host info && no region endpoints + { + // default mode + auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); + APSARA_TEST_EQUAL(project, info->GetProject()); + APSARA_TEST_EQUAL(region, info->GetRegion()); + APSARA_TEST_EQUAL(EndpointMode::DEFAULT, info->GetMode()); + APSARA_TEST_EQUAL(0U, info->mCandidateHosts.size()); + { + APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); + auto& infos = mManager->mProjectCandidateHostsInfosMap[project]; + APSARA_TEST_EQUAL(1U, infos.size()); + auto& weakInfo = *infos.begin(); + APSARA_TEST_FALSE(weakInfo.expired()); + APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); + } + { + APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap.size()); + auto& infos = mManager->mRegionCandidateHostsInfosMap[region]; + APSARA_TEST_EQUAL(1U, infos.size()); + auto& weakInfo = *infos.begin(); + APSARA_TEST_FALSE(weakInfo.expired()); + APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); + } + APSARA_TEST_EQUAL(1U, mManager->mUnInitializedCandidateHostsInfos.size()); + APSARA_TEST_EQUAL(info.get(), mManager->mUnInitializedCandidateHostsInfos[0].lock().get()); + } + { + // accelerate mode + auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); + APSARA_TEST_EQUAL(project, info->GetProject()); + APSARA_TEST_EQUAL(region, info->GetRegion()); + APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, info->GetMode()); + APSARA_TEST_EQUAL(1U, info->mCandidateHosts.size()); + APSARA_TEST_EQUAL(1U, info->mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(globalHost, info->mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds::max(), info->mCandidateHosts[0][0].GetLatency()); + { + APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); + auto& infos = mManager->mProjectCandidateHostsInfosMap[project]; + APSARA_TEST_EQUAL(2U, infos.size()); + auto it = infos.begin(); + APSARA_TEST_TRUE(it->expired()); + ++it; + APSARA_TEST_FALSE(it->expired()); + APSARA_TEST_EQUAL(info.get(), it->lock().get()); + } + { + APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap.size()); + auto& infos = mManager->mRegionCandidateHostsInfosMap[region]; + APSARA_TEST_EQUAL(2U, infos.size()); + auto it = infos.begin(); + APSARA_TEST_TRUE(it->expired()); + ++it; + APSARA_TEST_FALSE(it->expired()); + APSARA_TEST_EQUAL(info.get(), it->lock().get()); + } + APSARA_TEST_EQUAL(2U, mManager->mUnInitializedCandidateHostsInfos.size()); + APSARA_TEST_EQUAL(info.get(), mManager->mUnInitializedCandidateHostsInfos[1].lock().get()); + } + mManager->Clear(); + } + { + // no candidate host info && with region endpoints && region endpoint mode is default + mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {privateEndpoint}); + { + // default mode + auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); + APSARA_TEST_EQUAL(project, info->GetProject()); + APSARA_TEST_EQUAL(region, info->GetRegion()); + APSARA_TEST_EQUAL(EndpointMode::DEFAULT, info->GetMode()); + APSARA_TEST_EQUAL(1U, info->mCandidateHosts.size()); + APSARA_TEST_EQUAL(1U, info->mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(privateHost, info->mCandidateHosts[0][0].GetHostname()); + { + APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); + auto& infos = mManager->mProjectCandidateHostsInfosMap[project]; + APSARA_TEST_EQUAL(1U, infos.size()); + auto& weakInfo = *infos.begin(); + APSARA_TEST_FALSE(weakInfo.expired()); + APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); + } + { + APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap.size()); + auto& infos = mManager->mRegionCandidateHostsInfosMap[region]; + APSARA_TEST_EQUAL(1U, infos.size()); + auto& weakInfo = *infos.begin(); + APSARA_TEST_FALSE(weakInfo.expired()); + APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); + } + APSARA_TEST_EQUAL(1U, mManager->mUnInitializedCandidateHostsInfos.size()); + APSARA_TEST_EQUAL(info.get(), mManager->mUnInitializedCandidateHostsInfos[0].lock().get()); + } + { + // accelerate mode + auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); + APSARA_TEST_EQUAL(project, info->GetProject()); + APSARA_TEST_EQUAL(region, info->GetRegion()); + APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, info->GetMode()); + APSARA_TEST_EQUAL(1U, info->mCandidateHosts.size()); + APSARA_TEST_EQUAL(1U, info->mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(globalHost, info->mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds::max(), info->mCandidateHosts[0][0].GetLatency()); + { + APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); + auto& infos = mManager->mProjectCandidateHostsInfosMap[project]; + APSARA_TEST_EQUAL(2U, infos.size()); + auto it = infos.begin(); + APSARA_TEST_TRUE(it->expired()); + ++it; + APSARA_TEST_FALSE(it->expired()); + APSARA_TEST_EQUAL(info.get(), it->lock().get()); + } + { + APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap.size()); + auto& infos = mManager->mRegionCandidateHostsInfosMap[region]; + APSARA_TEST_EQUAL(2U, infos.size()); + auto it = infos.begin(); + APSARA_TEST_TRUE(it->expired()); + ++it; + APSARA_TEST_FALSE(it->expired()); + APSARA_TEST_EQUAL(info.get(), it->lock().get()); + } + APSARA_TEST_EQUAL(2U, mManager->mUnInitializedCandidateHostsInfos.size()); + APSARA_TEST_EQUAL(info.get(), mManager->mUnInitializedCandidateHostsInfos[1].lock().get()); + } + mManager->Clear(); + } + { + // no candidate host info && with region endpoints && region endpoint mode is accelerate + mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {globalEndpoint}); + { + // default mode + auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); + APSARA_TEST_EQUAL(project, info->GetProject()); + APSARA_TEST_EQUAL(region, info->GetRegion()); + APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, info->GetMode()); + APSARA_TEST_EQUAL(1U, info->mCandidateHosts.size()); + APSARA_TEST_EQUAL(1U, info->mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(globalHost, info->mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds::max(), info->mCandidateHosts[0][0].GetLatency()); + { + APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); + auto& infos = mManager->mProjectCandidateHostsInfosMap[project]; + APSARA_TEST_EQUAL(1U, infos.size()); + auto& weakInfo = *infos.begin(); + APSARA_TEST_FALSE(weakInfo.expired()); + APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); + } + { + APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap.size()); + auto& infos = mManager->mRegionCandidateHostsInfosMap[region]; + APSARA_TEST_EQUAL(1U, infos.size()); + auto& weakInfo = *infos.begin(); + APSARA_TEST_FALSE(weakInfo.expired()); + APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); + } + APSARA_TEST_EQUAL(1U, mManager->mUnInitializedCandidateHostsInfos.size()); + APSARA_TEST_EQUAL(info.get(), mManager->mUnInitializedCandidateHostsInfos[0].lock().get()); + } + { + // accelerate mode + auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); + APSARA_TEST_EQUAL(project, info->GetProject()); + APSARA_TEST_EQUAL(region, info->GetRegion()); + APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, info->GetMode()); + APSARA_TEST_EQUAL(1U, info->mCandidateHosts.size()); + APSARA_TEST_EQUAL(1U, info->mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(globalHost, info->mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds::max(), info->mCandidateHosts[0][0].GetLatency()); + { + APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); + auto& infos = mManager->mProjectCandidateHostsInfosMap[project]; + APSARA_TEST_EQUAL(2U, infos.size()); + auto it = infos.begin(); + APSARA_TEST_TRUE(it->expired()); + ++it; + APSARA_TEST_FALSE(it->expired()); + APSARA_TEST_EQUAL(info.get(), it->lock().get()); + } + { + APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap.size()); + auto& infos = mManager->mRegionCandidateHostsInfosMap[region]; + APSARA_TEST_EQUAL(2U, infos.size()); + auto it = infos.begin(); + APSARA_TEST_TRUE(it->expired()); + ++it; + APSARA_TEST_FALSE(it->expired()); + APSARA_TEST_EQUAL(info.get(), it->lock().get()); + } + APSARA_TEST_EQUAL(2U, mManager->mUnInitializedCandidateHostsInfos.size()); + APSARA_TEST_EQUAL(info.get(), mManager->mUnInitializedCandidateHostsInfos[1].lock().get()); + } + mManager->Clear(); + } + { + // no candidate host info && with region endpoints && region endpoint mode is custom + mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {customEndpoint}); + { + // default mode + auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); + APSARA_TEST_EQUAL(project, info->GetProject()); + APSARA_TEST_EQUAL(region, info->GetRegion()); + APSARA_TEST_EQUAL(EndpointMode::CUSTOM, info->GetMode()); + APSARA_TEST_EQUAL(1U, info->mCandidateHosts.size()); + APSARA_TEST_EQUAL(1U, info->mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(customHost, info->mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds::max(), info->mCandidateHosts[0][0].GetLatency()); + { + APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); + auto& infos = mManager->mProjectCandidateHostsInfosMap[project]; + APSARA_TEST_EQUAL(1U, infos.size()); + auto& weakInfo = *infos.begin(); + APSARA_TEST_FALSE(weakInfo.expired()); + APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); + } + { + APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap.size()); + auto& infos = mManager->mRegionCandidateHostsInfosMap[region]; + APSARA_TEST_EQUAL(1U, infos.size()); + auto& weakInfo = *infos.begin(); + APSARA_TEST_FALSE(weakInfo.expired()); + APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); + } + APSARA_TEST_EQUAL(1U, mManager->mUnInitializedCandidateHostsInfos.size()); + APSARA_TEST_EQUAL(info.get(), mManager->mUnInitializedCandidateHostsInfos[0].lock().get()); + } + { + // accelerate mode + auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); + APSARA_TEST_EQUAL(project, info->GetProject()); + APSARA_TEST_EQUAL(region, info->GetRegion()); + APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, info->GetMode()); + APSARA_TEST_EQUAL(1U, info->mCandidateHosts.size()); + APSARA_TEST_EQUAL(1U, info->mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(globalHost, info->mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds::max(), info->mCandidateHosts[0][0].GetLatency()); + { + APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); + auto& infos = mManager->mProjectCandidateHostsInfosMap[project]; + APSARA_TEST_EQUAL(2U, infos.size()); + auto& weakInfo = *(++infos.begin()); + APSARA_TEST_FALSE(weakInfo.expired()); + APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); + } + { + APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap.size()); + auto& infos = mManager->mRegionCandidateHostsInfosMap[region]; + APSARA_TEST_EQUAL(2U, infos.size()); + auto& weakInfo = *(++infos.begin()); + APSARA_TEST_FALSE(weakInfo.expired()); + APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); + } + APSARA_TEST_EQUAL(2U, mManager->mUnInitializedCandidateHostsInfos.size()); + APSARA_TEST_EQUAL(info.get(), mManager->mUnInitializedCandidateHostsInfos[1].lock().get()); + } + mManager->Clear(); + } + { + // candidate host info exists && no region endpoints + auto info1 = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); + auto info2 = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); + APSARA_TEST_EQUAL(info1.get(), info2.get()); + APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); + APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap[project].size()); + APSARA_TEST_EQUAL(1U, mManager->mUnInitializedCandidateHostsInfos.size()); + APSARA_TEST_EQUAL(info1.get(), mManager->mUnInitializedCandidateHostsInfos[0].lock().get()); + + mManager->Clear(); + } + { + // candidate host info exists && with region endpoints && region endpoint mode is default + mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {privateEndpoint}); + auto infoDefault = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); + auto infoAcc = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); + APSARA_TEST_EQUAL(2U, mManager->mUnInitializedCandidateHostsInfos.size()); + APSARA_TEST_EQUAL(infoDefault.get(), mManager->mUnInitializedCandidateHostsInfos[0].lock().get()); + APSARA_TEST_EQUAL(infoAcc.get(), mManager->mUnInitializedCandidateHostsInfos[1].lock().get()); + { + // default mode + auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); + APSARA_TEST_EQUAL(infoDefault.get(), info.get()); + APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); + APSARA_TEST_EQUAL(2U, mManager->mProjectCandidateHostsInfosMap[project].size()); + } + { + // accelerate mode + auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); + APSARA_TEST_EQUAL(infoAcc.get(), info.get()); + APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); + APSARA_TEST_EQUAL(2U, mManager->mProjectCandidateHostsInfosMap[project].size()); + } + mManager->Clear(); + } + { + // candidate host info exists && with region endpoints && region endpoint mode is accelerate + mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {globalEndpoint}); + auto infoAcc = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); + APSARA_TEST_EQUAL(1U, mManager->mUnInitializedCandidateHostsInfos.size()); + APSARA_TEST_EQUAL(infoAcc.get(), mManager->mUnInitializedCandidateHostsInfos[0].lock().get()); + { + // default mode + auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); + APSARA_TEST_EQUAL(infoAcc.get(), info.get()); + APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); + APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap[project].size()); + } + { + // accelerate mode + auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); + APSARA_TEST_EQUAL(infoAcc.get(), info.get()); + APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); + APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap[project].size()); + } + mManager->Clear(); + } + { + // candidate host info exists && with region endpoints && region endpoint mode is custom + mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {customEndpoint}); + auto infoCustom = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); + auto infoAcc = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); + APSARA_TEST_EQUAL(2U, mManager->mUnInitializedCandidateHostsInfos.size()); + APSARA_TEST_EQUAL(infoCustom.get(), mManager->mUnInitializedCandidateHostsInfos[0].lock().get()); + APSARA_TEST_EQUAL(infoAcc.get(), mManager->mUnInitializedCandidateHostsInfos[1].lock().get()); + { + // default mode + auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); + APSARA_TEST_EQUAL(infoCustom.get(), info.get()); + APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); + APSARA_TEST_EQUAL(2U, mManager->mProjectCandidateHostsInfosMap[project].size()); + } + { + // accelerate mode + auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); + APSARA_TEST_EQUAL(infoAcc.get(), info.get()); + APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); + APSARA_TEST_EQUAL(2U, mManager->mProjectCandidateHostsInfosMap[project].size()); + } + mManager->Clear(); + } +} + +void SLSClientManagerUnittest::TestUpdateHostInfo() { + const string region = "region"; + const string project = "project"; + const EndpointMode mode = EndpointMode::DEFAULT; + const string endpoint1 = region + ".log.aliyuncs.com"; + const string endpoint2 = region + "-intranet.log.aliyuncs.com"; + const string host1 = project + "." + endpoint1; + const string host2 = project + "." + endpoint2; + mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); + { + auto info = mManager->GetCandidateHostsInfo(region, project, mode); + + APSARA_TEST_TRUE(mManager->UpdateHostInfo(project, mode, host1, chrono::milliseconds(100))); + APSARA_TEST_EQUAL(chrono::milliseconds(100), info->mCandidateHosts[0][0].GetLatency()); + } + + // expired info + APSARA_TEST_FALSE(mManager->UpdateHostInfo(project, mode, host1, chrono::milliseconds(50))); + + // unknown project + APSARA_TEST_FALSE(mManager->UpdateHostInfo("unknown_project", mode, host1, chrono::milliseconds(50))); +} + +void SLSClientManagerUnittest::TestUsingHttps() { + const string region = "region"; + const string host = region + ".log.aliyuncs.com"; + mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {host}); + APSARA_TEST_FALSE(mManager->UsingHttps(region)); + + AppConfig::GetInstance()->mSendDataPort = 443; + APSARA_TEST_TRUE(mManager->UsingHttps(region)); +} + +void SLSClientManagerUnittest::TestProbeNetwork() { + ProbeNetworkMock::Prepare(); + const string publicHost1 = ProbeNetworkMock::project1 + "." + ProbeNetworkMock::publicEndpoint1; + const string privateHost1 = ProbeNetworkMock::project1 + "." + ProbeNetworkMock::privateEndpoint1; + const string internalHost1 = ProbeNetworkMock::project1 + "." + ProbeNetworkMock::internalEndpoint1; + const string globalHost1 = ProbeNetworkMock::project1 + "." + ProbeNetworkMock::globalEndpoint; + const string publicHost2 = ProbeNetworkMock::project2 + "." + ProbeNetworkMock::publicEndpoint2; + const string globalHost2 = ProbeNetworkMock::project2 + "." + ProbeNetworkMock::globalEndpoint; + const string globalHost3 = ProbeNetworkMock::project3 + "." + ProbeNetworkMock::globalEndpoint; + const string customHost3 = ProbeNetworkMock::project3 + "." + ProbeNetworkMock::customEndpoint; + + // ! means not available + vector> infos; + infos.push_back(mManager->GetCandidateHostsInfo( + ProbeNetworkMock::region1, ProbeNetworkMock::project1, EndpointMode::DEFAULT)); // public !internal !private + infos.push_back(mManager->GetCandidateHostsInfo( + ProbeNetworkMock::region1, ProbeNetworkMock::project1, EndpointMode::ACCELERATE)); // !accelerate public + infos.push_back(mManager->GetCandidateHostsInfo( + ProbeNetworkMock::region2, ProbeNetworkMock::project2, EndpointMode::DEFAULT)); // accelerate public + infos.push_back(mManager->GetCandidateHostsInfo( + ProbeNetworkMock::region3, ProbeNetworkMock::project3, EndpointMode::DEFAULT)); // custom + infos.push_back(mManager->GetCandidateHostsInfo( + ProbeNetworkMock::region3, ProbeNetworkMock::project3, EndpointMode::ACCELERATE)); // !accelerate + + // probe uninitialized host + mManager->DoProbeUnInitializedHost(); + APSARA_TEST_TRUE(mManager->mUnInitializedCandidateHostsInfos.empty()); + APSARA_TEST_EQUAL(infos.size(), mManager->mPartiallyInitializedCandidateHostsInfos.size()); + for (size_t i = 0; i < infos.size(); ++i) { + APSARA_TEST_EQUAL(infos[i].get(), mManager->mPartiallyInitializedCandidateHostsInfos[i].lock().get()); + } + + APSARA_TEST_EQUAL(publicHost1, infos[0]->GetCurrentHost()); + APSARA_TEST_EQUAL("", infos[1]->GetCurrentHost()); + APSARA_TEST_EQUAL(globalHost2, infos[2]->GetCurrentHost()); + APSARA_TEST_EQUAL(customHost3, infos[3]->GetCurrentHost()); + APSARA_TEST_EQUAL("", infos[4]->GetCurrentHost()); + vector len = {1, 2, 1, 1, 1}; + for (size_t i = 0; i < infos.size(); ++i) { + vector res; + infos[i]->GetProbeHosts(res); + APSARA_TEST_EQUAL(len[i], res.size()); + } + + // probe partially initialized host + mManager->DoProbeHost(); + APSARA_TEST_TRUE(mManager->mPartiallyInitializedCandidateHostsInfos.empty()); + + APSARA_TEST_EQUAL(publicHost1, infos[0]->GetCurrentHost()); + APSARA_TEST_EQUAL(publicHost1, infos[1]->GetCurrentHost()); + APSARA_TEST_EQUAL(globalHost2, infos[2]->GetCurrentHost()); + APSARA_TEST_EQUAL(customHost3, infos[3]->GetCurrentHost()); + APSARA_TEST_EQUAL("", infos[4]->GetCurrentHost()); + len = {1, 1, 2, 1, 1}; + for (size_t i = 0; i < infos.size(); ++i) { + vector res; + infos[i]->GetProbeHosts(res); + APSARA_TEST_EQUAL(len[i], res.size()); + } + + vector> oldLatencies; + for (const auto& info : infos) { + auto& latency = oldLatencies.emplace_back(); + for (const auto& item : info->mCandidateHosts) { + for (const auto& entry : item) { + latency.push_back(entry.GetLatency()); + } + } + } + + // probe all avaialble hosts + INT32_FLAG(sls_hosts_probe_interval_sec) = 0; + mManager->DoProbeHost(); + + APSARA_TEST_EQUAL(publicHost1, infos[0]->GetCurrentHost()); + APSARA_TEST_EQUAL(publicHost1, infos[1]->GetCurrentHost()); + APSARA_TEST_EQUAL(globalHost2, infos[2]->GetCurrentHost()); + APSARA_TEST_EQUAL(customHost3, infos[3]->GetCurrentHost()); + APSARA_TEST_EQUAL("", infos[4]->GetCurrentHost()); + len = {1, 1, 2, 1, 1}; + for (size_t i = 0; i < infos.size(); ++i) { + vector res; + infos[i]->GetProbeHosts(res); + APSARA_TEST_EQUAL(len[i], res.size()); + } + + vector> newLatencies; + for (const auto& info : infos) { + auto& latency = newLatencies.emplace_back(); + for (const auto& item : info->mCandidateHosts) { + for (const auto& entry : item) { + latency.push_back(entry.GetLatency()); + } + } + } + + for (size_t i = 0; i < oldLatencies.size(); ++i) { + for (size_t j = 0; j < oldLatencies[i].size(); ++j) { + if (oldLatencies[i][j] != chrono::milliseconds::max()) { + APSARA_TEST_NOT_EQUAL(newLatencies[i][j], oldLatencies[i][j]); + } else { + APSARA_TEST_EQUAL(newLatencies[i][j], oldLatencies[i][j]); + } + } + } + INT32_FLAG(sls_hosts_probe_interval_sec) = 60; + + // probe all hosts + INT32_FLAG(sls_all_hosts_probe_interval_sec) = 0; + ProbeNetworkMock::enableUnavailableHosts = true; + mManager->DoProbeHost(); + + APSARA_TEST_EQUAL(publicHost1, infos[0]->GetCurrentHost()); + APSARA_TEST_EQUAL(globalHost1, infos[1]->GetCurrentHost()); + APSARA_TEST_EQUAL(globalHost2, infos[2]->GetCurrentHost()); + APSARA_TEST_EQUAL(customHost3, infos[3]->GetCurrentHost()); + APSARA_TEST_EQUAL(globalHost3, infos[4]->GetCurrentHost()); + + len = {3, 2, 2, 1, 1}; + for (size_t i = 0; i < infos.size(); ++i) { + vector res; + infos[i]->GetProbeHosts(res); + APSARA_TEST_EQUAL(len[i], res.size()); + } + ProbeNetworkMock::enableUnavailableHosts = false; + INT32_FLAG(sls_all_hosts_probe_interval_sec) = 300; + + // expired info + infos[2].reset(); + infos[4].reset(); + mManager->DoProbeHost(); + APSARA_TEST_EQUAL(2U, mManager->mProjectCandidateHostsInfosMap.size()); + APSARA_TEST_EQUAL(2U, mManager->mProjectCandidateHostsInfosMap[ProbeNetworkMock::project1].size()); + APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap[ProbeNetworkMock::project3].size()); + APSARA_TEST_EQUAL(2U, mManager->mRegionCandidateHostsInfosMap.size()); + APSARA_TEST_EQUAL(2U, mManager->mRegionCandidateHostsInfosMap[ProbeNetworkMock::region1].size()); + APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap[ProbeNetworkMock::region3].size()); +} + +void SLSClientManagerUnittest::TestRealIp() { + BOOL_FLAG(send_prefer_real_ip) = true; + mManager->mDoProbeNetwork = GetRealIpMock::DoProbeNetwork; + + const string unknownRegion = "unknown_region"; + const string normalRegionEndpoint1 = GetRealIpMock::normalRegion + "-intranet.log.aliyuncs.com"; + const string normalRegionEndpoint2 = GetRealIpMock::normalRegion + ".log.aliyuncs.com"; + const string unavailableRegionEndpoint = GetRealIpMock::unavailableRegion + ".log.aliyuncs.com"; + const string unknownRegionEndpoint = unknownRegion + ".log.aliyuncs.com"; + mManager->UpdateRemoteRegionEndpoints(GetRealIpMock::normalRegion, {normalRegionEndpoint1, normalRegionEndpoint2}); + mManager->UpdateRemoteRegionEndpoints(GetRealIpMock::unavailableRegion, {unavailableRegionEndpoint}); + mManager->UpdateRemoteRegionEndpoints(unknownRegion, {unknownRegionEndpoint}); + APSARA_TEST_EQUAL(3U, mManager->mRegionRealIpCandidateHostsInfosMap.size()); + { + vector hosts; + mManager->mRegionRealIpCandidateHostsInfosMap.at(GetRealIpMock::normalRegion).GetAllHosts(hosts); + APSARA_TEST_EQUAL(2U, hosts.size()); + APSARA_TEST_EQUAL(normalRegionEndpoint1, hosts[0]); + APSARA_TEST_EQUAL(normalRegionEndpoint2, hosts[1]); + } + { + vector hosts; + mManager->mRegionRealIpCandidateHostsInfosMap.at(GetRealIpMock::unavailableRegion).GetAllHosts(hosts); + APSARA_TEST_EQUAL(1U, hosts.size()); + APSARA_TEST_EQUAL(unavailableRegionEndpoint, hosts[0]); + } + { + vector hosts; + mManager->mRegionRealIpCandidateHostsInfosMap.at(unknownRegion).GetAllHosts(hosts); + APSARA_TEST_EQUAL(1U, hosts.size()); + APSARA_TEST_EQUAL(unknownRegionEndpoint, hosts[0]); + } + + // no endpoint available + mManager->DoUpdateRealIp(); + APSARA_TEST_EQUAL(0U, mManager->mRegionRealIpMap.size()); + + // probe host + INT32_FLAG(sls_all_hosts_probe_interval_sec) = 0; + mManager->DoProbeHost(); + INT32_FLAG(sls_all_hosts_probe_interval_sec) = 300; + for (const auto& item : mManager->mRegionRealIpCandidateHostsInfosMap) { + APSARA_TEST_NOT_EQUAL("", item.second.GetCurrentHost()); + } + + // update all regions + INT32_FLAG(send_switch_real_ip_interval) = 0; + mManager->UpdateOutdatedRealIpRegions(GetRealIpMock::unavailableRegion); + mManager->DoUpdateRealIp(); + APSARA_TEST_TRUE(mManager->mOutdatedRealIpRegions.empty()); + APSARA_TEST_EQUAL(2U, mManager->mRegionRealIpMap.size()); + APSARA_TEST_EQUAL(GetRealIpMock::normalRegionIps[0], mManager->GetRealIp(GetRealIpMock::normalRegion)); + APSARA_TEST_EQUAL(GetRealIpMock::unavailableRegionIps[0], mManager->GetRealIp(GetRealIpMock::unavailableRegion)); + INT32_FLAG(send_switch_real_ip_interval) = 60; + + // update only outdated regions + mManager->UpdateOutdatedRealIpRegions(GetRealIpMock::normalRegion); + mManager->DoUpdateRealIp(); + APSARA_TEST_TRUE(mManager->mOutdatedRealIpRegions.empty()); + APSARA_TEST_EQUAL(2U, mManager->mRegionRealIpMap.size()); + APSARA_TEST_EQUAL(GetRealIpMock::normalRegionIps[1], mManager->GetRealIp(GetRealIpMock::normalRegion)); + APSARA_TEST_EQUAL(GetRealIpMock::unavailableRegionIps[0], mManager->GetRealIp(GetRealIpMock::unavailableRegion)); + + mManager->mDoProbeNetwork = ProbeNetworkMock::DoProbeNetwork; + BOOL_FLAG(send_prefer_real_ip) = false; +} + +void SLSClientManagerUnittest::TestAccessKeyManagement() { + string accessKeyId, accessKeySecret; + SLSClientManager::AuthType type; + mManager->GetAccessKey("", type, accessKeyId, accessKeySecret); + APSARA_TEST_EQUAL(SLSClientManager::AuthType::AK, type); + APSARA_TEST_EQUAL(STRING_FLAG(default_access_key_id), accessKeyId); + APSARA_TEST_EQUAL(STRING_FLAG(default_access_key), accessKeySecret); +} + +void SLSClientManagerUnittest::TestGetCandidateHostsInfoOpen() { + const string project = "project"; + const string endpoint = "endpoint"; + CandidateHostsInfo* infoPtr = nullptr; + { + auto info1 = mManager->GetCandidateHostsInfo(project, endpoint); + auto info2 = mManager->GetCandidateHostsInfo(project, endpoint); + infoPtr = info1.get(); + APSARA_TEST_EQUAL(info1.get(), info2.get()); + APSARA_TEST_EQUAL(project, info1->GetProject()); + APSARA_TEST_EQUAL("", info1->GetRegion()); + APSARA_TEST_EQUAL(EndpointMode::CUSTOM, info1->GetMode()); + APSARA_TEST_EQUAL(1U, info1->mCandidateHosts.size()); + APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[0].size()); + APSARA_TEST_EQUAL(project + "." + endpoint, info1->mCandidateHosts[0][0].GetHostname()); + APSARA_TEST_EQUAL(chrono::milliseconds::max(), info1->mCandidateHosts[0][0].GetLatency()); + APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); + auto& infos = mManager->mProjectCandidateHostsInfosMap[project]; + APSARA_TEST_EQUAL(1U, infos.size()); + auto& weakInfo = *infos.begin(); + APSARA_TEST_FALSE(weakInfo.expired()); + APSARA_TEST_EQUAL(info1.get(), weakInfo.lock().get()); + } + { + auto info = mManager->GetCandidateHostsInfo(project, endpoint); + APSARA_TEST_NOT_EQUAL(infoPtr, info.get()); + APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); + APSARA_TEST_EQUAL(2U, mManager->mProjectCandidateHostsInfosMap[project].size()); + } +} + +UNIT_TEST_CASE(SLSClientManagerUnittest, TestLocalRegionEndpoints) +UNIT_TEST_CASE(SLSClientManagerUnittest, TestRemoteRegionEndpoints) +UNIT_TEST_CASE(SLSClientManagerUnittest, TestGetCandidateHostsInfo) +UNIT_TEST_CASE(SLSClientManagerUnittest, TestUpdateHostInfo) +UNIT_TEST_CASE(SLSClientManagerUnittest, TestUsingHttps) +UNIT_TEST_CASE(SLSClientManagerUnittest, TestProbeNetwork) +UNIT_TEST_CASE(SLSClientManagerUnittest, TestRealIp) +UNIT_TEST_CASE(SLSClientManagerUnittest, TestAccessKeyManagement) +UNIT_TEST_CASE(SLSClientManagerUnittest, TestGetCandidateHostsInfoOpen) + +} // namespace logtail + +UNIT_TEST_MAIN diff --git a/core/unittest/sdk/CMakeLists.txt b/core/unittest/sdk/CMakeLists.txt deleted file mode 100644 index f052c1a847..0000000000 --- a/core/unittest/sdk/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2022 iLogtail Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -cmake_minimum_required(VERSION 3.22) -project(sdk_unittest) - -# add_executable(sdk_common_unittest SDKCommonUnittest.cpp) -# target_link_libraries(sdk_common_unittest ${UT_BASE_TARGET}) diff --git a/core/unittest/sdk/SDKCommonUnittest.cpp b/core/unittest/sdk/SDKCommonUnittest.cpp deleted file mode 100644 index 315a3b450d..0000000000 --- a/core/unittest/sdk/SDKCommonUnittest.cpp +++ /dev/null @@ -1,290 +0,0 @@ -// Copyright 2022 iLogtail Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "unittest/Unittest.h" -#include "sdk/Common.h" -#include "sdk/Client.h" -#include "sdk/Exception.h" -#include "common/CompressTools.h" -#include "plugin/flusher/sls/EnterpriseSLSClientManager.h" - -DECLARE_FLAG_STRING(default_access_key_id); -DECLARE_FLAG_STRING(default_access_key); - -namespace logtail { - -class HttpMessageUnittest : public ::testing::Test { -public: - void TestGetServerTimeFromHeader(); -}; - -UNIT_TEST_CASE(HttpMessageUnittest, TestGetServerTimeFromHeader); - -void HttpMessageUnittest::TestGetServerTimeFromHeader() { - sdk::HttpMessage httpMsg; - EXPECT_EQ(0, httpMsg.GetServerTimeFromHeader()); - - auto& header = httpMsg.header; - header["Date"] = "Thu, 18 Feb 2021 10:11:10 GMT"; - EXPECT_EQ(1613643070, httpMsg.GetServerTimeFromHeader()); - - const time_t kTimestamp = 1613588970; - header["x-log-time"] = std::to_string(kTimestamp); - EXPECT_EQ(kTimestamp, httpMsg.GetServerTimeFromHeader()); -} - -class SDKClientUnittest : public ::testing::Test {}; - -TEST_F(SDKClientUnittest, TestNetwork) { - sdk::Client client("log-global.aliyuncs.com", - STRING_FLAG(default_access_key_id), - STRING_FLAG(default_access_key), - INT32_FLAG(sls_client_send_timeout), - "192.168.1.1", - ""); - try { - client.TestNetwork(); - ASSERT_TRUE(false); - } catch (const sdk::LOGException& e) { - const std::string& errorCode = e.GetErrorCode(); - ASSERT_EQ(errorCode, sdk::LOGE_REQUEST_ERROR); - std::cout << "ErrorMessage: " << e.GetMessage() << std::endl; - } - - // Machine to run the test might have accesibility to Internet. - client.SetSlsHost("cn-hangzhou.log.aliyuncs.com"); - try { - client.TestNetwork(); - ASSERT_TRUE(false); - } catch (const sdk::LOGException& e) { - const std::string errorCode = e.GetErrorCode(); - std::cout << errorCode << std::endl; - std::cout << e.GetMessage() << std::endl; - if (e.GetHttpCode() == 404) { - EXPECT_EQ(errorCode, sdk::LOGE_PROJECT_NOT_EXIST); - } else if (e.GetHttpCode() == 401) { - EXPECT_EQ(ConvertErrorCode(errorCode), SEND_UNAUTHORIZED); - } else if (e.GetHttpCode() == 400) { - EXPECT_EQ(ConvertErrorCode(errorCode), SEND_PARAMETER_INVALID); - } else { - std::cout << "HttpCode: " << e.GetHttpCode() << std::endl; - EXPECT_EQ(ConvertErrorCode(errorCode), SEND_NETWORK_ERROR); - } - } -} - -TEST_F(SDKClientUnittest, TestGetRealIp) { - sdk::Client client("cn-shanghai-corp.sls.aliyuncs.com", - STRING_FLAG(default_access_key_id), - STRING_FLAG(default_access_key), - INT32_FLAG(sls_client_send_timeout), - "192.168.1.1", - ""); - logtail::sdk::GetRealIpResponse resp = client.GetRealIp(); - std::cout << "realIp: " << resp.realIp << std::endl; - EXPECT_GT(resp.realIp.size(), 0L); - - client.SetSlsHost("cn-shanghai.sls.aliyuncs.com"); - resp = client.GetRealIp(); - std::cout << "realIp: " << resp.realIp << std::endl; - EXPECT_EQ(resp.realIp.size(), 0L); -} - -/* -TEST_F(SDKClientUnittest, PostLogstoreLogsSuccessOpenSource) { - std::string uid = ""; - std::string accessKeyId = ""; - std::string accessKey = ""; - std::string region = "cn-wulanchabu"; - std::string project = ""; - std::string logstore = ""; - sdk::Client client("cn-wulanchabu.log.aliyuncs.com", - accessKeyId, - accessKey, - INT32_FLAG(sls_client_send_timeout), - "192.168.1.1", - ""); - SLSControl::Instance()->SetSlsSendClientCommonParam(&client); - client.SetKeyProvider(""); - sls_logs::LogGroup logGroup; - - logGroup.set_source("192.168.1.1"); - logGroup.set_category(logstore); - logGroup.set_topic("unittest"); - - sls_logs::Log* log = logGroup.add_logs(); - log->set_time(time(NULL)); - sls_logs::Log_Content* content = nullptr; - content = log->add_contents(); - content->set_key("kk1"); - content->set_value("vv1"); - content = log->add_contents(); - content->set_key("kk2"); - content->set_value("vv2"); - - std::string oriData; - logGroup.SerializeToString(&oriData); - int32_t logSize = (int32_t)logGroup.logs_size(); - time_t curTime = time(NULL); - sls_logs::SlsCompressType compressType = sls_logs::SLS_CMP_ZSTD; - - LogGroupContext logGroupContext(region, project, logstore, compressType); - - LoggroupTimeValue* data = new LoggroupTimeValue(project, - logstore, - "ut-config", - "ut.log", - false, - uid, - "cn-huhehaote", - LOGGROUP_COMPRESSED, - logSize, - oriData.size(), - curTime, - "", - 0, - logGroupContext); - - ASSERT_TRUE(CompressData(compressType, oriData, data->mLogData)); - - try { - sdk::PostLogStoreLogsResponse resp = client.PostLogStoreLogs( - data->mProjectName, data->mLogstore, data->mLogGroupContext.mCompressType, data->mLogData, data->mRawSize); - std::cout << resp.requestId << "," << resp.statusCode << "," << resp.bodyBytes << std::endl; - } catch (const sdk::LOGException& e) { - const std::string& errorCode = e.GetErrorCode(); - std::cerr << "errorCode:" << errorCode << " errorMessage: " << e.GetMessage() << std::endl; - if (e.GetMessage().find("x-log-compresstype : zstd") == std::string::npos) { // ignore compresstype error - ASSERT_TRUE(false); - } - std::cerr << "compresstype zstd is not supported, fallback to lz4" << std::endl; - } - - // fallback to lz4 - ASSERT_TRUE(UncompressData(compressType, data->mLogData, data->mRawSize, oriData)); - - compressType = sls_logs::SLS_CMP_LZ4; - - logGroupContext.mCompressType = compressType; - - data->mLogGroupContext = logGroupContext; - - ASSERT_TRUE(CompressData(compressType, oriData, data->mLogData)); - - try { - sdk::PostLogStoreLogsResponse resp = client.PostLogStoreLogs( - data->mProjectName, data->mLogstore, data->mLogGroupContext.mCompressType, data->mLogData, data->mRawSize); - std::cout << resp.requestId << "," << resp.statusCode << "," << resp.bodyBytes << std::endl; - } catch (const sdk::LOGException& e) { - const std::string& errorCode = e.GetErrorCode(); - std::cerr << "errorCode:" << errorCode << " errorMessage: " << e.GetMessage() << std::endl; - ASSERT_TRUE(false); - } -} - -TEST_F(SDKClientUnittest, PostLogstoreLogsSuccessClosedSource) { - std::string uid = ""; - std::string accessKeyId = ""; // start with ## - std::string accessKey = ""; - std::string region = "cn-wulanchabu"; - std::string project = ""; - std::string logstore = ""; - sdk::Client client("cn-wulanchabu.log.aliyuncs.com", - accessKeyId, - accessKey, - INT32_FLAG(sls_client_send_timeout), - "192.168.1.1", - ""); - SLSControl::Instance()->SetSlsSendClientCommonParam(&client); - client.SetKeyProvider(sdk::MD5_SHA1_SALT_KEYPROVIDER); - sls_logs::LogGroup logGroup; - - logGroup.set_source("192.168.1.1"); - logGroup.set_category(logstore); - logGroup.set_topic("unittest"); - - sls_logs::Log* log = logGroup.add_logs(); - log->set_time(time(NULL)); - sls_logs::Log_Content* content = nullptr; - content = log->add_contents(); - content->set_key("kk1"); - content->set_value("vv1"); - content = log->add_contents(); - content->set_key("kk2"); - content->set_value("vv2"); - - std::string oriData; - logGroup.SerializeToString(&oriData); - int32_t logSize = (int32_t)logGroup.logs_size(); - time_t curTime = time(NULL); - - // try zstd first - sls_logs::SlsCompressType compressType = sls_logs::SLS_CMP_ZSTD; - - LogGroupContext logGroupContext(region, project, logstore, compressType); - - LoggroupTimeValue* data = new LoggroupTimeValue(project, - logstore, - "ut-config", - "ut.log", - false, - uid, - "cn-huhehaote", - LOGGROUP_COMPRESSED, - logSize, - oriData.size(), - curTime, - "", - 0, - logGroupContext); - - ASSERT_TRUE(CompressData(compressType, oriData, data->mLogData)); - try { - sdk::PostLogStoreLogsResponse resp = client.PostLogStoreLogs( - data->mProjectName, data->mLogstore, data->mLogGroupContext.mCompressType, data->mLogData, data->mRawSize); - std::cout << resp.requestId << "," << resp.statusCode << "," << resp.bodyBytes << std::endl; - } catch (const sdk::LOGException& e) { - const std::string& errorCode = e.GetErrorCode(); - std::cerr << "errorCode:" << errorCode << " errorMessage: " << e.GetMessage() << std::endl; - if (e.GetMessage().find("x-log-compresstype : zstd") == std::string::npos) { // ignore compresstype error - ASSERT_TRUE(false); - } - std::cerr << "compresstype zstd is not supported, fallback to lz4" << std::endl; - } - - // fallback to lz4 - ASSERT_TRUE(UncompressData(compressType, data->mLogData, data->mRawSize, oriData)); - - compressType = sls_logs::SLS_CMP_LZ4; - - logGroupContext.mCompressType = compressType; - - data->mLogGroupContext = logGroupContext; - - ASSERT_TRUE(CompressData(compressType, oriData, data->mLogData)); - - try { - sdk::PostLogStoreLogsResponse resp = client.PostLogStoreLogs( - data->mProjectName, data->mLogstore, data->mLogGroupContext.mCompressType, data->mLogData, data->mRawSize); - std::cout << resp.requestId << "," << resp.statusCode << "," << resp.bodyBytes << std::endl; - } catch (const sdk::LOGException& e) { - const std::string& errorCode = e.GetErrorCode(); - std::cerr << "errorCode:" << errorCode << " errorMessage: " << e.GetMessage() << std::endl; - ASSERT_TRUE(false); - } -} -*/ -} // namespace logtail - -UNIT_TEST_MAIN diff --git a/core/unittest/serializer/SLSSerializerUnittest.cpp b/core/unittest/serializer/SLSSerializerUnittest.cpp index 75a36a307d..d542ec95cb 100644 --- a/core/unittest/serializer/SLSSerializerUnittest.cpp +++ b/core/unittest/serializer/SLSSerializerUnittest.cpp @@ -218,7 +218,7 @@ void SLSSerializerUnittest::TestSerializeEventGroup() { // span string res, errorMsg; auto events = CreateBatchedSpanEvents(); - APSARA_TEST_EQUAL(events.mEvents.size(), 1); + APSARA_TEST_EQUAL(events.mEvents.size(), 1U); APSARA_TEST_TRUE(events.mEvents[0]->GetType() == PipelineEvent::Type::SPAN); APSARA_TEST_TRUE(serializer.DoSerialize(std::move(events), res, errorMsg)); sls_logs::LogGroup logGroup; @@ -256,7 +256,7 @@ void SLSSerializerUnittest::TestSerializeEventGroup() { std::istringstream s(attrs); bool ret = Json::parseFromStream(readerBuilder, s, &jsonVal, &errs); APSARA_TEST_TRUE(ret); - APSARA_TEST_EQUAL(jsonVal.size(), 10); + APSARA_TEST_EQUAL(jsonVal.size(), 10U); APSARA_TEST_EQUAL(jsonVal["rpcType"].asString(), "25"); APSARA_TEST_EQUAL(jsonVal["scope-tag-0"].asString(), "scope-value-0"); // APSARA_TEST_EQUAL(logGroup.logs(0).contents(7).value(), ""); @@ -268,7 +268,7 @@ void SLSSerializerUnittest::TestSerializeEventGroup() { std::istringstream ss(linksStr); ret = Json::parseFromStream(readerBuilder, ss, &jsonVal, &errs); APSARA_TEST_TRUE(ret); - APSARA_TEST_EQUAL(jsonVal.size(), 1); + APSARA_TEST_EQUAL(jsonVal.size(), 1U); for (auto& link : jsonVal) { APSARA_TEST_EQUAL(link["spanId"].asString(), "inner-link-spanid"); APSARA_TEST_EQUAL(link["traceId"].asString(), "inner-link-traceid"); @@ -280,7 +280,7 @@ void SLSSerializerUnittest::TestSerializeEventGroup() { std::istringstream sss(eventsStr); ret = Json::parseFromStream(readerBuilder, sss, &jsonVal, &errs); APSARA_TEST_TRUE(ret); - APSARA_TEST_EQUAL(jsonVal.size(), 1); + APSARA_TEST_EQUAL(jsonVal.size(), 1U); for (auto& event : jsonVal) { APSARA_TEST_EQUAL(event["name"].asString(), "inner-event"); APSARA_TEST_EQUAL(event["timestamp"].asString(), "1000"); From 6d4f05dc769f80ddc063ec79a8629203deca341c Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Mon, 9 Dec 2024 09:53:12 +0000 Subject: [PATCH 02/41] polish --- core/plugin/flusher/sls/SLSConstant.cpp | 2 ++ core/plugin/flusher/sls/SLSConstant.h | 2 ++ 2 files changed, 4 insertions(+) diff --git a/core/plugin/flusher/sls/SLSConstant.cpp b/core/plugin/flusher/sls/SLSConstant.cpp index f6c340ddc4..d81dbf284e 100644 --- a/core/plugin/flusher/sls/SLSConstant.cpp +++ b/core/plugin/flusher/sls/SLSConstant.cpp @@ -22,6 +22,8 @@ const string LOGSTORES = "/logstores"; const string METRICSTORES = "/prometheus"; const string HEALTH = "/health"; +const string LOGTAIL_USER_AGENT = "ali-log-logtail"; + const string CONTENT_MD5 = "Content-MD5"; const string LOG_HEADER_PREFIX = "x-log-"; diff --git a/core/plugin/flusher/sls/SLSConstant.h b/core/plugin/flusher/sls/SLSConstant.h index fbf5956908..5874d5f2ec 100644 --- a/core/plugin/flusher/sls/SLSConstant.h +++ b/core/plugin/flusher/sls/SLSConstant.h @@ -26,6 +26,8 @@ extern const std::string HEALTH; extern const std::string CONTENT_MD5; +extern const std::string LOGTAIL_USER_AGENT; + extern const std::string LOG_HEADER_PREFIX; extern const std::string LOG_OLD_HEADER_PREFIX; extern const std::string ACS_HEADER_PREFIX; From c9d725b67ac257b3581d3789c24ceea1ad6163dc Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Tue, 10 Dec 2024 08:17:25 +0000 Subject: [PATCH 03/41] polish --- core/app_config/AppConfig.cpp | 17 - core/app_config/AppConfig.h | 4 +- core/application/Application.cpp | 13 - core/common/EndpointUtil.cpp | 38 +- core/common/EndpointUtil.h | 4 +- core/file_server/event_handler/LogInput.cpp | 5 - core/monitor/Monitor.cpp | 1 - core/plugin/flusher/sls/DiskBufferWriter.cpp | 10 +- core/plugin/flusher/sls/FlusherSLS.cpp | 25 +- core/plugin/flusher/sls/SLSClientManager.cpp | 30 +- core/plugin/flusher/sls/SLSClientManager.h | 88 +- core/unittest/flusher/FlusherSLSUnittest.cpp | 14 +- .../flusher/SLSClientManagerUnittest.cpp | 1784 ++++++++--------- 13 files changed, 1017 insertions(+), 1016 deletions(-) diff --git a/core/app_config/AppConfig.cpp b/core/app_config/AppConfig.cpp index 202d9f67f1..c3bcf00fbb 100644 --- a/core/app_config/AppConfig.cpp +++ b/core/app_config/AppConfig.cpp @@ -122,8 +122,6 @@ DECLARE_FLAG_INT32(reader_close_unused_file_time); DECLARE_FLAG_INT32(batch_send_interval); DECLARE_FLAG_INT32(batch_send_metric_size); -DECLARE_FLAG_BOOL(send_prefer_real_ip); -DECLARE_FLAG_INT32(send_switch_real_ip_interval); DECLARE_FLAG_INT32(truncate_pos_skip_bytes); DECLARE_FLAG_INT32(default_tail_limit_kb); @@ -1002,26 +1000,11 @@ void AppConfig::LoadResourceConf(const Json::Value& confJson) { mCheckPointFilePath = AbsolutePath(mCheckPointFilePath, mProcessExecutionDir); LOG_INFO(sLogger, ("logtail checkpoint path", mCheckPointFilePath)); - if (confJson.isMember("send_prefer_real_ip") && confJson["send_prefer_real_ip"].isBool()) { - BOOL_FLAG(send_prefer_real_ip) = confJson["send_prefer_real_ip"].asBool(); - } - - if (confJson.isMember("send_switch_real_ip_interval") && confJson["send_switch_real_ip_interval"].isInt()) { - INT32_FLAG(send_switch_real_ip_interval) = confJson["send_switch_real_ip_interval"].asInt(); - } - LoadInt32Parameter(INT32_FLAG(truncate_pos_skip_bytes), confJson, "truncate_pos_skip_bytes", "ALIYUN_LOGTAIL_TRUNCATE_POS_SKIP_BYTES"); - if (BOOL_FLAG(send_prefer_real_ip)) { - LOG_INFO(sLogger, - ("change send policy, prefer use real ip, switch interval seconds", - INT32_FLAG(send_switch_real_ip_interval))("truncate skip read offset", - INT32_FLAG(truncate_pos_skip_bytes))); - } - if (confJson.isMember("ignore_dir_inode_changed") && confJson["ignore_dir_inode_changed"].isBool()) { mIgnoreDirInodeChanged = confJson["ignore_dir_inode_changed"].asBool(); } diff --git a/core/app_config/AppConfig.h b/core/app_config/AppConfig.h index 7499df5e4e..bf4c7098d3 100644 --- a/core/app_config/AppConfig.h +++ b/core/app_config/AppConfig.h @@ -306,7 +306,7 @@ class AppConfig { public: AppConfig(); - ~AppConfig(){}; + ~AppConfig() {}; void LoadInstanceConfig(const std::map>&); @@ -519,7 +519,7 @@ class AppConfig { friend class InputPrometheusUnittest; friend class InputContainerStdioUnittest; friend class BatcherUnittest; - friend class SLSClientManagerUnittest; + friend class EnterpriseSLSClientManagerUnittest; #endif }; diff --git a/core/application/Application.cpp b/core/application/Application.cpp index 88cb9e5a31..a000020970 100644 --- a/core/application/Application.cpp +++ b/core/application/Application.cpp @@ -73,9 +73,6 @@ DEFINE_FLAG_INT32(queue_check_gc_interval_sec, "30s", 30); DEFINE_FLAG_BOOL(enable_cgroup, "", false); #endif -DECLARE_FLAG_BOOL(send_prefer_real_ip); -DECLARE_FLAG_BOOL(global_network_success); - using namespace std; namespace logtail { @@ -393,16 +390,6 @@ void Application::CheckCriticalCondition(int32_t curTime) { _exit(1); } #endif - // if network is fail in 2 hours, force exit (for ant only) - // work around for no network when docker start - if (BOOL_FLAG(send_prefer_real_ip) && !BOOL_FLAG(global_network_success) && curTime - mStartTime > 7200) { - LOG_ERROR(sLogger, ("network is fail", "prepare force exit")); - AlarmManager::GetInstance()->SendAlarm(LOGTAIL_CRASH_ALARM, - "network is fail since " + ToString(mStartTime) + " force exit"); - AlarmManager::GetInstance()->ForceToSend(); - sleep(10); - _exit(1); - } } bool Application::GetUUIDThread() { diff --git a/core/common/EndpointUtil.cpp b/core/common/EndpointUtil.cpp index 16e7839160..c466ab2eb3 100644 --- a/core/common/EndpointUtil.cpp +++ b/core/common/EndpointUtil.cpp @@ -16,6 +16,7 @@ #include "common/EndpointUtil.h" +#include "common/StringTools.h" #include "logger/Logger.h" using namespace std; @@ -23,22 +24,47 @@ using namespace std; namespace logtail { bool IsHttpsEndpoint(const string& endpoint) { - return endpoint.find("https://") == 0; + string trimmedEndpoint = TrimString(endpoint); + return trimmedEndpoint.find("https://") == 0; } -string StandardizeEndpoint(const string& endpoint) { - auto bpos = endpoint.find("://"); +string StandardizeHost(const string& endpoint, const string& defaultEndpoint) { + string res = endpoint; + if (endpoint.find("https://") == 0) { + if (endpoint.size() < string("https://x").size()) { + LOG_WARNING(sLogger, ("invalid endpoint", endpoint)("use default instead", defaultEndpoint)); + return defaultEndpoint; + } + } else if (endpoint.find("http://") == 0) { + if (endpoint.size() < string("http://x").size()) { + LOG_WARNING(sLogger, ("invalid endpoint", endpoint)("use default instead", defaultEndpoint)); + return defaultEndpoint; + } + } else { + res = string("http://") + endpoint; + LOG_DEBUG(sLogger, ("add default protocol for endpoint, old", endpoint)("new", res)); + } + // trim the last '/' + if (res[res.size() - 1] == '/') { + return res.substr(0, res.size() - 1); + } + return res; +} + +string ExtractEndpoint(const string& endpoint) { + string trimmedEndpoint = TrimString(endpoint); + auto bpos = trimmedEndpoint.find("://"); if (bpos == string::npos) { bpos = 0; } else { bpos += strlen("://"); } - auto epos = endpoint.find("/", bpos); + auto epos = trimmedEndpoint.find("/", bpos); if (epos == string::npos) { - epos = endpoint.length(); + epos = trimmedEndpoint.length(); } - return endpoint.substr(bpos, epos - bpos); + return trimmedEndpoint.substr(bpos, epos - bpos); } string GetHostFromEndpoint(const std::string& endpoint) { diff --git a/core/common/EndpointUtil.h b/core/common/EndpointUtil.h index 1791706c2a..3651a10e48 100644 --- a/core/common/EndpointUtil.h +++ b/core/common/EndpointUtil.h @@ -22,7 +22,9 @@ namespace logtail { bool IsHttpsEndpoint(const std::string& endpoint); -std::string StandardizeEndpoint(const std::string& endpoint); +std::string ExtractEndpoint(const std::string& endpoint); + +std::string StandardizeHost(const std::string& endpoint, const std::string& defaultEndpoint); std::string GetHostFromEndpoint(const std::string& endpoint); diff --git a/core/file_server/event_handler/LogInput.cpp b/core/file_server/event_handler/LogInput.cpp index 340a6b6763..aea2e54358 100644 --- a/core/file_server/event_handler/LogInput.cpp +++ b/core/file_server/event_handler/LogInput.cpp @@ -39,9 +39,6 @@ #include "logger/Logger.h" #include "monitor/AlarmManager.h" #include "monitor/Monitor.h" -#ifdef __ENTERPRISE__ -#include "config/provider/EnterpriseConfigProvider.h" -#endif #include "file_server/FileServer.h" using namespace std; @@ -60,8 +57,6 @@ DEFINE_FLAG_BOOL(force_close_file_on_container_stopped, "whether close file handler immediately when associate container stopped", false); -DECLARE_FLAG_BOOL(send_prefer_real_ip); - namespace logtail { LogInput::LogInput() : mAccessMainThreadRWL(ReadWriteLock::PREFER_WRITER) { diff --git a/core/monitor/Monitor.cpp b/core/monitor/Monitor.cpp index bcb05ebf6f..015c9690d4 100644 --- a/core/monitor/Monitor.cpp +++ b/core/monitor/Monitor.cpp @@ -52,7 +52,6 @@ using namespace std; using namespace sls_logs; DEFINE_FLAG_BOOL(logtail_dump_monitor_info, "enable to dump Logtail monitor info (CPU, mem)", false); -DECLARE_FLAG_BOOL(send_prefer_real_ip); DECLARE_FLAG_BOOL(check_profile_region); namespace logtail { diff --git a/core/plugin/flusher/sls/DiskBufferWriter.cpp b/core/plugin/flusher/sls/DiskBufferWriter.cpp index 077f91754a..9cfb3a9917 100644 --- a/core/plugin/flusher/sls/DiskBufferWriter.cpp +++ b/core/plugin/flusher/sls/DiskBufferWriter.cpp @@ -28,6 +28,9 @@ #include "pipeline/queue/QueueKeyManager.h" #include "pipeline/queue/SLSSenderQueueItem.h" #include "plugin/flusher/sls/FlusherSLS.h" +#ifdef __ENTERPRISE__ +#include "plugin/flusher/sls/EnterpriseSLSClientManager.h" +#endif #include "plugin/flusher/sls/SLSClientManager.h" #include "plugin/flusher/sls/SLSConstant.h" #include "protobuf/sls/sls_logs.pb.h" @@ -809,11 +812,10 @@ SLSResponse DiskBufferWriter::SendBufferFileData(const sls_logs::LogtailBufferMe #ifdef __ENTERPRISE__ if (bufferMeta.endpointmode() == sls_logs::EndpointMode::DEFAULT) { - mRegion = region; - SLSClientManager::GetInstance()->UpdateRemoteRegionEndpoints( - region, {bufferMeta.endpoint()}, SLSClientManager::RemoteEndpointUpdateAction::CREATE); + EnterpriseSLSClientManager::GetInstance()->UpdateRemoteRegionEndpoints( + region, {bufferMeta.endpoint()}, EnterpriseSLSClientManager::RemoteEndpointUpdateAction::CREATE); } - auto info = SLSClientManager::GetInstance()->GetCandidateHostsInfo( + auto info = EnterpriseSLSClientManager::GetInstance()->GetCandidateHostsInfo( region, bufferMeta.project(), GetEndpointMode(bufferMeta.endpointmode())); #else auto info = SLSClientManager::GetInstance()->GetCandidateHostsInfo(bufferMeta.project(), bufferMeta.endpoint()); diff --git a/core/plugin/flusher/sls/FlusherSLS.cpp b/core/plugin/flusher/sls/FlusherSLS.cpp index 50d58e57ec..a13dfa62c8 100644 --- a/core/plugin/flusher/sls/FlusherSLS.cpp +++ b/core/plugin/flusher/sls/FlusherSLS.cpp @@ -31,6 +31,9 @@ #include "pipeline/queue/QueueKeyManager.h" #include "pipeline/queue/SLSSenderQueueItem.h" #include "pipeline/queue/SenderQueueManager.h" +#ifdef __ENTERPRISE__ +#include "plugin/flusher/sls/EnterpriseSLSClientManager.h" +#endif #include "plugin/flusher/sls/PackIdManager.h" #include "plugin/flusher/sls/SLSClientManager.h" #include "plugin/flusher/sls/SLSConstant.h" @@ -57,7 +60,6 @@ DEFINE_FLAG_INT32(unauthorized_allowed_delay_after_reset, "allowed delay to retr DEFINE_FLAG_INT32(discard_send_fail_interval, "discard data when send fail after 6 * 3600 seconds", 6 * 3600); DEFINE_FLAG_INT32(profile_data_send_retrytimes, "how many times should retry if profile data send fail", 5); DEFINE_FLAG_INT32(unknow_error_try_max, "discard data when try times > this value", 5); -DEFINE_FLAG_BOOL(global_network_success, "global network success flag, default false", false); DEFINE_FLAG_BOOL(enable_metricstore_channel, "only works for metrics data for enhance metrics query performance", true); DEFINE_FLAG_INT32(max_send_log_group_size, "bytes", 10 * 1024 * 1024); DEFINE_FLAG_DOUBLE(sls_serialize_size_expansion_ratio, "", 1.2); @@ -356,7 +358,7 @@ bool FlusherSLS::Init(const Json::Value& config, Json::Value& optionalGoPipeline if (mEndpointMode == EndpointMode::DEFAULT) { // for local pipeline whose flusher region is neither specified in local info nor included by config provider, // param Endpoint should be used, and the mode is set to default. - // warning: if inconsistency exists among configs, only the first config would be considered in this situation. + // warning: if inconsistency exists among configs, only the first config would be considered in this situation. if (!GetOptionalStringParam(config, "Endpoint", mEndpoint, errorMsg)) { PARAM_WARNING_IGNORE(mContext->GetLogger(), mContext->GetAlarm(), @@ -367,10 +369,11 @@ bool FlusherSLS::Init(const Json::Value& config, Json::Value& optionalGoPipeline mContext->GetLogstoreName(), mContext->GetRegion()); } - SLSClientManager::GetInstance()->UpdateRemoteRegionEndpoints( - mRegion, {mEndpoint}, SLSClientManager::RemoteEndpointUpdateAction::CREATE); + EnterpriseSLSClientManager::GetInstance()->UpdateRemoteRegionEndpoints( + mRegion, {mEndpoint}, EnterpriseSLSClientManager::RemoteEndpointUpdateAction::CREATE); } - mCandidateHostsInfo = SLSClientManager::GetInstance()->GetCandidateHostsInfo(mRegion, mProject, mEndpointMode); + mCandidateHostsInfo + = EnterpriseSLSClientManager::GetInstance()->GetCandidateHostsInfo(mRegion, mProject, mEndpointMode); #else // Endpoint if (!GetMandatoryStringParam(config, "Endpoint", mEndpoint, errorMsg)) { @@ -591,8 +594,9 @@ bool FlusherSLS::BuildRequest(SenderQueueItem* item, unique_ptr } auto data = static_cast(item); +#ifdef __ENTERPRISE__ if (BOOL_FLAG(send_prefer_real_ip)) { - data->mCurrentHost = SLSClientManager::GetInstance()->GetRealIp(mRegion); + data->mCurrentHost = EnterpriseSLSClientManager::GetInstance()->GetRealIp(mRegion); if (data->mCurrentHost.empty()) { data->mCurrentHost = mCandidateHostsInfo->GetCurrentHost(); data->mRealIpFlag = false; @@ -600,8 +604,11 @@ bool FlusherSLS::BuildRequest(SenderQueueItem* item, unique_ptr data->mRealIpFlag = true; } } else { +#endif data->mCurrentHost = mCandidateHostsInfo->GetCurrentHost(); +#ifdef __ENTERPRISE__ } +#endif if (data->mCurrentHost.empty()) { *keepItem = true; GetRegionConcurrencyLimiter(mRegion)->OnFail(); @@ -675,17 +682,18 @@ void FlusherSLS::OnSendDone(const HttpResponse& response, SenderQueueItem* item) } } suggestion << "check network connection to endpoint"; +#ifdef __ENTERPRISE__ if (data->mRealIpFlag) { // connect refused, use vip directly failDetail << ", real ip may be stale, force update"; - SLSClientManager::GetInstance()->UpdateOutdatedRealIpRegions(mRegion); + EnterpriseSLSClientManager::GetInstance()->UpdateOutdatedRealIpRegions(mRegion); } +#endif operation = data->mBufferOrNot ? OperationOnFail::RETRY_LATER : OperationOnFail::DISCARD; GetRegionConcurrencyLimiter(mRegion)->OnFail(); GetProjectConcurrencyLimiter(mProject)->OnSuccess(); GetLogstoreConcurrencyLimiter(mProject, mLogstore)->OnSuccess(); } else if (sendResult == SEND_QUOTA_EXCEED) { - BOOL_FLAG(global_network_success) = true; if (slsResponse.mErrorCode == LOGE_SHARD_WRITE_QUOTA_EXCEED) { failDetail << "shard write quota exceed"; suggestion << "Split logstore shards. https://help.aliyun.com/zh/sls/user-guide/expansion-of-resources"; @@ -718,7 +726,6 @@ void FlusherSLS::OnSendDone(const HttpResponse& response, SenderQueueItem* item) failDetail << "write unauthorized"; suggestion << "check access keys provided"; operation = OperationOnFail::RETRY_LATER; - BOOL_FLAG(global_network_success) = true; hasAuthError = true; if (mUnauthErrorCnt) { mUnauthErrorCnt->Add(1); diff --git a/core/plugin/flusher/sls/SLSClientManager.cpp b/core/plugin/flusher/sls/SLSClientManager.cpp index 0b19ca1be4..343e8525b9 100644 --- a/core/plugin/flusher/sls/SLSClientManager.cpp +++ b/core/plugin/flusher/sls/SLSClientManager.cpp @@ -20,6 +20,9 @@ #include "app_config/AppConfig.h" #include "common/EndpointUtil.h" +#ifdef __ENTERPRISE__ +#include "common/EnterpriseEndpointUtil.h" +#endif #include "common/Flags.h" #include "common/HashUtil.h" #include "common/LogtailCommonFlags.h" @@ -81,7 +84,7 @@ namespace logtail { // return EndpointAddressType::INNER; // } -// static const string globalAccelerationEndpoint = "log.global.aliyuncs.com"; +// static const string kAccelerationDataEndpoint = "log-global.aliyuncs.com"; const string& EndpointModeToString(EndpointMode mode) { switch (mode) { @@ -161,7 +164,7 @@ void CandidateHostsInfo::UpdateHosts(const CandidateEndpoints& regionEndpoints) break; } case EndpointMode::ACCELERATE: { - vector endpoints{globalAccelerationEndpoint}; + vector endpoints{kAccelerationDataEndpoint}; for (const auto& item : regionEndpoints.mRemoteEndpoints) { if (GetEndpointAddressType(item) == EndpointAddressType::PUBLIC) { endpoints.emplace_back(item); @@ -348,11 +351,11 @@ void CandidateHostsInfo::SetCurrentHost(const string& host) { SLSClientManager* SLSClientManager::GetInstance() { #ifdef __ENTERPRISE__ - static auto ptr = unique_ptr(new EnterpriseSLSClientManager()); + return EnterpriseSLSClientManager::GetInstance(); #else static auto ptr = unique_ptr(new SLSClientManager()); -#endif return ptr.get(); +#endif } void SLSClientManager::Init() { @@ -433,7 +436,7 @@ bool SLSClientManager::GetAccessKey(const string& aliuid, // const vector& rawEndpoints) { // vector endpoints; // for (const auto& item : rawEndpoints) { -// auto tmp = StandardizeEndpoint(item); +// auto tmp = ExtracteEndpoint(item); // if (!tmp.empty()) { // endpoints.emplace_back(tmp); // } @@ -445,7 +448,7 @@ bool SLSClientManager::GetAccessKey(const string& aliuid, // candidate.mLocalEndpoints.clear(); // for (const auto& item : endpoints) { // // if both acclerate and custom endpoints are given, we ignore custom endpoints -// if (item == globalAccelerationEndpoint) { +// if (item == kAccelerationDataEndpoint) { // candidate.mMode = EndpointMode::ACCELERATE; // break; // } @@ -483,7 +486,7 @@ bool SLSClientManager::GetAccessKey(const string& aliuid, // RemoteEndpointUpdateAction action) { // vector endpoints; // for (const auto& item : rawEndpoints) { -// auto tmp = StandardizeEndpoint(item); +// auto tmp = ExtractEndpoint(item); // if (!tmp.empty()) { // endpoints.emplace_back(tmp); // } @@ -625,7 +628,7 @@ shared_ptr SLSClientManager::GetCandidateHostsInfo(const str return nullptr; } - string standardEndpoint = StandardizeEndpoint(endpoint); + string standardEndpoint = ExtractEndpoint(endpoint); { lock_guard lock(mCandidateHostsInfosMapMux); auto& hostsInfo = mProjectCandidateHostsInfosMap[project]; @@ -649,9 +652,7 @@ shared_ptr SLSClientManager::GetCandidateHostsInfo(const str } bool SLSClientManager::UsingHttps(const string& region) const { - if (AppConfig::GetInstance()->GetDataServerPort() == 443) { - return true; - } + return true; // lock_guard lock(mHttpsRegionsMux); // return mHttpsRegions.find(region) != mHttpsRegions.end(); } @@ -1067,14 +1068,7 @@ bool SLSClientManager::PingEndpoint(const string& host, const string& path) { #ifdef APSARA_UNIT_TEST_MAIN void SLSClientManager::Clear() { - mRegionCandidateEndpointsMap.clear(); - mHttpsRegions.clear(); - mRegionCandidateHostsInfosMap.clear(); mProjectCandidateHostsInfosMap.clear(); - mUnInitializedCandidateHostsInfos.clear(); - mRegionRealIpMap.clear(); - mOutdatedRealIpRegions.clear(); - mRegionRealIpCandidateHostsInfosMap.clear(); } #endif diff --git a/core/plugin/flusher/sls/SLSClientManager.h b/core/plugin/flusher/sls/SLSClientManager.h index 39035f2272..d72cfd5e62 100644 --- a/core/plugin/flusher/sls/SLSClientManager.h +++ b/core/plugin/flusher/sls/SLSClientManager.h @@ -16,26 +16,26 @@ #pragma once -#include +// #include -#include +// #include #include -#include +// #include #include -#include -#include +// #include +// #include #include #include #include #include #include #include -#include -#include +// #include +// #include #include -#include "common/http/HttpRequest.h" -#include "common/http/HttpResponse.h" +// #include "common/http/HttpRequest.h" +// #include "common/http/HttpResponse.h" #include "pipeline/queue/SenderQueueItem.h" #include "plugin/flusher/sls/SLSResponse.h" @@ -108,6 +108,7 @@ class CandidateHostsInfo { #ifdef APSARA_UNIT_TEST_MAIN friend class CandidateHostsInfoUnittest; friend class SLSClientManagerUnittest; + friend class EnterpriseSLSClientManagerUnittest; #endif }; @@ -132,8 +133,8 @@ class SLSClientManager { virtual void UpdateAccessKeyStatus(const std::string& aliuid, bool success) {} // // currently not support hot relead - // void UpdateLocalRegionEndpointsAndHttpsInfo(const std::string& region, const std::vector& endpoints); - // void UpdateRemoteRegionEndpoints(const std::string& region, + // void UpdateLocalRegionEndpointsAndHttpsInfo(const std::string& region, const std::vector& + // endpoints); void UpdateRemoteRegionEndpoints(const std::string& region, // const std::vector& endpoints, // RemoteEndpointUpdateAction action = RemoteEndpointUpdateAction::OVERWRITE); @@ -144,7 +145,8 @@ class SLSClientManager { // const std::string& host, // const std::chrono::milliseconds& latency); // // only for real ip - // bool UpdateHostInfo(const std::string& region, const std::string& host, const std::chrono::milliseconds& latency); + // bool UpdateHostInfo(const std::string& region, const std::string& host, const std::chrono::milliseconds& + // latency); // only for open source std::shared_ptr GetCandidateHostsInfo(const std::string& project, const std::string& endpoint); @@ -157,7 +159,7 @@ class SLSClientManager { std::string GetRegionFromEndpoint(const std::string& endpoint); // for backward compatibility #ifdef APSARA_UNIT_TEST_MAIN - void Clear(); + virtual void Clear(); #endif protected: @@ -206,49 +208,47 @@ class SLSClientManager { // std::unordered_set mHttpsRegions; mutable std::mutex mCandidateHostsInfosMapMux; - std::map>> mRegionCandidateHostsInfosMap; - // for opensource, only custom mode is supported, and one project supports multiple custom endpoints - // for enterprise, only one info for each mode is supported + // only custom mode is supported, and one project supports multiple custom endpoints std::map>> mProjectCandidateHostsInfosMap; -// CURLM* mUnInitializedHostProbeClient = nullptr; -// mutable std::mutex mUnInitializedCandidateHostsInfosMux; -// std::vector> mUnInitializedCandidateHostsInfos; + // CURLM* mUnInitializedHostProbeClient = nullptr; + // mutable std::mutex mUnInitializedCandidateHostsInfosMux; + // std::vector> mUnInitializedCandidateHostsInfos; -// std::future mUnInitializedHostProbeThreadRes; -// mutable std::mutex mUnInitializedHostProbeThreadRunningMux; -// bool mIsUnInitializedHostProbeThreadRunning = true; + // std::future mUnInitializedHostProbeThreadRes; + // mutable std::mutex mUnInitializedHostProbeThreadRunningMux; + // bool mIsUnInitializedHostProbeThreadRunning = true; -// CURLM* mHostProbeClient = nullptr; -// mutable std::mutex mPartiallyInitializedCandidateHostsInfosMux; -// std::vector> mPartiallyInitializedCandidateHostsInfos; + // CURLM* mHostProbeClient = nullptr; + // mutable std::mutex mPartiallyInitializedCandidateHostsInfosMux; + // std::vector> mPartiallyInitializedCandidateHostsInfos; -// std::future mHostProbeThreadRes; -// mutable std::mutex mHostProbeThreadRunningMux; -// bool mIsHostProbeThreadRunning = true; -// #ifdef APSARA_UNIT_TEST_MAIN -// HttpResponse (*mDoProbeNetwork)(const std::unique_ptr& req) = nullptr; -// #endif + // std::future mHostProbeThreadRes; + // mutable std::mutex mHostProbeThreadRunningMux; + // bool mIsHostProbeThreadRunning = true; + // #ifdef APSARA_UNIT_TEST_MAIN + // HttpResponse (*mDoProbeNetwork)(const std::unique_ptr& req) = nullptr; + // #endif -// mutable std::mutex mRegionRealIpMux; -// std::map mRegionRealIpMap; -// mutable std::mutex mOutdatedRealIpRegionsMux; -// std::vector mOutdatedRealIpRegions; -// mutable std::mutex mRegionRealIpCandidateHostsInfosMapMux; -// std::map mRegionRealIpCandidateHostsInfosMap; -// bool (*mGetEndpointRealIp)(const std::string& endpoint, std::string& ip) = nullptr; + // mutable std::mutex mRegionRealIpMux; + // std::map mRegionRealIpMap; + // mutable std::mutex mOutdatedRealIpRegionsMux; + // std::vector mOutdatedRealIpRegions; + // mutable std::mutex mRegionRealIpCandidateHostsInfosMapMux; + // std::map mRegionRealIpCandidateHostsInfosMap; + // bool (*mGetEndpointRealIp)(const std::string& endpoint, std::string& ip) = nullptr; -// std::future mUpdateRealIpThreadRes; -// mutable std::mutex mUpdateRealIpThreadRunningMux; -// bool mIsUpdateRealIpThreadRunning = true; + // std::future mUpdateRealIpThreadRes; + // mutable std::mutex mUpdateRealIpThreadRunningMux; + // bool mIsUpdateRealIpThreadRunning = true; -// mutable std::condition_variable mStopCV; + // mutable std::condition_variable mStopCV; #ifdef APSARA_UNIT_TEST_MAIN friend class FlusherSLSUnittest; friend class SLSClientManagerUnittest; - friend class ProbeNetworkMock; - friend class GetRealIpMock; + // friend class ProbeNetworkMock; + // friend class GetRealIpMock; #endif }; diff --git a/core/unittest/flusher/FlusherSLSUnittest.cpp b/core/unittest/flusher/FlusherSLSUnittest.cpp index c8ad20ba19..a414f22987 100644 --- a/core/unittest/flusher/FlusherSLSUnittest.cpp +++ b/core/unittest/flusher/FlusherSLSUnittest.cpp @@ -30,6 +30,9 @@ #include "pipeline/queue/QueueKeyManager.h" #include "pipeline/queue/SLSSenderQueueItem.h" #include "pipeline/queue/SenderQueueManager.h" +#ifdef __ENTERPRISE__ +#include "plugin/flusher/sls/EnterpriseSLSClientManager.h" +#endif #include "plugin/flusher/sls/FlusherSLS.h" #include "plugin/flusher/sls/PackIdManager.h" #include "plugin/flusher/sls/SLSClientManager.h" @@ -240,8 +243,9 @@ void FlusherSLSUnittest::OnSuccessfulInit() { flusher->SetMetricsRecordRef(FlusherSLS::sName, "1"); APSARA_TEST_TRUE(flusher->Init(configJson, optionalGoPipeline)); APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, flusher->mEndpointMode); - APSARA_TEST_TRUE( - SLSClientManager::GetInstance()->mRegionCandidateEndpointsMap["cn-hangzhou"].mRemoteEndpoints.empty()); + APSARA_TEST_TRUE(EnterpriseSLSClientManager::GetInstance() + ->mRegionCandidateEndpointsMap["cn-hangzhou"] + .mRemoteEndpoints.empty()); APSARA_TEST_EQUAL(flusher->mProject, flusher->mCandidateHostsInfo->GetProject()); APSARA_TEST_EQUAL(flusher->mRegion, flusher->mCandidateHostsInfo->GetRegion()); APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, flusher->mCandidateHostsInfo->GetMode()); @@ -263,7 +267,8 @@ void FlusherSLSUnittest::OnSuccessfulInit() { flusher->SetMetricsRecordRef(FlusherSLS::sName, "1"); APSARA_TEST_TRUE(flusher->Init(configJson, optionalGoPipeline)); APSARA_TEST_EQUAL(EndpointMode::DEFAULT, flusher->mEndpointMode); - auto& endpoints = SLSClientManager::GetInstance()->mRegionCandidateEndpointsMap["cn-hangzhou"].mRemoteEndpoints; + auto& endpoints + = EnterpriseSLSClientManager::GetInstance()->mRegionCandidateEndpointsMap["cn-hangzhou"].mRemoteEndpoints; APSARA_TEST_EQUAL(1U, endpoints.size()); APSARA_TEST_EQUAL("cn-hangzhou.log.aliyuncs.com", endpoints[0]); APSARA_TEST_EQUAL(flusher->mProject, flusher->mCandidateHostsInfo->GetProject()); @@ -287,7 +292,8 @@ void FlusherSLSUnittest::OnSuccessfulInit() { flusher->SetMetricsRecordRef(FlusherSLS::sName, "1"); APSARA_TEST_TRUE(flusher->Init(configJson, optionalGoPipeline)); APSARA_TEST_EQUAL(EndpointMode::DEFAULT, flusher->mEndpointMode); - auto& endpoints = SLSClientManager::GetInstance()->mRegionCandidateEndpointsMap["cn-hangzhou"].mRemoteEndpoints; + auto& endpoints + = EnterpriseSLSClientManager::GetInstance()->mRegionCandidateEndpointsMap["cn-hangzhou"].mRemoteEndpoints; APSARA_TEST_EQUAL(1U, endpoints.size()); APSARA_TEST_EQUAL("cn-hangzhou.log.aliyuncs.com", endpoints[0]); APSARA_TEST_EQUAL(flusher->mProject, flusher->mCandidateHostsInfo->GetProject()); diff --git a/core/unittest/flusher/SLSClientManagerUnittest.cpp b/core/unittest/flusher/SLSClientManagerUnittest.cpp index bd2ee85718..e4c5b6df8f 100644 --- a/core/unittest/flusher/SLSClientManagerUnittest.cpp +++ b/core/unittest/flusher/SLSClientManagerUnittest.cpp @@ -12,16 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "app_config/AppConfig.h" +// #include "app_config/AppConfig.h" #include "plugin/flusher/sls/SLSClientManager.h" #include "unittest/Unittest.h" DECLARE_FLAG_STRING(default_access_key_id); DECLARE_FLAG_STRING(default_access_key); -DECLARE_FLAG_INT32(sls_hosts_probe_interval_sec); -DECLARE_FLAG_INT32(sls_all_hosts_probe_interval_sec); -DECLARE_FLAG_BOOL(send_prefer_real_ip); -DECLARE_FLAG_INT32(send_switch_real_ip_interval); +// DECLARE_FLAG_INT32(sls_hosts_probe_interval_sec); +// DECLARE_FLAG_INT32(sls_all_hosts_probe_interval_sec); +// DECLARE_FLAG_BOOL(send_prefer_real_ip); +// DECLARE_FLAG_INT32(send_switch_real_ip_interval); using namespace std; @@ -234,7 +234,7 @@ void CandidateHostsInfoUnittest::TestUpdateHosts() { const string publicEndpoint = region + ".log.aliyuncs.com"; const string privateEndpoint = region + "-intranet.log.aliyuncs.com"; const string internalEndpoint = region + "-internal.log.aliyuncs.com"; - const string globalEndpoint = "log.global.aliyuncs.com"; + const string globalEndpoint = "log-global.aliyuncs.com"; const string customEndpoint = "custom.endpoint"; const string publicHost = mProject + "." + publicEndpoint; const string privateHost = mProject + "." + privateEndpoint; @@ -434,896 +434,896 @@ UNIT_TEST_CASE(CandidateHostsInfoUnittest, TestHostsInfo) UNIT_TEST_CASE(CandidateHostsInfoUnittest, TestUpdateHosts) UNIT_TEST_CASE(CandidateHostsInfoUnittest, TestFirstHost) -class ProbeNetworkMock { -public: - static HttpResponse DoProbeNetwork(const std::unique_ptr& req) { - HttpResponse response; - chrono::milliseconds latency = chrono::milliseconds::max(); - if (!enableUnavailableHosts) { - if (req->mProject == project1 && req->mHost == project1 + "." + publicEndpoint1) { - static uint32_t sec = 1; - latency = chrono::milliseconds(sec++); - } else if (req->mProject == project2) { - if (req->mHost == project2 + "." + publicEndpoint2) { - static uint32_t sec = 10; - latency = chrono::milliseconds(sec++); - } else if (req->mHost == project2 + "." + globalEndpoint) { - static uint32_t sec = 5; - latency = chrono::milliseconds(sec++); - } - } else if (req->mProject == project3 && req->mHost == project3 + "." + customEndpoint) { - static uint32_t sec = 20; - latency = chrono::milliseconds(sec++); - } - } else { - static uint32_t sec = 200; - latency = chrono::milliseconds(sec++); - } - - response.SetResponseTime(latency); - return response; - } - - static void Prepare() { - publicEndpoint1 = region1 + ".log.aliyuncs.com"; - privateEndpoint1 = region1 + "-intranet.log.aliyuncs.com"; - internalEndpoint1 = region1 + "-internal.log.aliyuncs.com"; - publicEndpoint2 = region2 + ".log.aliyuncs.com"; - privateEndpoint2 = region2 + "-intranet.log.aliyuncs.com"; - internalEndpoint2 = region2 + "-internal.log.aliyuncs.com"; - - // region_1: only public endpoint is available - // region_2: both accelerate and public endpoints are available - // region_3: only custom endpoint is available - // project_x belongs to region_x - SLSClientManager::GetInstance()->UpdateLocalRegionEndpointsAndHttpsInfo(region1, {publicEndpoint1}); - SLSClientManager::GetInstance()->UpdateLocalRegionEndpointsAndHttpsInfo(region2, {globalEndpoint}); - SLSClientManager::GetInstance()->UpdateLocalRegionEndpointsAndHttpsInfo(region3, {customEndpoint}); - SLSClientManager::GetInstance()->UpdateRemoteRegionEndpoints( - region1, {internalEndpoint1, privateEndpoint1, publicEndpoint1}); - SLSClientManager::GetInstance()->UpdateRemoteRegionEndpoints( - region2, {internalEndpoint2, privateEndpoint2, publicEndpoint2}); - } - - static bool enableUnavailableHosts; - - static string project1; - static string project2; - static string project3; - static string region1; - static string region2; - static string region3; - static string publicEndpoint1; - static string privateEndpoint1; - static string internalEndpoint1; - static string publicEndpoint2; - static string privateEndpoint2; - static string internalEndpoint2; - static string globalEndpoint; - static string customEndpoint; -}; - -bool ProbeNetworkMock::enableUnavailableHosts = false; -string ProbeNetworkMock::region1 = "region_1"; -string ProbeNetworkMock::region2 = "region_2"; -string ProbeNetworkMock::region3 = "region_3"; -string ProbeNetworkMock::project1 = "project_1"; -string ProbeNetworkMock::project2 = "project_2"; -string ProbeNetworkMock::project3 = "project_3"; -string ProbeNetworkMock::publicEndpoint1; -string ProbeNetworkMock::privateEndpoint1; -string ProbeNetworkMock::internalEndpoint1; -string ProbeNetworkMock::publicEndpoint2; -string ProbeNetworkMock::privateEndpoint2; -string ProbeNetworkMock::internalEndpoint2; -string ProbeNetworkMock::globalEndpoint = "log.global.aliyuncs.com"; -string ProbeNetworkMock::customEndpoint = "custom.endpoint"; - -class GetRealIpMock { -public: - static bool GetEndpointRealIpMock(const string& endpoint, string& ip) { - if (endpoint.find(normalRegion) != string::npos) { - static size_t i = 0; - ip = normalRegionIps[i]; - i = (i + 1) % normalRegionIps.size(); - return true; - } else if (endpoint.find(unavailableRegion) != string::npos) { - static size_t i = 0; - ip = unavailableRegionIps[i]; - i = (i + 1) % unavailableRegionIps.size(); - return true; - } else { - return false; - } - } - - static HttpResponse DoProbeNetwork(const std::unique_ptr& req) { - HttpResponse response; - response.SetResponseTime(chrono::milliseconds(100)); - return response; - } - - static string normalRegion; - static string unavailableRegion; - static vector normalRegionIps; - static vector unavailableRegionIps; -}; - -string GetRealIpMock::normalRegion = "region_1"; -string GetRealIpMock::unavailableRegion = "region_2"; -vector GetRealIpMock::normalRegionIps = {"192.168.0.1", "192.168.0.2"}; -vector GetRealIpMock::unavailableRegionIps = {"10.0.0.1", "10.0.0.2"}; +// class ProbeNetworkMock { +// public: +// static HttpResponse DoProbeNetwork(const std::unique_ptr& req) { +// HttpResponse response; +// chrono::milliseconds latency = chrono::milliseconds::max(); +// if (!enableUnavailableHosts) { +// if (req->mProject == project1 && req->mHost == project1 + "." + publicEndpoint1) { +// static uint32_t sec = 1; +// latency = chrono::milliseconds(sec++); +// } else if (req->mProject == project2) { +// if (req->mHost == project2 + "." + publicEndpoint2) { +// static uint32_t sec = 10; +// latency = chrono::milliseconds(sec++); +// } else if (req->mHost == project2 + "." + globalEndpoint) { +// static uint32_t sec = 5; +// latency = chrono::milliseconds(sec++); +// } +// } else if (req->mProject == project3 && req->mHost == project3 + "." + customEndpoint) { +// static uint32_t sec = 20; +// latency = chrono::milliseconds(sec++); +// } +// } else { +// static uint32_t sec = 200; +// latency = chrono::milliseconds(sec++); +// } + +// response.SetResponseTime(latency); +// return response; +// } + +// static void Prepare() { +// publicEndpoint1 = region1 + ".log.aliyuncs.com"; +// privateEndpoint1 = region1 + "-intranet.log.aliyuncs.com"; +// internalEndpoint1 = region1 + "-internal.log.aliyuncs.com"; +// publicEndpoint2 = region2 + ".log.aliyuncs.com"; +// privateEndpoint2 = region2 + "-intranet.log.aliyuncs.com"; +// internalEndpoint2 = region2 + "-internal.log.aliyuncs.com"; + +// // region_1: only public endpoint is available +// // region_2: both accelerate and public endpoints are available +// // region_3: only custom endpoint is available +// // project_x belongs to region_x +// SLSClientManager::GetInstance()->UpdateLocalRegionEndpointsAndHttpsInfo(region1, {publicEndpoint1}); +// SLSClientManager::GetInstance()->UpdateLocalRegionEndpointsAndHttpsInfo(region2, {globalEndpoint}); +// SLSClientManager::GetInstance()->UpdateLocalRegionEndpointsAndHttpsInfo(region3, {customEndpoint}); +// SLSClientManager::GetInstance()->UpdateRemoteRegionEndpoints( +// region1, {internalEndpoint1, privateEndpoint1, publicEndpoint1}); +// SLSClientManager::GetInstance()->UpdateRemoteRegionEndpoints( +// region2, {internalEndpoint2, privateEndpoint2, publicEndpoint2}); +// } + +// static bool enableUnavailableHosts; + +// static string project1; +// static string project2; +// static string project3; +// static string region1; +// static string region2; +// static string region3; +// static string publicEndpoint1; +// static string privateEndpoint1; +// static string internalEndpoint1; +// static string publicEndpoint2; +// static string privateEndpoint2; +// static string internalEndpoint2; +// static string globalEndpoint; +// static string customEndpoint; +// }; + +// bool ProbeNetworkMock::enableUnavailableHosts = false; +// string ProbeNetworkMock::region1 = "region_1"; +// string ProbeNetworkMock::region2 = "region_2"; +// string ProbeNetworkMock::region3 = "region_3"; +// string ProbeNetworkMock::project1 = "project_1"; +// string ProbeNetworkMock::project2 = "project_2"; +// string ProbeNetworkMock::project3 = "project_3"; +// string ProbeNetworkMock::publicEndpoint1; +// string ProbeNetworkMock::privateEndpoint1; +// string ProbeNetworkMock::internalEndpoint1; +// string ProbeNetworkMock::publicEndpoint2; +// string ProbeNetworkMock::privateEndpoint2; +// string ProbeNetworkMock::internalEndpoint2; +// string ProbeNetworkMock::globalEndpoint = "log-global.aliyuncs.com"; +// string ProbeNetworkMock::customEndpoint = "custom.endpoint"; + +// class GetRealIpMock { +// public: +// static bool GetEndpointRealIpMock(const string& endpoint, string& ip) { +// if (endpoint.find(normalRegion) != string::npos) { +// static size_t i = 0; +// ip = normalRegionIps[i]; +// i = (i + 1) % normalRegionIps.size(); +// return true; +// } else if (endpoint.find(unavailableRegion) != string::npos) { +// static size_t i = 0; +// ip = unavailableRegionIps[i]; +// i = (i + 1) % unavailableRegionIps.size(); +// return true; +// } else { +// return false; +// } +// } + +// static HttpResponse DoProbeNetwork(const std::unique_ptr& req) { +// HttpResponse response; +// response.SetResponseTime(chrono::milliseconds(100)); +// return response; +// } + +// static string normalRegion; +// static string unavailableRegion; +// static vector normalRegionIps; +// static vector unavailableRegionIps; +// }; + +// string GetRealIpMock::normalRegion = "region_1"; +// string GetRealIpMock::unavailableRegion = "region_2"; +// vector GetRealIpMock::normalRegionIps = {"192.168.0.1", "192.168.0.2"}; +// vector GetRealIpMock::unavailableRegionIps = {"10.0.0.1", "10.0.0.2"}; class SLSClientManagerUnittest : public ::testing::Test { public: - void TestLocalRegionEndpoints(); - void TestRemoteRegionEndpoints(); - void TestGetCandidateHostsInfo(); - void TestUpdateHostInfo(); - void TestUsingHttps(); - void TestProbeNetwork(); - void TestRealIp(); + // void TestLocalRegionEndpoints(); + // void TestRemoteRegionEndpoints(); + // void TestGetCandidateHostsInfo(); + // void TestUpdateHostInfo(); + // void TestUsingHttps(); + // void TestProbeNetwork(); + // void TestRealIp(); void TestAccessKeyManagement(); - void TestGetCandidateHostsInfoOpen(); + void TestGetCandidateHostsInfo(); protected: - static void SetUpTestCase() { - SLSClientManager::GetInstance()->mDoProbeNetwork = ProbeNetworkMock::DoProbeNetwork; - SLSClientManager::GetInstance()->mGetEndpointRealIp = GetRealIpMock::GetEndpointRealIpMock; - } + // static void SetUpTestCase() { + // SLSClientManager::GetInstance()->mDoProbeNetwork = ProbeNetworkMock::DoProbeNetwork; + // SLSClientManager::GetInstance()->mGetEndpointRealIp = GetRealIpMock::GetEndpointRealIpMock; + // } - void TearDown() override { mManager->Clear(); } + // void TearDown() override { mManager->Clear(); } private: SLSClientManager* mManager = SLSClientManager::GetInstance(); }; -void SLSClientManagerUnittest::TestLocalRegionEndpoints() { - const string region = "region"; - { - // default - const string endpoint = region + ".log.aliyuncs.com"; - mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint}); - - auto& item = mManager->mRegionCandidateEndpointsMap[region]; - APSARA_TEST_EQUAL(EndpointMode::DEFAULT, item.mMode); - APSARA_TEST_EQUAL(1U, item.mLocalEndpoints.size()); - APSARA_TEST_EQUAL(endpoint, item.mLocalEndpoints[0]); - APSARA_TEST_FALSE(mManager->UsingHttps(region)); - } - { - // accelerate (+ default) - const string endpoint1 = region + "-intranet.log.aliyuncs.com"; - const string endpoint2 = "log.global.aliyuncs.com"; - mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); - - auto& item = mManager->mRegionCandidateEndpointsMap[region]; - APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, item.mMode); - APSARA_TEST_EQUAL(0U, item.mLocalEndpoints.size()); - APSARA_TEST_FALSE(mManager->UsingHttps(region)); - } - { - // custom (+ default) - const string endpoint1 = region + "-intranet.log.aliyuncs.com"; - const string endpoint2 = "custom.endpoint"; - mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); - - auto& item = mManager->mRegionCandidateEndpointsMap[region]; - APSARA_TEST_EQUAL(EndpointMode::CUSTOM, item.mMode); - APSARA_TEST_EQUAL(1U, item.mLocalEndpoints.size()); - APSARA_TEST_EQUAL(endpoint2, item.mLocalEndpoints[0]); - APSARA_TEST_FALSE(mManager->UsingHttps(region)); - } - { - // accelerate + custom (+ default) - const string endpoint1 = "custom.endpoint"; - const string endpoint2 = region + "-intranet.log.aliyuncs.com"; - const string endpoint3 = "log.global.aliyuncs.com"; - mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2, endpoint3}); - - auto& item = mManager->mRegionCandidateEndpointsMap[region]; - APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, item.mMode); - APSARA_TEST_EQUAL(0U, item.mLocalEndpoints.size()); - APSARA_TEST_FALSE(mManager->UsingHttps(region)); - } - { - // http -> https -> https - const string endpoint1 = region + "-intranet.log.aliyuncs.com"; - const string endpoint2 = "https://log.global.aliyuncs.com"; - mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); - APSARA_TEST_TRUE(mManager->UsingHttps(region)); - - mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); - APSARA_TEST_TRUE(mManager->UsingHttps(region)); - } - { - // https -> http -> http - const string endpoint1 = region + "-intranet.log.aliyuncs.com"; - const string endpoint2 = "log.global.aliyuncs.com"; - mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); - APSARA_TEST_FALSE(mManager->UsingHttps(region)); - - mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); - APSARA_TEST_FALSE(mManager->UsingHttps(region)); - } -} - -void SLSClientManagerUnittest::TestRemoteRegionEndpoints() { - { - auto info1 = mManager->GetCandidateHostsInfo("region_1", "project_1", EndpointMode::DEFAULT); - auto info2 = mManager->GetCandidateHostsInfo("region_1", "project_2", EndpointMode::DEFAULT); - auto info3 = mManager->GetCandidateHostsInfo("region_2", "project_3", EndpointMode::DEFAULT); - APSARA_TEST_EQUAL(0U, info3->mCandidateHosts.size()); - - // create remote info - mManager->UpdateRemoteRegionEndpoints("region_1", {"endpoint_1", "endpoint_2"}); - APSARA_TEST_EQUAL(2U, info1->mCandidateHosts.size()); - APSARA_TEST_EQUAL(2U, info2->mCandidateHosts.size()); - APSARA_TEST_EQUAL(0U, info3->mCandidateHosts.size()); - APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[0].size()); - APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[1].size()); - APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[0].size()); - APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[1].size()); - APSARA_TEST_EQUAL("project_1.endpoint_1", info1->mCandidateHosts[0][0].GetHostname()); - APSARA_TEST_EQUAL("project_1.endpoint_2", info1->mCandidateHosts[1][0].GetHostname()); - APSARA_TEST_EQUAL("project_2.endpoint_1", info2->mCandidateHosts[0][0].GetHostname()); - APSARA_TEST_EQUAL("project_2.endpoint_2", info2->mCandidateHosts[1][0].GetHostname()); - - // update remote info with overwrite - mManager->UpdateRemoteRegionEndpoints("region_1", {"endpoint_2", "endpoint_3"}); - APSARA_TEST_EQUAL(2U, info1->mCandidateHosts.size()); - APSARA_TEST_EQUAL(2U, info2->mCandidateHosts.size()); - APSARA_TEST_EQUAL(0U, info3->mCandidateHosts.size()); - APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[0].size()); - APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[1].size()); - APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[0].size()); - APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[1].size()); - APSARA_TEST_EQUAL("project_1.endpoint_2", info1->mCandidateHosts[0][0].GetHostname()); - APSARA_TEST_EQUAL("project_1.endpoint_3", info1->mCandidateHosts[1][0].GetHostname()); - APSARA_TEST_EQUAL("project_2.endpoint_2", info2->mCandidateHosts[0][0].GetHostname()); - APSARA_TEST_EQUAL("project_2.endpoint_3", info2->mCandidateHosts[1][0].GetHostname()); - - // update remote info with create - mManager->UpdateRemoteRegionEndpoints( - "region_1", {"endpoint_4"}, SLSClientManager::RemoteEndpointUpdateAction::CREATE); - APSARA_TEST_EQUAL(2U, info1->mCandidateHosts.size()); - APSARA_TEST_EQUAL(2U, info2->mCandidateHosts.size()); - APSARA_TEST_EQUAL(0U, info3->mCandidateHosts.size()); - APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[0].size()); - APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[1].size()); - APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[0].size()); - APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[1].size()); - APSARA_TEST_EQUAL("project_1.endpoint_2", info1->mCandidateHosts[0][0].GetHostname()); - APSARA_TEST_EQUAL("project_1.endpoint_3", info1->mCandidateHosts[1][0].GetHostname()); - APSARA_TEST_EQUAL("project_2.endpoint_2", info2->mCandidateHosts[0][0].GetHostname()); - APSARA_TEST_EQUAL("project_2.endpoint_3", info2->mCandidateHosts[1][0].GetHostname()); - - // update remote info with append - mManager->UpdateRemoteRegionEndpoints( - "region_1", {"endpoint_3", "endpoint_1"}, SLSClientManager::RemoteEndpointUpdateAction::APPEND); - APSARA_TEST_EQUAL(3U, info1->mCandidateHosts.size()); - APSARA_TEST_EQUAL(3U, info2->mCandidateHosts.size()); - APSARA_TEST_EQUAL(0U, info3->mCandidateHosts.size()); - APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[0].size()); - APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[1].size()); - APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[2].size()); - APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[0].size()); - APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[1].size()); - APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[2].size()); - APSARA_TEST_EQUAL("project_1.endpoint_2", info1->mCandidateHosts[0][0].GetHostname()); - APSARA_TEST_EQUAL("project_1.endpoint_3", info1->mCandidateHosts[1][0].GetHostname()); - APSARA_TEST_EQUAL("project_1.endpoint_1", info1->mCandidateHosts[2][0].GetHostname()); - APSARA_TEST_EQUAL("project_2.endpoint_2", info2->mCandidateHosts[0][0].GetHostname()); - APSARA_TEST_EQUAL("project_2.endpoint_3", info2->mCandidateHosts[1][0].GetHostname()); - APSARA_TEST_EQUAL("project_2.endpoint_1", info2->mCandidateHosts[2][0].GetHostname()); - } - { - // get candidate host info after remote info is updated - auto info = mManager->GetCandidateHostsInfo("region_1", "project_1", EndpointMode::DEFAULT); - APSARA_TEST_EQUAL(3U, info->mCandidateHosts.size()); - APSARA_TEST_EQUAL(1U, info->mCandidateHosts[0].size()); - APSARA_TEST_EQUAL(1U, info->mCandidateHosts[1].size()); - APSARA_TEST_EQUAL(1U, info->mCandidateHosts[2].size()); - APSARA_TEST_EQUAL("project_1.endpoint_2", info->mCandidateHosts[0][0].GetHostname()); - APSARA_TEST_EQUAL("project_1.endpoint_3", info->mCandidateHosts[1][0].GetHostname()); - APSARA_TEST_EQUAL("project_1.endpoint_1", info->mCandidateHosts[2][0].GetHostname()); - } -} - -void SLSClientManagerUnittest::TestGetCandidateHostsInfo() { - const string region = "region"; - const string project = "project"; - const string publicEndpoint = region + ".log.aliyuncs.com"; - const string privateEndpoint = region + "-intranet.log.aliyuncs.com"; - const string internalEndpoint = region + "-internal.log.aliyuncs.com"; - const string globalEndpoint = "log.global.aliyuncs.com"; - const string customEndpoint = "custom.endpoint"; - const string publicHost = project + "." + publicEndpoint; - const string privateHost = project + "." + privateEndpoint; - const string internalHost = project + "." + internalEndpoint; - const string globalHost = project + "." + globalEndpoint; - const string customHost = project + "." + customEndpoint; - { - // no candidate host info && no region endpoints - { - // default mode - auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); - APSARA_TEST_EQUAL(project, info->GetProject()); - APSARA_TEST_EQUAL(region, info->GetRegion()); - APSARA_TEST_EQUAL(EndpointMode::DEFAULT, info->GetMode()); - APSARA_TEST_EQUAL(0U, info->mCandidateHosts.size()); - { - APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); - auto& infos = mManager->mProjectCandidateHostsInfosMap[project]; - APSARA_TEST_EQUAL(1U, infos.size()); - auto& weakInfo = *infos.begin(); - APSARA_TEST_FALSE(weakInfo.expired()); - APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); - } - { - APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap.size()); - auto& infos = mManager->mRegionCandidateHostsInfosMap[region]; - APSARA_TEST_EQUAL(1U, infos.size()); - auto& weakInfo = *infos.begin(); - APSARA_TEST_FALSE(weakInfo.expired()); - APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); - } - APSARA_TEST_EQUAL(1U, mManager->mUnInitializedCandidateHostsInfos.size()); - APSARA_TEST_EQUAL(info.get(), mManager->mUnInitializedCandidateHostsInfos[0].lock().get()); - } - { - // accelerate mode - auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); - APSARA_TEST_EQUAL(project, info->GetProject()); - APSARA_TEST_EQUAL(region, info->GetRegion()); - APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, info->GetMode()); - APSARA_TEST_EQUAL(1U, info->mCandidateHosts.size()); - APSARA_TEST_EQUAL(1U, info->mCandidateHosts[0].size()); - APSARA_TEST_EQUAL(globalHost, info->mCandidateHosts[0][0].GetHostname()); - APSARA_TEST_EQUAL(chrono::milliseconds::max(), info->mCandidateHosts[0][0].GetLatency()); - { - APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); - auto& infos = mManager->mProjectCandidateHostsInfosMap[project]; - APSARA_TEST_EQUAL(2U, infos.size()); - auto it = infos.begin(); - APSARA_TEST_TRUE(it->expired()); - ++it; - APSARA_TEST_FALSE(it->expired()); - APSARA_TEST_EQUAL(info.get(), it->lock().get()); - } - { - APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap.size()); - auto& infos = mManager->mRegionCandidateHostsInfosMap[region]; - APSARA_TEST_EQUAL(2U, infos.size()); - auto it = infos.begin(); - APSARA_TEST_TRUE(it->expired()); - ++it; - APSARA_TEST_FALSE(it->expired()); - APSARA_TEST_EQUAL(info.get(), it->lock().get()); - } - APSARA_TEST_EQUAL(2U, mManager->mUnInitializedCandidateHostsInfos.size()); - APSARA_TEST_EQUAL(info.get(), mManager->mUnInitializedCandidateHostsInfos[1].lock().get()); - } - mManager->Clear(); - } - { - // no candidate host info && with region endpoints && region endpoint mode is default - mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {privateEndpoint}); - { - // default mode - auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); - APSARA_TEST_EQUAL(project, info->GetProject()); - APSARA_TEST_EQUAL(region, info->GetRegion()); - APSARA_TEST_EQUAL(EndpointMode::DEFAULT, info->GetMode()); - APSARA_TEST_EQUAL(1U, info->mCandidateHosts.size()); - APSARA_TEST_EQUAL(1U, info->mCandidateHosts[0].size()); - APSARA_TEST_EQUAL(privateHost, info->mCandidateHosts[0][0].GetHostname()); - { - APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); - auto& infos = mManager->mProjectCandidateHostsInfosMap[project]; - APSARA_TEST_EQUAL(1U, infos.size()); - auto& weakInfo = *infos.begin(); - APSARA_TEST_FALSE(weakInfo.expired()); - APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); - } - { - APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap.size()); - auto& infos = mManager->mRegionCandidateHostsInfosMap[region]; - APSARA_TEST_EQUAL(1U, infos.size()); - auto& weakInfo = *infos.begin(); - APSARA_TEST_FALSE(weakInfo.expired()); - APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); - } - APSARA_TEST_EQUAL(1U, mManager->mUnInitializedCandidateHostsInfos.size()); - APSARA_TEST_EQUAL(info.get(), mManager->mUnInitializedCandidateHostsInfos[0].lock().get()); - } - { - // accelerate mode - auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); - APSARA_TEST_EQUAL(project, info->GetProject()); - APSARA_TEST_EQUAL(region, info->GetRegion()); - APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, info->GetMode()); - APSARA_TEST_EQUAL(1U, info->mCandidateHosts.size()); - APSARA_TEST_EQUAL(1U, info->mCandidateHosts[0].size()); - APSARA_TEST_EQUAL(globalHost, info->mCandidateHosts[0][0].GetHostname()); - APSARA_TEST_EQUAL(chrono::milliseconds::max(), info->mCandidateHosts[0][0].GetLatency()); - { - APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); - auto& infos = mManager->mProjectCandidateHostsInfosMap[project]; - APSARA_TEST_EQUAL(2U, infos.size()); - auto it = infos.begin(); - APSARA_TEST_TRUE(it->expired()); - ++it; - APSARA_TEST_FALSE(it->expired()); - APSARA_TEST_EQUAL(info.get(), it->lock().get()); - } - { - APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap.size()); - auto& infos = mManager->mRegionCandidateHostsInfosMap[region]; - APSARA_TEST_EQUAL(2U, infos.size()); - auto it = infos.begin(); - APSARA_TEST_TRUE(it->expired()); - ++it; - APSARA_TEST_FALSE(it->expired()); - APSARA_TEST_EQUAL(info.get(), it->lock().get()); - } - APSARA_TEST_EQUAL(2U, mManager->mUnInitializedCandidateHostsInfos.size()); - APSARA_TEST_EQUAL(info.get(), mManager->mUnInitializedCandidateHostsInfos[1].lock().get()); - } - mManager->Clear(); - } - { - // no candidate host info && with region endpoints && region endpoint mode is accelerate - mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {globalEndpoint}); - { - // default mode - auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); - APSARA_TEST_EQUAL(project, info->GetProject()); - APSARA_TEST_EQUAL(region, info->GetRegion()); - APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, info->GetMode()); - APSARA_TEST_EQUAL(1U, info->mCandidateHosts.size()); - APSARA_TEST_EQUAL(1U, info->mCandidateHosts[0].size()); - APSARA_TEST_EQUAL(globalHost, info->mCandidateHosts[0][0].GetHostname()); - APSARA_TEST_EQUAL(chrono::milliseconds::max(), info->mCandidateHosts[0][0].GetLatency()); - { - APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); - auto& infos = mManager->mProjectCandidateHostsInfosMap[project]; - APSARA_TEST_EQUAL(1U, infos.size()); - auto& weakInfo = *infos.begin(); - APSARA_TEST_FALSE(weakInfo.expired()); - APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); - } - { - APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap.size()); - auto& infos = mManager->mRegionCandidateHostsInfosMap[region]; - APSARA_TEST_EQUAL(1U, infos.size()); - auto& weakInfo = *infos.begin(); - APSARA_TEST_FALSE(weakInfo.expired()); - APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); - } - APSARA_TEST_EQUAL(1U, mManager->mUnInitializedCandidateHostsInfos.size()); - APSARA_TEST_EQUAL(info.get(), mManager->mUnInitializedCandidateHostsInfos[0].lock().get()); - } - { - // accelerate mode - auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); - APSARA_TEST_EQUAL(project, info->GetProject()); - APSARA_TEST_EQUAL(region, info->GetRegion()); - APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, info->GetMode()); - APSARA_TEST_EQUAL(1U, info->mCandidateHosts.size()); - APSARA_TEST_EQUAL(1U, info->mCandidateHosts[0].size()); - APSARA_TEST_EQUAL(globalHost, info->mCandidateHosts[0][0].GetHostname()); - APSARA_TEST_EQUAL(chrono::milliseconds::max(), info->mCandidateHosts[0][0].GetLatency()); - { - APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); - auto& infos = mManager->mProjectCandidateHostsInfosMap[project]; - APSARA_TEST_EQUAL(2U, infos.size()); - auto it = infos.begin(); - APSARA_TEST_TRUE(it->expired()); - ++it; - APSARA_TEST_FALSE(it->expired()); - APSARA_TEST_EQUAL(info.get(), it->lock().get()); - } - { - APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap.size()); - auto& infos = mManager->mRegionCandidateHostsInfosMap[region]; - APSARA_TEST_EQUAL(2U, infos.size()); - auto it = infos.begin(); - APSARA_TEST_TRUE(it->expired()); - ++it; - APSARA_TEST_FALSE(it->expired()); - APSARA_TEST_EQUAL(info.get(), it->lock().get()); - } - APSARA_TEST_EQUAL(2U, mManager->mUnInitializedCandidateHostsInfos.size()); - APSARA_TEST_EQUAL(info.get(), mManager->mUnInitializedCandidateHostsInfos[1].lock().get()); - } - mManager->Clear(); - } - { - // no candidate host info && with region endpoints && region endpoint mode is custom - mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {customEndpoint}); - { - // default mode - auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); - APSARA_TEST_EQUAL(project, info->GetProject()); - APSARA_TEST_EQUAL(region, info->GetRegion()); - APSARA_TEST_EQUAL(EndpointMode::CUSTOM, info->GetMode()); - APSARA_TEST_EQUAL(1U, info->mCandidateHosts.size()); - APSARA_TEST_EQUAL(1U, info->mCandidateHosts[0].size()); - APSARA_TEST_EQUAL(customHost, info->mCandidateHosts[0][0].GetHostname()); - APSARA_TEST_EQUAL(chrono::milliseconds::max(), info->mCandidateHosts[0][0].GetLatency()); - { - APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); - auto& infos = mManager->mProjectCandidateHostsInfosMap[project]; - APSARA_TEST_EQUAL(1U, infos.size()); - auto& weakInfo = *infos.begin(); - APSARA_TEST_FALSE(weakInfo.expired()); - APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); - } - { - APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap.size()); - auto& infos = mManager->mRegionCandidateHostsInfosMap[region]; - APSARA_TEST_EQUAL(1U, infos.size()); - auto& weakInfo = *infos.begin(); - APSARA_TEST_FALSE(weakInfo.expired()); - APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); - } - APSARA_TEST_EQUAL(1U, mManager->mUnInitializedCandidateHostsInfos.size()); - APSARA_TEST_EQUAL(info.get(), mManager->mUnInitializedCandidateHostsInfos[0].lock().get()); - } - { - // accelerate mode - auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); - APSARA_TEST_EQUAL(project, info->GetProject()); - APSARA_TEST_EQUAL(region, info->GetRegion()); - APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, info->GetMode()); - APSARA_TEST_EQUAL(1U, info->mCandidateHosts.size()); - APSARA_TEST_EQUAL(1U, info->mCandidateHosts[0].size()); - APSARA_TEST_EQUAL(globalHost, info->mCandidateHosts[0][0].GetHostname()); - APSARA_TEST_EQUAL(chrono::milliseconds::max(), info->mCandidateHosts[0][0].GetLatency()); - { - APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); - auto& infos = mManager->mProjectCandidateHostsInfosMap[project]; - APSARA_TEST_EQUAL(2U, infos.size()); - auto& weakInfo = *(++infos.begin()); - APSARA_TEST_FALSE(weakInfo.expired()); - APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); - } - { - APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap.size()); - auto& infos = mManager->mRegionCandidateHostsInfosMap[region]; - APSARA_TEST_EQUAL(2U, infos.size()); - auto& weakInfo = *(++infos.begin()); - APSARA_TEST_FALSE(weakInfo.expired()); - APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); - } - APSARA_TEST_EQUAL(2U, mManager->mUnInitializedCandidateHostsInfos.size()); - APSARA_TEST_EQUAL(info.get(), mManager->mUnInitializedCandidateHostsInfos[1].lock().get()); - } - mManager->Clear(); - } - { - // candidate host info exists && no region endpoints - auto info1 = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); - auto info2 = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); - APSARA_TEST_EQUAL(info1.get(), info2.get()); - APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); - APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap[project].size()); - APSARA_TEST_EQUAL(1U, mManager->mUnInitializedCandidateHostsInfos.size()); - APSARA_TEST_EQUAL(info1.get(), mManager->mUnInitializedCandidateHostsInfos[0].lock().get()); - - mManager->Clear(); - } - { - // candidate host info exists && with region endpoints && region endpoint mode is default - mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {privateEndpoint}); - auto infoDefault = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); - auto infoAcc = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); - APSARA_TEST_EQUAL(2U, mManager->mUnInitializedCandidateHostsInfos.size()); - APSARA_TEST_EQUAL(infoDefault.get(), mManager->mUnInitializedCandidateHostsInfos[0].lock().get()); - APSARA_TEST_EQUAL(infoAcc.get(), mManager->mUnInitializedCandidateHostsInfos[1].lock().get()); - { - // default mode - auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); - APSARA_TEST_EQUAL(infoDefault.get(), info.get()); - APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); - APSARA_TEST_EQUAL(2U, mManager->mProjectCandidateHostsInfosMap[project].size()); - } - { - // accelerate mode - auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); - APSARA_TEST_EQUAL(infoAcc.get(), info.get()); - APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); - APSARA_TEST_EQUAL(2U, mManager->mProjectCandidateHostsInfosMap[project].size()); - } - mManager->Clear(); - } - { - // candidate host info exists && with region endpoints && region endpoint mode is accelerate - mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {globalEndpoint}); - auto infoAcc = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); - APSARA_TEST_EQUAL(1U, mManager->mUnInitializedCandidateHostsInfos.size()); - APSARA_TEST_EQUAL(infoAcc.get(), mManager->mUnInitializedCandidateHostsInfos[0].lock().get()); - { - // default mode - auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); - APSARA_TEST_EQUAL(infoAcc.get(), info.get()); - APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); - APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap[project].size()); - } - { - // accelerate mode - auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); - APSARA_TEST_EQUAL(infoAcc.get(), info.get()); - APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); - APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap[project].size()); - } - mManager->Clear(); - } - { - // candidate host info exists && with region endpoints && region endpoint mode is custom - mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {customEndpoint}); - auto infoCustom = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); - auto infoAcc = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); - APSARA_TEST_EQUAL(2U, mManager->mUnInitializedCandidateHostsInfos.size()); - APSARA_TEST_EQUAL(infoCustom.get(), mManager->mUnInitializedCandidateHostsInfos[0].lock().get()); - APSARA_TEST_EQUAL(infoAcc.get(), mManager->mUnInitializedCandidateHostsInfos[1].lock().get()); - { - // default mode - auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); - APSARA_TEST_EQUAL(infoCustom.get(), info.get()); - APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); - APSARA_TEST_EQUAL(2U, mManager->mProjectCandidateHostsInfosMap[project].size()); - } - { - // accelerate mode - auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); - APSARA_TEST_EQUAL(infoAcc.get(), info.get()); - APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); - APSARA_TEST_EQUAL(2U, mManager->mProjectCandidateHostsInfosMap[project].size()); - } - mManager->Clear(); - } -} - -void SLSClientManagerUnittest::TestUpdateHostInfo() { - const string region = "region"; - const string project = "project"; - const EndpointMode mode = EndpointMode::DEFAULT; - const string endpoint1 = region + ".log.aliyuncs.com"; - const string endpoint2 = region + "-intranet.log.aliyuncs.com"; - const string host1 = project + "." + endpoint1; - const string host2 = project + "." + endpoint2; - mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); - { - auto info = mManager->GetCandidateHostsInfo(region, project, mode); - - APSARA_TEST_TRUE(mManager->UpdateHostInfo(project, mode, host1, chrono::milliseconds(100))); - APSARA_TEST_EQUAL(chrono::milliseconds(100), info->mCandidateHosts[0][0].GetLatency()); - } - - // expired info - APSARA_TEST_FALSE(mManager->UpdateHostInfo(project, mode, host1, chrono::milliseconds(50))); - - // unknown project - APSARA_TEST_FALSE(mManager->UpdateHostInfo("unknown_project", mode, host1, chrono::milliseconds(50))); -} - -void SLSClientManagerUnittest::TestUsingHttps() { - const string region = "region"; - const string host = region + ".log.aliyuncs.com"; - mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {host}); - APSARA_TEST_FALSE(mManager->UsingHttps(region)); - - AppConfig::GetInstance()->mSendDataPort = 443; - APSARA_TEST_TRUE(mManager->UsingHttps(region)); -} - -void SLSClientManagerUnittest::TestProbeNetwork() { - ProbeNetworkMock::Prepare(); - const string publicHost1 = ProbeNetworkMock::project1 + "." + ProbeNetworkMock::publicEndpoint1; - const string privateHost1 = ProbeNetworkMock::project1 + "." + ProbeNetworkMock::privateEndpoint1; - const string internalHost1 = ProbeNetworkMock::project1 + "." + ProbeNetworkMock::internalEndpoint1; - const string globalHost1 = ProbeNetworkMock::project1 + "." + ProbeNetworkMock::globalEndpoint; - const string publicHost2 = ProbeNetworkMock::project2 + "." + ProbeNetworkMock::publicEndpoint2; - const string globalHost2 = ProbeNetworkMock::project2 + "." + ProbeNetworkMock::globalEndpoint; - const string globalHost3 = ProbeNetworkMock::project3 + "." + ProbeNetworkMock::globalEndpoint; - const string customHost3 = ProbeNetworkMock::project3 + "." + ProbeNetworkMock::customEndpoint; - - // ! means not available - vector> infos; - infos.push_back(mManager->GetCandidateHostsInfo( - ProbeNetworkMock::region1, ProbeNetworkMock::project1, EndpointMode::DEFAULT)); // public !internal !private - infos.push_back(mManager->GetCandidateHostsInfo( - ProbeNetworkMock::region1, ProbeNetworkMock::project1, EndpointMode::ACCELERATE)); // !accelerate public - infos.push_back(mManager->GetCandidateHostsInfo( - ProbeNetworkMock::region2, ProbeNetworkMock::project2, EndpointMode::DEFAULT)); // accelerate public - infos.push_back(mManager->GetCandidateHostsInfo( - ProbeNetworkMock::region3, ProbeNetworkMock::project3, EndpointMode::DEFAULT)); // custom - infos.push_back(mManager->GetCandidateHostsInfo( - ProbeNetworkMock::region3, ProbeNetworkMock::project3, EndpointMode::ACCELERATE)); // !accelerate - - // probe uninitialized host - mManager->DoProbeUnInitializedHost(); - APSARA_TEST_TRUE(mManager->mUnInitializedCandidateHostsInfos.empty()); - APSARA_TEST_EQUAL(infos.size(), mManager->mPartiallyInitializedCandidateHostsInfos.size()); - for (size_t i = 0; i < infos.size(); ++i) { - APSARA_TEST_EQUAL(infos[i].get(), mManager->mPartiallyInitializedCandidateHostsInfos[i].lock().get()); - } - - APSARA_TEST_EQUAL(publicHost1, infos[0]->GetCurrentHost()); - APSARA_TEST_EQUAL("", infos[1]->GetCurrentHost()); - APSARA_TEST_EQUAL(globalHost2, infos[2]->GetCurrentHost()); - APSARA_TEST_EQUAL(customHost3, infos[3]->GetCurrentHost()); - APSARA_TEST_EQUAL("", infos[4]->GetCurrentHost()); - vector len = {1, 2, 1, 1, 1}; - for (size_t i = 0; i < infos.size(); ++i) { - vector res; - infos[i]->GetProbeHosts(res); - APSARA_TEST_EQUAL(len[i], res.size()); - } - - // probe partially initialized host - mManager->DoProbeHost(); - APSARA_TEST_TRUE(mManager->mPartiallyInitializedCandidateHostsInfos.empty()); - - APSARA_TEST_EQUAL(publicHost1, infos[0]->GetCurrentHost()); - APSARA_TEST_EQUAL(publicHost1, infos[1]->GetCurrentHost()); - APSARA_TEST_EQUAL(globalHost2, infos[2]->GetCurrentHost()); - APSARA_TEST_EQUAL(customHost3, infos[3]->GetCurrentHost()); - APSARA_TEST_EQUAL("", infos[4]->GetCurrentHost()); - len = {1, 1, 2, 1, 1}; - for (size_t i = 0; i < infos.size(); ++i) { - vector res; - infos[i]->GetProbeHosts(res); - APSARA_TEST_EQUAL(len[i], res.size()); - } - - vector> oldLatencies; - for (const auto& info : infos) { - auto& latency = oldLatencies.emplace_back(); - for (const auto& item : info->mCandidateHosts) { - for (const auto& entry : item) { - latency.push_back(entry.GetLatency()); - } - } - } - - // probe all avaialble hosts - INT32_FLAG(sls_hosts_probe_interval_sec) = 0; - mManager->DoProbeHost(); - - APSARA_TEST_EQUAL(publicHost1, infos[0]->GetCurrentHost()); - APSARA_TEST_EQUAL(publicHost1, infos[1]->GetCurrentHost()); - APSARA_TEST_EQUAL(globalHost2, infos[2]->GetCurrentHost()); - APSARA_TEST_EQUAL(customHost3, infos[3]->GetCurrentHost()); - APSARA_TEST_EQUAL("", infos[4]->GetCurrentHost()); - len = {1, 1, 2, 1, 1}; - for (size_t i = 0; i < infos.size(); ++i) { - vector res; - infos[i]->GetProbeHosts(res); - APSARA_TEST_EQUAL(len[i], res.size()); - } - - vector> newLatencies; - for (const auto& info : infos) { - auto& latency = newLatencies.emplace_back(); - for (const auto& item : info->mCandidateHosts) { - for (const auto& entry : item) { - latency.push_back(entry.GetLatency()); - } - } - } - - for (size_t i = 0; i < oldLatencies.size(); ++i) { - for (size_t j = 0; j < oldLatencies[i].size(); ++j) { - if (oldLatencies[i][j] != chrono::milliseconds::max()) { - APSARA_TEST_NOT_EQUAL(newLatencies[i][j], oldLatencies[i][j]); - } else { - APSARA_TEST_EQUAL(newLatencies[i][j], oldLatencies[i][j]); - } - } - } - INT32_FLAG(sls_hosts_probe_interval_sec) = 60; - - // probe all hosts - INT32_FLAG(sls_all_hosts_probe_interval_sec) = 0; - ProbeNetworkMock::enableUnavailableHosts = true; - mManager->DoProbeHost(); - - APSARA_TEST_EQUAL(publicHost1, infos[0]->GetCurrentHost()); - APSARA_TEST_EQUAL(globalHost1, infos[1]->GetCurrentHost()); - APSARA_TEST_EQUAL(globalHost2, infos[2]->GetCurrentHost()); - APSARA_TEST_EQUAL(customHost3, infos[3]->GetCurrentHost()); - APSARA_TEST_EQUAL(globalHost3, infos[4]->GetCurrentHost()); - - len = {3, 2, 2, 1, 1}; - for (size_t i = 0; i < infos.size(); ++i) { - vector res; - infos[i]->GetProbeHosts(res); - APSARA_TEST_EQUAL(len[i], res.size()); - } - ProbeNetworkMock::enableUnavailableHosts = false; - INT32_FLAG(sls_all_hosts_probe_interval_sec) = 300; - - // expired info - infos[2].reset(); - infos[4].reset(); - mManager->DoProbeHost(); - APSARA_TEST_EQUAL(2U, mManager->mProjectCandidateHostsInfosMap.size()); - APSARA_TEST_EQUAL(2U, mManager->mProjectCandidateHostsInfosMap[ProbeNetworkMock::project1].size()); - APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap[ProbeNetworkMock::project3].size()); - APSARA_TEST_EQUAL(2U, mManager->mRegionCandidateHostsInfosMap.size()); - APSARA_TEST_EQUAL(2U, mManager->mRegionCandidateHostsInfosMap[ProbeNetworkMock::region1].size()); - APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap[ProbeNetworkMock::region3].size()); -} - -void SLSClientManagerUnittest::TestRealIp() { - BOOL_FLAG(send_prefer_real_ip) = true; - mManager->mDoProbeNetwork = GetRealIpMock::DoProbeNetwork; - - const string unknownRegion = "unknown_region"; - const string normalRegionEndpoint1 = GetRealIpMock::normalRegion + "-intranet.log.aliyuncs.com"; - const string normalRegionEndpoint2 = GetRealIpMock::normalRegion + ".log.aliyuncs.com"; - const string unavailableRegionEndpoint = GetRealIpMock::unavailableRegion + ".log.aliyuncs.com"; - const string unknownRegionEndpoint = unknownRegion + ".log.aliyuncs.com"; - mManager->UpdateRemoteRegionEndpoints(GetRealIpMock::normalRegion, {normalRegionEndpoint1, normalRegionEndpoint2}); - mManager->UpdateRemoteRegionEndpoints(GetRealIpMock::unavailableRegion, {unavailableRegionEndpoint}); - mManager->UpdateRemoteRegionEndpoints(unknownRegion, {unknownRegionEndpoint}); - APSARA_TEST_EQUAL(3U, mManager->mRegionRealIpCandidateHostsInfosMap.size()); - { - vector hosts; - mManager->mRegionRealIpCandidateHostsInfosMap.at(GetRealIpMock::normalRegion).GetAllHosts(hosts); - APSARA_TEST_EQUAL(2U, hosts.size()); - APSARA_TEST_EQUAL(normalRegionEndpoint1, hosts[0]); - APSARA_TEST_EQUAL(normalRegionEndpoint2, hosts[1]); - } - { - vector hosts; - mManager->mRegionRealIpCandidateHostsInfosMap.at(GetRealIpMock::unavailableRegion).GetAllHosts(hosts); - APSARA_TEST_EQUAL(1U, hosts.size()); - APSARA_TEST_EQUAL(unavailableRegionEndpoint, hosts[0]); - } - { - vector hosts; - mManager->mRegionRealIpCandidateHostsInfosMap.at(unknownRegion).GetAllHosts(hosts); - APSARA_TEST_EQUAL(1U, hosts.size()); - APSARA_TEST_EQUAL(unknownRegionEndpoint, hosts[0]); - } - - // no endpoint available - mManager->DoUpdateRealIp(); - APSARA_TEST_EQUAL(0U, mManager->mRegionRealIpMap.size()); - - // probe host - INT32_FLAG(sls_all_hosts_probe_interval_sec) = 0; - mManager->DoProbeHost(); - INT32_FLAG(sls_all_hosts_probe_interval_sec) = 300; - for (const auto& item : mManager->mRegionRealIpCandidateHostsInfosMap) { - APSARA_TEST_NOT_EQUAL("", item.second.GetCurrentHost()); - } - - // update all regions - INT32_FLAG(send_switch_real_ip_interval) = 0; - mManager->UpdateOutdatedRealIpRegions(GetRealIpMock::unavailableRegion); - mManager->DoUpdateRealIp(); - APSARA_TEST_TRUE(mManager->mOutdatedRealIpRegions.empty()); - APSARA_TEST_EQUAL(2U, mManager->mRegionRealIpMap.size()); - APSARA_TEST_EQUAL(GetRealIpMock::normalRegionIps[0], mManager->GetRealIp(GetRealIpMock::normalRegion)); - APSARA_TEST_EQUAL(GetRealIpMock::unavailableRegionIps[0], mManager->GetRealIp(GetRealIpMock::unavailableRegion)); - INT32_FLAG(send_switch_real_ip_interval) = 60; - - // update only outdated regions - mManager->UpdateOutdatedRealIpRegions(GetRealIpMock::normalRegion); - mManager->DoUpdateRealIp(); - APSARA_TEST_TRUE(mManager->mOutdatedRealIpRegions.empty()); - APSARA_TEST_EQUAL(2U, mManager->mRegionRealIpMap.size()); - APSARA_TEST_EQUAL(GetRealIpMock::normalRegionIps[1], mManager->GetRealIp(GetRealIpMock::normalRegion)); - APSARA_TEST_EQUAL(GetRealIpMock::unavailableRegionIps[0], mManager->GetRealIp(GetRealIpMock::unavailableRegion)); - - mManager->mDoProbeNetwork = ProbeNetworkMock::DoProbeNetwork; - BOOL_FLAG(send_prefer_real_ip) = false; -} +// void SLSClientManagerUnittest::TestLocalRegionEndpoints() { +// const string region = "region"; +// { +// // default +// const string endpoint = region + ".log.aliyuncs.com"; +// mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint}); + +// auto& item = mManager->mRegionCandidateEndpointsMap[region]; +// APSARA_TEST_EQUAL(EndpointMode::DEFAULT, item.mMode); +// APSARA_TEST_EQUAL(1U, item.mLocalEndpoints.size()); +// APSARA_TEST_EQUAL(endpoint, item.mLocalEndpoints[0]); +// APSARA_TEST_FALSE(mManager->UsingHttps(region)); +// } +// { +// // accelerate (+ default) +// const string endpoint1 = region + "-intranet.log.aliyuncs.com"; +// const string endpoint2 = kAccelerationDataEndpoint; +// mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); + +// auto& item = mManager->mRegionCandidateEndpointsMap[region]; +// APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, item.mMode); +// APSARA_TEST_EQUAL(0U, item.mLocalEndpoints.size()); +// APSARA_TEST_FALSE(mManager->UsingHttps(region)); +// } +// { +// // custom (+ default) +// const string endpoint1 = region + "-intranet.log.aliyuncs.com"; +// const string endpoint2 = "custom.endpoint"; +// mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); + +// auto& item = mManager->mRegionCandidateEndpointsMap[region]; +// APSARA_TEST_EQUAL(EndpointMode::CUSTOM, item.mMode); +// APSARA_TEST_EQUAL(1U, item.mLocalEndpoints.size()); +// APSARA_TEST_EQUAL(endpoint2, item.mLocalEndpoints[0]); +// APSARA_TEST_FALSE(mManager->UsingHttps(region)); +// } +// { +// // accelerate + custom (+ default) +// const string endpoint1 = "custom.endpoint"; +// const string endpoint2 = region + "-intranet.log.aliyuncs.com"; +// const string endpoint3 = kAccelerationDataEndpoint; +// mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2, endpoint3}); + +// auto& item = mManager->mRegionCandidateEndpointsMap[region]; +// APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, item.mMode); +// APSARA_TEST_EQUAL(0U, item.mLocalEndpoints.size()); +// APSARA_TEST_FALSE(mManager->UsingHttps(region)); +// } +// { +// // http -> https -> https +// const string endpoint1 = region + "-intranet.log.aliyuncs.com"; +// const string endpoint2 = "https://" + kAccelerationDataEndpoint; +// mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); +// APSARA_TEST_TRUE(mManager->UsingHttps(region)); + +// mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); +// APSARA_TEST_TRUE(mManager->UsingHttps(region)); +// } +// { +// // https -> http -> http +// const string endpoint1 = region + "-intranet.log.aliyuncs.com"; +// const string endpoint2 = kAccelerationDataEndpoint; +// mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); +// APSARA_TEST_FALSE(mManager->UsingHttps(region)); + +// mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); +// APSARA_TEST_FALSE(mManager->UsingHttps(region)); +// } +// } + +// void SLSClientManagerUnittest::TestRemoteRegionEndpoints() { +// { +// auto info1 = mManager->GetCandidateHostsInfo("region_1", "project_1", EndpointMode::DEFAULT); +// auto info2 = mManager->GetCandidateHostsInfo("region_1", "project_2", EndpointMode::DEFAULT); +// auto info3 = mManager->GetCandidateHostsInfo("region_2", "project_3", EndpointMode::DEFAULT); +// APSARA_TEST_EQUAL(0U, info3->mCandidateHosts.size()); + +// // create remote info +// mManager->UpdateRemoteRegionEndpoints("region_1", {"endpoint_1", "endpoint_2"}); +// APSARA_TEST_EQUAL(2U, info1->mCandidateHosts.size()); +// APSARA_TEST_EQUAL(2U, info2->mCandidateHosts.size()); +// APSARA_TEST_EQUAL(0U, info3->mCandidateHosts.size()); +// APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[0].size()); +// APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[1].size()); +// APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[0].size()); +// APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[1].size()); +// APSARA_TEST_EQUAL("project_1.endpoint_1", info1->mCandidateHosts[0][0].GetHostname()); +// APSARA_TEST_EQUAL("project_1.endpoint_2", info1->mCandidateHosts[1][0].GetHostname()); +// APSARA_TEST_EQUAL("project_2.endpoint_1", info2->mCandidateHosts[0][0].GetHostname()); +// APSARA_TEST_EQUAL("project_2.endpoint_2", info2->mCandidateHosts[1][0].GetHostname()); + +// // update remote info with overwrite +// mManager->UpdateRemoteRegionEndpoints("region_1", {"endpoint_2", "endpoint_3"}); +// APSARA_TEST_EQUAL(2U, info1->mCandidateHosts.size()); +// APSARA_TEST_EQUAL(2U, info2->mCandidateHosts.size()); +// APSARA_TEST_EQUAL(0U, info3->mCandidateHosts.size()); +// APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[0].size()); +// APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[1].size()); +// APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[0].size()); +// APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[1].size()); +// APSARA_TEST_EQUAL("project_1.endpoint_2", info1->mCandidateHosts[0][0].GetHostname()); +// APSARA_TEST_EQUAL("project_1.endpoint_3", info1->mCandidateHosts[1][0].GetHostname()); +// APSARA_TEST_EQUAL("project_2.endpoint_2", info2->mCandidateHosts[0][0].GetHostname()); +// APSARA_TEST_EQUAL("project_2.endpoint_3", info2->mCandidateHosts[1][0].GetHostname()); + +// // update remote info with create +// mManager->UpdateRemoteRegionEndpoints( +// "region_1", {"endpoint_4"}, SLSClientManager::RemoteEndpointUpdateAction::CREATE); +// APSARA_TEST_EQUAL(2U, info1->mCandidateHosts.size()); +// APSARA_TEST_EQUAL(2U, info2->mCandidateHosts.size()); +// APSARA_TEST_EQUAL(0U, info3->mCandidateHosts.size()); +// APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[0].size()); +// APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[1].size()); +// APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[0].size()); +// APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[1].size()); +// APSARA_TEST_EQUAL("project_1.endpoint_2", info1->mCandidateHosts[0][0].GetHostname()); +// APSARA_TEST_EQUAL("project_1.endpoint_3", info1->mCandidateHosts[1][0].GetHostname()); +// APSARA_TEST_EQUAL("project_2.endpoint_2", info2->mCandidateHosts[0][0].GetHostname()); +// APSARA_TEST_EQUAL("project_2.endpoint_3", info2->mCandidateHosts[1][0].GetHostname()); + +// // update remote info with append +// mManager->UpdateRemoteRegionEndpoints( +// "region_1", {"endpoint_3", "endpoint_1"}, SLSClientManager::RemoteEndpointUpdateAction::APPEND); +// APSARA_TEST_EQUAL(3U, info1->mCandidateHosts.size()); +// APSARA_TEST_EQUAL(3U, info2->mCandidateHosts.size()); +// APSARA_TEST_EQUAL(0U, info3->mCandidateHosts.size()); +// APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[0].size()); +// APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[1].size()); +// APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[2].size()); +// APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[0].size()); +// APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[1].size()); +// APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[2].size()); +// APSARA_TEST_EQUAL("project_1.endpoint_2", info1->mCandidateHosts[0][0].GetHostname()); +// APSARA_TEST_EQUAL("project_1.endpoint_3", info1->mCandidateHosts[1][0].GetHostname()); +// APSARA_TEST_EQUAL("project_1.endpoint_1", info1->mCandidateHosts[2][0].GetHostname()); +// APSARA_TEST_EQUAL("project_2.endpoint_2", info2->mCandidateHosts[0][0].GetHostname()); +// APSARA_TEST_EQUAL("project_2.endpoint_3", info2->mCandidateHosts[1][0].GetHostname()); +// APSARA_TEST_EQUAL("project_2.endpoint_1", info2->mCandidateHosts[2][0].GetHostname()); +// } +// { +// // get candidate host info after remote info is updated +// auto info = mManager->GetCandidateHostsInfo("region_1", "project_1", EndpointMode::DEFAULT); +// APSARA_TEST_EQUAL(3U, info->mCandidateHosts.size()); +// APSARA_TEST_EQUAL(1U, info->mCandidateHosts[0].size()); +// APSARA_TEST_EQUAL(1U, info->mCandidateHosts[1].size()); +// APSARA_TEST_EQUAL(1U, info->mCandidateHosts[2].size()); +// APSARA_TEST_EQUAL("project_1.endpoint_2", info->mCandidateHosts[0][0].GetHostname()); +// APSARA_TEST_EQUAL("project_1.endpoint_3", info->mCandidateHosts[1][0].GetHostname()); +// APSARA_TEST_EQUAL("project_1.endpoint_1", info->mCandidateHosts[2][0].GetHostname()); +// } +// } + +// void SLSClientManagerUnittest::TestGetCandidateHostsInfo() { +// const string region = "region"; +// const string project = "project"; +// const string publicEndpoint = region + ".log.aliyuncs.com"; +// const string privateEndpoint = region + "-intranet.log.aliyuncs.com"; +// const string internalEndpoint = region + "-internal.log.aliyuncs.com"; +// const string globalEndpoint = kAccelerationDataEndpoint; +// const string customEndpoint = "custom.endpoint"; +// const string publicHost = project + "." + publicEndpoint; +// const string privateHost = project + "." + privateEndpoint; +// const string internalHost = project + "." + internalEndpoint; +// const string globalHost = project + "." + globalEndpoint; +// const string customHost = project + "." + customEndpoint; +// { +// // no candidate host info && no region endpoints +// { +// // default mode +// auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); +// APSARA_TEST_EQUAL(project, info->GetProject()); +// APSARA_TEST_EQUAL(region, info->GetRegion()); +// APSARA_TEST_EQUAL(EndpointMode::DEFAULT, info->GetMode()); +// APSARA_TEST_EQUAL(0U, info->mCandidateHosts.size()); +// { +// APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); +// auto& infos = mManager->mProjectCandidateHostsInfosMap[project]; +// APSARA_TEST_EQUAL(1U, infos.size()); +// auto& weakInfo = *infos.begin(); +// APSARA_TEST_FALSE(weakInfo.expired()); +// APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); +// } +// { +// APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap.size()); +// auto& infos = mManager->mRegionCandidateHostsInfosMap[region]; +// APSARA_TEST_EQUAL(1U, infos.size()); +// auto& weakInfo = *infos.begin(); +// APSARA_TEST_FALSE(weakInfo.expired()); +// APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); +// } +// APSARA_TEST_EQUAL(1U, mManager->mUnInitializedCandidateHostsInfos.size()); +// APSARA_TEST_EQUAL(info.get(), mManager->mUnInitializedCandidateHostsInfos[0].lock().get()); +// } +// { +// // accelerate mode +// auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); +// APSARA_TEST_EQUAL(project, info->GetProject()); +// APSARA_TEST_EQUAL(region, info->GetRegion()); +// APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, info->GetMode()); +// APSARA_TEST_EQUAL(1U, info->mCandidateHosts.size()); +// APSARA_TEST_EQUAL(1U, info->mCandidateHosts[0].size()); +// APSARA_TEST_EQUAL(globalHost, info->mCandidateHosts[0][0].GetHostname()); +// APSARA_TEST_EQUAL(chrono::milliseconds::max(), info->mCandidateHosts[0][0].GetLatency()); +// { +// APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); +// auto& infos = mManager->mProjectCandidateHostsInfosMap[project]; +// APSARA_TEST_EQUAL(2U, infos.size()); +// auto it = infos.begin(); +// APSARA_TEST_TRUE(it->expired()); +// ++it; +// APSARA_TEST_FALSE(it->expired()); +// APSARA_TEST_EQUAL(info.get(), it->lock().get()); +// } +// { +// APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap.size()); +// auto& infos = mManager->mRegionCandidateHostsInfosMap[region]; +// APSARA_TEST_EQUAL(2U, infos.size()); +// auto it = infos.begin(); +// APSARA_TEST_TRUE(it->expired()); +// ++it; +// APSARA_TEST_FALSE(it->expired()); +// APSARA_TEST_EQUAL(info.get(), it->lock().get()); +// } +// APSARA_TEST_EQUAL(2U, mManager->mUnInitializedCandidateHostsInfos.size()); +// APSARA_TEST_EQUAL(info.get(), mManager->mUnInitializedCandidateHostsInfos[1].lock().get()); +// } +// mManager->Clear(); +// } +// { +// // no candidate host info && with region endpoints && region endpoint mode is default +// mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {privateEndpoint}); +// { +// // default mode +// auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); +// APSARA_TEST_EQUAL(project, info->GetProject()); +// APSARA_TEST_EQUAL(region, info->GetRegion()); +// APSARA_TEST_EQUAL(EndpointMode::DEFAULT, info->GetMode()); +// APSARA_TEST_EQUAL(1U, info->mCandidateHosts.size()); +// APSARA_TEST_EQUAL(1U, info->mCandidateHosts[0].size()); +// APSARA_TEST_EQUAL(privateHost, info->mCandidateHosts[0][0].GetHostname()); +// { +// APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); +// auto& infos = mManager->mProjectCandidateHostsInfosMap[project]; +// APSARA_TEST_EQUAL(1U, infos.size()); +// auto& weakInfo = *infos.begin(); +// APSARA_TEST_FALSE(weakInfo.expired()); +// APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); +// } +// { +// APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap.size()); +// auto& infos = mManager->mRegionCandidateHostsInfosMap[region]; +// APSARA_TEST_EQUAL(1U, infos.size()); +// auto& weakInfo = *infos.begin(); +// APSARA_TEST_FALSE(weakInfo.expired()); +// APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); +// } +// APSARA_TEST_EQUAL(1U, mManager->mUnInitializedCandidateHostsInfos.size()); +// APSARA_TEST_EQUAL(info.get(), mManager->mUnInitializedCandidateHostsInfos[0].lock().get()); +// } +// { +// // accelerate mode +// auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); +// APSARA_TEST_EQUAL(project, info->GetProject()); +// APSARA_TEST_EQUAL(region, info->GetRegion()); +// APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, info->GetMode()); +// APSARA_TEST_EQUAL(1U, info->mCandidateHosts.size()); +// APSARA_TEST_EQUAL(1U, info->mCandidateHosts[0].size()); +// APSARA_TEST_EQUAL(globalHost, info->mCandidateHosts[0][0].GetHostname()); +// APSARA_TEST_EQUAL(chrono::milliseconds::max(), info->mCandidateHosts[0][0].GetLatency()); +// { +// APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); +// auto& infos = mManager->mProjectCandidateHostsInfosMap[project]; +// APSARA_TEST_EQUAL(2U, infos.size()); +// auto it = infos.begin(); +// APSARA_TEST_TRUE(it->expired()); +// ++it; +// APSARA_TEST_FALSE(it->expired()); +// APSARA_TEST_EQUAL(info.get(), it->lock().get()); +// } +// { +// APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap.size()); +// auto& infos = mManager->mRegionCandidateHostsInfosMap[region]; +// APSARA_TEST_EQUAL(2U, infos.size()); +// auto it = infos.begin(); +// APSARA_TEST_TRUE(it->expired()); +// ++it; +// APSARA_TEST_FALSE(it->expired()); +// APSARA_TEST_EQUAL(info.get(), it->lock().get()); +// } +// APSARA_TEST_EQUAL(2U, mManager->mUnInitializedCandidateHostsInfos.size()); +// APSARA_TEST_EQUAL(info.get(), mManager->mUnInitializedCandidateHostsInfos[1].lock().get()); +// } +// mManager->Clear(); +// } +// { +// // no candidate host info && with region endpoints && region endpoint mode is accelerate +// mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {globalEndpoint}); +// { +// // default mode +// auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); +// APSARA_TEST_EQUAL(project, info->GetProject()); +// APSARA_TEST_EQUAL(region, info->GetRegion()); +// APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, info->GetMode()); +// APSARA_TEST_EQUAL(1U, info->mCandidateHosts.size()); +// APSARA_TEST_EQUAL(1U, info->mCandidateHosts[0].size()); +// APSARA_TEST_EQUAL(globalHost, info->mCandidateHosts[0][0].GetHostname()); +// APSARA_TEST_EQUAL(chrono::milliseconds::max(), info->mCandidateHosts[0][0].GetLatency()); +// { +// APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); +// auto& infos = mManager->mProjectCandidateHostsInfosMap[project]; +// APSARA_TEST_EQUAL(1U, infos.size()); +// auto& weakInfo = *infos.begin(); +// APSARA_TEST_FALSE(weakInfo.expired()); +// APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); +// } +// { +// APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap.size()); +// auto& infos = mManager->mRegionCandidateHostsInfosMap[region]; +// APSARA_TEST_EQUAL(1U, infos.size()); +// auto& weakInfo = *infos.begin(); +// APSARA_TEST_FALSE(weakInfo.expired()); +// APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); +// } +// APSARA_TEST_EQUAL(1U, mManager->mUnInitializedCandidateHostsInfos.size()); +// APSARA_TEST_EQUAL(info.get(), mManager->mUnInitializedCandidateHostsInfos[0].lock().get()); +// } +// { +// // accelerate mode +// auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); +// APSARA_TEST_EQUAL(project, info->GetProject()); +// APSARA_TEST_EQUAL(region, info->GetRegion()); +// APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, info->GetMode()); +// APSARA_TEST_EQUAL(1U, info->mCandidateHosts.size()); +// APSARA_TEST_EQUAL(1U, info->mCandidateHosts[0].size()); +// APSARA_TEST_EQUAL(globalHost, info->mCandidateHosts[0][0].GetHostname()); +// APSARA_TEST_EQUAL(chrono::milliseconds::max(), info->mCandidateHosts[0][0].GetLatency()); +// { +// APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); +// auto& infos = mManager->mProjectCandidateHostsInfosMap[project]; +// APSARA_TEST_EQUAL(2U, infos.size()); +// auto it = infos.begin(); +// APSARA_TEST_TRUE(it->expired()); +// ++it; +// APSARA_TEST_FALSE(it->expired()); +// APSARA_TEST_EQUAL(info.get(), it->lock().get()); +// } +// { +// APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap.size()); +// auto& infos = mManager->mRegionCandidateHostsInfosMap[region]; +// APSARA_TEST_EQUAL(2U, infos.size()); +// auto it = infos.begin(); +// APSARA_TEST_TRUE(it->expired()); +// ++it; +// APSARA_TEST_FALSE(it->expired()); +// APSARA_TEST_EQUAL(info.get(), it->lock().get()); +// } +// APSARA_TEST_EQUAL(2U, mManager->mUnInitializedCandidateHostsInfos.size()); +// APSARA_TEST_EQUAL(info.get(), mManager->mUnInitializedCandidateHostsInfos[1].lock().get()); +// } +// mManager->Clear(); +// } +// { +// // no candidate host info && with region endpoints && region endpoint mode is custom +// mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {customEndpoint}); +// { +// // default mode +// auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); +// APSARA_TEST_EQUAL(project, info->GetProject()); +// APSARA_TEST_EQUAL(region, info->GetRegion()); +// APSARA_TEST_EQUAL(EndpointMode::CUSTOM, info->GetMode()); +// APSARA_TEST_EQUAL(1U, info->mCandidateHosts.size()); +// APSARA_TEST_EQUAL(1U, info->mCandidateHosts[0].size()); +// APSARA_TEST_EQUAL(customHost, info->mCandidateHosts[0][0].GetHostname()); +// APSARA_TEST_EQUAL(chrono::milliseconds::max(), info->mCandidateHosts[0][0].GetLatency()); +// { +// APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); +// auto& infos = mManager->mProjectCandidateHostsInfosMap[project]; +// APSARA_TEST_EQUAL(1U, infos.size()); +// auto& weakInfo = *infos.begin(); +// APSARA_TEST_FALSE(weakInfo.expired()); +// APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); +// } +// { +// APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap.size()); +// auto& infos = mManager->mRegionCandidateHostsInfosMap[region]; +// APSARA_TEST_EQUAL(1U, infos.size()); +// auto& weakInfo = *infos.begin(); +// APSARA_TEST_FALSE(weakInfo.expired()); +// APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); +// } +// APSARA_TEST_EQUAL(1U, mManager->mUnInitializedCandidateHostsInfos.size()); +// APSARA_TEST_EQUAL(info.get(), mManager->mUnInitializedCandidateHostsInfos[0].lock().get()); +// } +// { +// // accelerate mode +// auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); +// APSARA_TEST_EQUAL(project, info->GetProject()); +// APSARA_TEST_EQUAL(region, info->GetRegion()); +// APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, info->GetMode()); +// APSARA_TEST_EQUAL(1U, info->mCandidateHosts.size()); +// APSARA_TEST_EQUAL(1U, info->mCandidateHosts[0].size()); +// APSARA_TEST_EQUAL(globalHost, info->mCandidateHosts[0][0].GetHostname()); +// APSARA_TEST_EQUAL(chrono::milliseconds::max(), info->mCandidateHosts[0][0].GetLatency()); +// { +// APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); +// auto& infos = mManager->mProjectCandidateHostsInfosMap[project]; +// APSARA_TEST_EQUAL(2U, infos.size()); +// auto& weakInfo = *(++infos.begin()); +// APSARA_TEST_FALSE(weakInfo.expired()); +// APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); +// } +// { +// APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap.size()); +// auto& infos = mManager->mRegionCandidateHostsInfosMap[region]; +// APSARA_TEST_EQUAL(2U, infos.size()); +// auto& weakInfo = *(++infos.begin()); +// APSARA_TEST_FALSE(weakInfo.expired()); +// APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); +// } +// APSARA_TEST_EQUAL(2U, mManager->mUnInitializedCandidateHostsInfos.size()); +// APSARA_TEST_EQUAL(info.get(), mManager->mUnInitializedCandidateHostsInfos[1].lock().get()); +// } +// mManager->Clear(); +// } +// { +// // candidate host info exists && no region endpoints +// auto info1 = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); +// auto info2 = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); +// APSARA_TEST_EQUAL(info1.get(), info2.get()); +// APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); +// APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap[project].size()); +// APSARA_TEST_EQUAL(1U, mManager->mUnInitializedCandidateHostsInfos.size()); +// APSARA_TEST_EQUAL(info1.get(), mManager->mUnInitializedCandidateHostsInfos[0].lock().get()); + +// mManager->Clear(); +// } +// { +// // candidate host info exists && with region endpoints && region endpoint mode is default +// mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {privateEndpoint}); +// auto infoDefault = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); +// auto infoAcc = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); +// APSARA_TEST_EQUAL(2U, mManager->mUnInitializedCandidateHostsInfos.size()); +// APSARA_TEST_EQUAL(infoDefault.get(), mManager->mUnInitializedCandidateHostsInfos[0].lock().get()); +// APSARA_TEST_EQUAL(infoAcc.get(), mManager->mUnInitializedCandidateHostsInfos[1].lock().get()); +// { +// // default mode +// auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); +// APSARA_TEST_EQUAL(infoDefault.get(), info.get()); +// APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); +// APSARA_TEST_EQUAL(2U, mManager->mProjectCandidateHostsInfosMap[project].size()); +// } +// { +// // accelerate mode +// auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); +// APSARA_TEST_EQUAL(infoAcc.get(), info.get()); +// APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); +// APSARA_TEST_EQUAL(2U, mManager->mProjectCandidateHostsInfosMap[project].size()); +// } +// mManager->Clear(); +// } +// { +// // candidate host info exists && with region endpoints && region endpoint mode is accelerate +// mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {globalEndpoint}); +// auto infoAcc = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); +// APSARA_TEST_EQUAL(1U, mManager->mUnInitializedCandidateHostsInfos.size()); +// APSARA_TEST_EQUAL(infoAcc.get(), mManager->mUnInitializedCandidateHostsInfos[0].lock().get()); +// { +// // default mode +// auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); +// APSARA_TEST_EQUAL(infoAcc.get(), info.get()); +// APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); +// APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap[project].size()); +// } +// { +// // accelerate mode +// auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); +// APSARA_TEST_EQUAL(infoAcc.get(), info.get()); +// APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); +// APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap[project].size()); +// } +// mManager->Clear(); +// } +// { +// // candidate host info exists && with region endpoints && region endpoint mode is custom +// mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {customEndpoint}); +// auto infoCustom = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); +// auto infoAcc = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); +// APSARA_TEST_EQUAL(2U, mManager->mUnInitializedCandidateHostsInfos.size()); +// APSARA_TEST_EQUAL(infoCustom.get(), mManager->mUnInitializedCandidateHostsInfos[0].lock().get()); +// APSARA_TEST_EQUAL(infoAcc.get(), mManager->mUnInitializedCandidateHostsInfos[1].lock().get()); +// { +// // default mode +// auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); +// APSARA_TEST_EQUAL(infoCustom.get(), info.get()); +// APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); +// APSARA_TEST_EQUAL(2U, mManager->mProjectCandidateHostsInfosMap[project].size()); +// } +// { +// // accelerate mode +// auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); +// APSARA_TEST_EQUAL(infoAcc.get(), info.get()); +// APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); +// APSARA_TEST_EQUAL(2U, mManager->mProjectCandidateHostsInfosMap[project].size()); +// } +// mManager->Clear(); +// } +// } + +// void SLSClientManagerUnittest::TestUpdateHostInfo() { +// const string region = "region"; +// const string project = "project"; +// const EndpointMode mode = EndpointMode::DEFAULT; +// const string endpoint1 = region + ".log.aliyuncs.com"; +// const string endpoint2 = region + "-intranet.log.aliyuncs.com"; +// const string host1 = project + "." + endpoint1; +// const string host2 = project + "." + endpoint2; +// mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); +// { +// auto info = mManager->GetCandidateHostsInfo(region, project, mode); + +// APSARA_TEST_TRUE(mManager->UpdateHostInfo(project, mode, host1, chrono::milliseconds(100))); +// APSARA_TEST_EQUAL(chrono::milliseconds(100), info->mCandidateHosts[0][0].GetLatency()); +// } + +// // expired info +// APSARA_TEST_FALSE(mManager->UpdateHostInfo(project, mode, host1, chrono::milliseconds(50))); + +// // unknown project +// APSARA_TEST_FALSE(mManager->UpdateHostInfo("unknown_project", mode, host1, chrono::milliseconds(50))); +// } + +// void SLSClientManagerUnittest::TestUsingHttps() { +// const string region = "region"; +// const string host = region + ".log.aliyuncs.com"; +// mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {host}); +// APSARA_TEST_FALSE(mManager->UsingHttps(region)); + +// AppConfig::GetInstance()->mSendDataPort = 443; +// APSARA_TEST_TRUE(mManager->UsingHttps(region)); +// } + +// void SLSClientManagerUnittest::TestProbeNetwork() { +// ProbeNetworkMock::Prepare(); +// const string publicHost1 = ProbeNetworkMock::project1 + "." + ProbeNetworkMock::publicEndpoint1; +// const string privateHost1 = ProbeNetworkMock::project1 + "." + ProbeNetworkMock::privateEndpoint1; +// const string internalHost1 = ProbeNetworkMock::project1 + "." + ProbeNetworkMock::internalEndpoint1; +// const string globalHost1 = ProbeNetworkMock::project1 + "." + ProbeNetworkMock::globalEndpoint; +// const string publicHost2 = ProbeNetworkMock::project2 + "." + ProbeNetworkMock::publicEndpoint2; +// const string globalHost2 = ProbeNetworkMock::project2 + "." + ProbeNetworkMock::globalEndpoint; +// const string globalHost3 = ProbeNetworkMock::project3 + "." + ProbeNetworkMock::globalEndpoint; +// const string customHost3 = ProbeNetworkMock::project3 + "." + ProbeNetworkMock::customEndpoint; + +// // ! means not available +// vector> infos; +// infos.push_back(mManager->GetCandidateHostsInfo( +// ProbeNetworkMock::region1, ProbeNetworkMock::project1, EndpointMode::DEFAULT)); // public !internal !private +// infos.push_back(mManager->GetCandidateHostsInfo( +// ProbeNetworkMock::region1, ProbeNetworkMock::project1, EndpointMode::ACCELERATE)); // !accelerate public +// infos.push_back(mManager->GetCandidateHostsInfo( +// ProbeNetworkMock::region2, ProbeNetworkMock::project2, EndpointMode::DEFAULT)); // accelerate public +// infos.push_back(mManager->GetCandidateHostsInfo( +// ProbeNetworkMock::region3, ProbeNetworkMock::project3, EndpointMode::DEFAULT)); // custom +// infos.push_back(mManager->GetCandidateHostsInfo( +// ProbeNetworkMock::region3, ProbeNetworkMock::project3, EndpointMode::ACCELERATE)); // !accelerate + +// // probe uninitialized host +// mManager->DoProbeUnInitializedHost(); +// APSARA_TEST_TRUE(mManager->mUnInitializedCandidateHostsInfos.empty()); +// APSARA_TEST_EQUAL(infos.size(), mManager->mPartiallyInitializedCandidateHostsInfos.size()); +// for (size_t i = 0; i < infos.size(); ++i) { +// APSARA_TEST_EQUAL(infos[i].get(), mManager->mPartiallyInitializedCandidateHostsInfos[i].lock().get()); +// } + +// APSARA_TEST_EQUAL(publicHost1, infos[0]->GetCurrentHost()); +// APSARA_TEST_EQUAL("", infos[1]->GetCurrentHost()); +// APSARA_TEST_EQUAL(globalHost2, infos[2]->GetCurrentHost()); +// APSARA_TEST_EQUAL(customHost3, infos[3]->GetCurrentHost()); +// APSARA_TEST_EQUAL("", infos[4]->GetCurrentHost()); +// vector len = {1, 2, 1, 1, 1}; +// for (size_t i = 0; i < infos.size(); ++i) { +// vector res; +// infos[i]->GetProbeHosts(res); +// APSARA_TEST_EQUAL(len[i], res.size()); +// } + +// // probe partially initialized host +// mManager->DoProbeHost(); +// APSARA_TEST_TRUE(mManager->mPartiallyInitializedCandidateHostsInfos.empty()); + +// APSARA_TEST_EQUAL(publicHost1, infos[0]->GetCurrentHost()); +// APSARA_TEST_EQUAL(publicHost1, infos[1]->GetCurrentHost()); +// APSARA_TEST_EQUAL(globalHost2, infos[2]->GetCurrentHost()); +// APSARA_TEST_EQUAL(customHost3, infos[3]->GetCurrentHost()); +// APSARA_TEST_EQUAL("", infos[4]->GetCurrentHost()); +// len = {1, 1, 2, 1, 1}; +// for (size_t i = 0; i < infos.size(); ++i) { +// vector res; +// infos[i]->GetProbeHosts(res); +// APSARA_TEST_EQUAL(len[i], res.size()); +// } + +// vector> oldLatencies; +// for (const auto& info : infos) { +// auto& latency = oldLatencies.emplace_back(); +// for (const auto& item : info->mCandidateHosts) { +// for (const auto& entry : item) { +// latency.push_back(entry.GetLatency()); +// } +// } +// } + +// // probe all avaialble hosts +// INT32_FLAG(sls_hosts_probe_interval_sec) = 0; +// mManager->DoProbeHost(); + +// APSARA_TEST_EQUAL(publicHost1, infos[0]->GetCurrentHost()); +// APSARA_TEST_EQUAL(publicHost1, infos[1]->GetCurrentHost()); +// APSARA_TEST_EQUAL(globalHost2, infos[2]->GetCurrentHost()); +// APSARA_TEST_EQUAL(customHost3, infos[3]->GetCurrentHost()); +// APSARA_TEST_EQUAL("", infos[4]->GetCurrentHost()); +// len = {1, 1, 2, 1, 1}; +// for (size_t i = 0; i < infos.size(); ++i) { +// vector res; +// infos[i]->GetProbeHosts(res); +// APSARA_TEST_EQUAL(len[i], res.size()); +// } + +// vector> newLatencies; +// for (const auto& info : infos) { +// auto& latency = newLatencies.emplace_back(); +// for (const auto& item : info->mCandidateHosts) { +// for (const auto& entry : item) { +// latency.push_back(entry.GetLatency()); +// } +// } +// } + +// for (size_t i = 0; i < oldLatencies.size(); ++i) { +// for (size_t j = 0; j < oldLatencies[i].size(); ++j) { +// if (oldLatencies[i][j] != chrono::milliseconds::max()) { +// APSARA_TEST_NOT_EQUAL(newLatencies[i][j], oldLatencies[i][j]); +// } else { +// APSARA_TEST_EQUAL(newLatencies[i][j], oldLatencies[i][j]); +// } +// } +// } +// INT32_FLAG(sls_hosts_probe_interval_sec) = 60; + +// // probe all hosts +// INT32_FLAG(sls_all_hosts_probe_interval_sec) = 0; +// ProbeNetworkMock::enableUnavailableHosts = true; +// mManager->DoProbeHost(); + +// APSARA_TEST_EQUAL(publicHost1, infos[0]->GetCurrentHost()); +// APSARA_TEST_EQUAL(globalHost1, infos[1]->GetCurrentHost()); +// APSARA_TEST_EQUAL(globalHost2, infos[2]->GetCurrentHost()); +// APSARA_TEST_EQUAL(customHost3, infos[3]->GetCurrentHost()); +// APSARA_TEST_EQUAL(globalHost3, infos[4]->GetCurrentHost()); + +// len = {3, 2, 2, 1, 1}; +// for (size_t i = 0; i < infos.size(); ++i) { +// vector res; +// infos[i]->GetProbeHosts(res); +// APSARA_TEST_EQUAL(len[i], res.size()); +// } +// ProbeNetworkMock::enableUnavailableHosts = false; +// INT32_FLAG(sls_all_hosts_probe_interval_sec) = 300; + +// // expired info +// infos[2].reset(); +// infos[4].reset(); +// mManager->DoProbeHost(); +// APSARA_TEST_EQUAL(2U, mManager->mProjectCandidateHostsInfosMap.size()); +// APSARA_TEST_EQUAL(2U, mManager->mProjectCandidateHostsInfosMap[ProbeNetworkMock::project1].size()); +// APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap[ProbeNetworkMock::project3].size()); +// APSARA_TEST_EQUAL(2U, mManager->mRegionCandidateHostsInfosMap.size()); +// APSARA_TEST_EQUAL(2U, mManager->mRegionCandidateHostsInfosMap[ProbeNetworkMock::region1].size()); +// APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap[ProbeNetworkMock::region3].size()); +// } + +// void SLSClientManagerUnittest::TestRealIp() { +// BOOL_FLAG(send_prefer_real_ip) = true; +// mManager->mDoProbeNetwork = GetRealIpMock::DoProbeNetwork; + +// const string unknownRegion = "unknown_region"; +// const string normalRegionEndpoint1 = GetRealIpMock::normalRegion + "-intranet.log.aliyuncs.com"; +// const string normalRegionEndpoint2 = GetRealIpMock::normalRegion + ".log.aliyuncs.com"; +// const string unavailableRegionEndpoint = GetRealIpMock::unavailableRegion + ".log.aliyuncs.com"; +// const string unknownRegionEndpoint = unknownRegion + ".log.aliyuncs.com"; +// mManager->UpdateRemoteRegionEndpoints(GetRealIpMock::normalRegion, {normalRegionEndpoint1, +// normalRegionEndpoint2}); mManager->UpdateRemoteRegionEndpoints(GetRealIpMock::unavailableRegion, +// {unavailableRegionEndpoint}); mManager->UpdateRemoteRegionEndpoints(unknownRegion, {unknownRegionEndpoint}); +// APSARA_TEST_EQUAL(3U, mManager->mRegionRealIpCandidateHostsInfosMap.size()); +// { +// vector hosts; +// mManager->mRegionRealIpCandidateHostsInfosMap.at(GetRealIpMock::normalRegion).GetAllHosts(hosts); +// APSARA_TEST_EQUAL(2U, hosts.size()); +// APSARA_TEST_EQUAL(normalRegionEndpoint1, hosts[0]); +// APSARA_TEST_EQUAL(normalRegionEndpoint2, hosts[1]); +// } +// { +// vector hosts; +// mManager->mRegionRealIpCandidateHostsInfosMap.at(GetRealIpMock::unavailableRegion).GetAllHosts(hosts); +// APSARA_TEST_EQUAL(1U, hosts.size()); +// APSARA_TEST_EQUAL(unavailableRegionEndpoint, hosts[0]); +// } +// { +// vector hosts; +// mManager->mRegionRealIpCandidateHostsInfosMap.at(unknownRegion).GetAllHosts(hosts); +// APSARA_TEST_EQUAL(1U, hosts.size()); +// APSARA_TEST_EQUAL(unknownRegionEndpoint, hosts[0]); +// } + +// // no endpoint available +// mManager->DoUpdateRealIp(); +// APSARA_TEST_EQUAL(0U, mManager->mRegionRealIpMap.size()); + +// // probe host +// INT32_FLAG(sls_all_hosts_probe_interval_sec) = 0; +// mManager->DoProbeHost(); +// INT32_FLAG(sls_all_hosts_probe_interval_sec) = 300; +// for (const auto& item : mManager->mRegionRealIpCandidateHostsInfosMap) { +// APSARA_TEST_NOT_EQUAL("", item.second.GetCurrentHost()); +// } + +// // update all regions +// INT32_FLAG(send_switch_real_ip_interval) = 0; +// mManager->UpdateOutdatedRealIpRegions(GetRealIpMock::unavailableRegion); +// mManager->DoUpdateRealIp(); +// APSARA_TEST_TRUE(mManager->mOutdatedRealIpRegions.empty()); +// APSARA_TEST_EQUAL(2U, mManager->mRegionRealIpMap.size()); +// APSARA_TEST_EQUAL(GetRealIpMock::normalRegionIps[0], mManager->GetRealIp(GetRealIpMock::normalRegion)); +// APSARA_TEST_EQUAL(GetRealIpMock::unavailableRegionIps[0], mManager->GetRealIp(GetRealIpMock::unavailableRegion)); +// INT32_FLAG(send_switch_real_ip_interval) = 60; + +// // update only outdated regions +// mManager->UpdateOutdatedRealIpRegions(GetRealIpMock::normalRegion); +// mManager->DoUpdateRealIp(); +// APSARA_TEST_TRUE(mManager->mOutdatedRealIpRegions.empty()); +// APSARA_TEST_EQUAL(2U, mManager->mRegionRealIpMap.size()); +// APSARA_TEST_EQUAL(GetRealIpMock::normalRegionIps[1], mManager->GetRealIp(GetRealIpMock::normalRegion)); +// APSARA_TEST_EQUAL(GetRealIpMock::unavailableRegionIps[0], mManager->GetRealIp(GetRealIpMock::unavailableRegion)); + +// mManager->mDoProbeNetwork = ProbeNetworkMock::DoProbeNetwork; +// BOOL_FLAG(send_prefer_real_ip) = false; +// } void SLSClientManagerUnittest::TestAccessKeyManagement() { string accessKeyId, accessKeySecret; @@ -1365,15 +1365,15 @@ void SLSClientManagerUnittest::TestGetCandidateHostsInfoOpen() { } } -UNIT_TEST_CASE(SLSClientManagerUnittest, TestLocalRegionEndpoints) -UNIT_TEST_CASE(SLSClientManagerUnittest, TestRemoteRegionEndpoints) -UNIT_TEST_CASE(SLSClientManagerUnittest, TestGetCandidateHostsInfo) -UNIT_TEST_CASE(SLSClientManagerUnittest, TestUpdateHostInfo) -UNIT_TEST_CASE(SLSClientManagerUnittest, TestUsingHttps) -UNIT_TEST_CASE(SLSClientManagerUnittest, TestProbeNetwork) -UNIT_TEST_CASE(SLSClientManagerUnittest, TestRealIp) +// UNIT_TEST_CASE(SLSClientManagerUnittest, TestLocalRegionEndpoints) +// UNIT_TEST_CASE(SLSClientManagerUnittest, TestRemoteRegionEndpoints) +// UNIT_TEST_CASE(SLSClientManagerUnittest, TestGetCandidateHostsInfo) +// UNIT_TEST_CASE(SLSClientManagerUnittest, TestUpdateHostInfo) +// UNIT_TEST_CASE(SLSClientManagerUnittest, TestUsingHttps) +// UNIT_TEST_CASE(SLSClientManagerUnittest, TestProbeNetwork) +// UNIT_TEST_CASE(SLSClientManagerUnittest, TestRealIp) UNIT_TEST_CASE(SLSClientManagerUnittest, TestAccessKeyManagement) -UNIT_TEST_CASE(SLSClientManagerUnittest, TestGetCandidateHostsInfoOpen) +UNIT_TEST_CASE(SLSClientManagerUnittest, TestGetCandidateHostsInfo) } // namespace logtail From 270a3119694733fb84e76f284bb941ff73afe46a Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Tue, 10 Dec 2024 08:20:30 +0000 Subject: [PATCH 04/41] polish --- core/unittest/flusher/FlusherSLSUnittest.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/unittest/flusher/FlusherSLSUnittest.cpp b/core/unittest/flusher/FlusherSLSUnittest.cpp index a414f22987..0b34e257a8 100644 --- a/core/unittest/flusher/FlusherSLSUnittest.cpp +++ b/core/unittest/flusher/FlusherSLSUnittest.cpp @@ -292,8 +292,7 @@ void FlusherSLSUnittest::OnSuccessfulInit() { flusher->SetMetricsRecordRef(FlusherSLS::sName, "1"); APSARA_TEST_TRUE(flusher->Init(configJson, optionalGoPipeline)); APSARA_TEST_EQUAL(EndpointMode::DEFAULT, flusher->mEndpointMode); - auto& endpoints - = EnterpriseSLSClientManager::GetInstance()->mRegionCandidateEndpointsMap["cn-hangzhou"].mRemoteEndpoints; + endpoints = EnterpriseSLSClientManager::GetInstance()->mRegionCandidateEndpointsMap["cn-hangzhou"].mRemoteEndpoints; APSARA_TEST_EQUAL(1U, endpoints.size()); APSARA_TEST_EQUAL("cn-hangzhou.log.aliyuncs.com", endpoints[0]); APSARA_TEST_EQUAL(flusher->mProject, flusher->mCandidateHostsInfo->GetProject()); From b83200c4d026b951e999663559743d26340bf529 Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Tue, 10 Dec 2024 08:28:00 +0000 Subject: [PATCH 05/41] polish --- core/plugin/flusher/sls/DiskBufferWriter.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/plugin/flusher/sls/DiskBufferWriter.cpp b/core/plugin/flusher/sls/DiskBufferWriter.cpp index 9cfb3a9917..e3c46e86c9 100644 --- a/core/plugin/flusher/sls/DiskBufferWriter.cpp +++ b/core/plugin/flusher/sls/DiskBufferWriter.cpp @@ -61,6 +61,7 @@ static EndpointMode GetEndpointMode(sls_logs::EndpointMode mode) { case sls_logs::EndpointMode::CUSTOM: return EndpointMode::CUSTOM; } + return EndpointMode::DEFAULT; } static sls_logs::EndpointMode GetEndpointMode(EndpointMode mode) { @@ -72,6 +73,7 @@ static sls_logs::EndpointMode GetEndpointMode(EndpointMode mode) { case EndpointMode::CUSTOM: return sls_logs::EndpointMode::CUSTOM; } + return sls_logs::EndpointMode::DEFAULT; } #endif From 37a8c1e6a14ac43f1b759b9b4a2f05405f87f2c1 Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Tue, 10 Dec 2024 08:35:52 +0000 Subject: [PATCH 06/41] polish --- core/unittest/flusher/SLSClientManagerUnittest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/unittest/flusher/SLSClientManagerUnittest.cpp b/core/unittest/flusher/SLSClientManagerUnittest.cpp index e4c5b6df8f..d28a53cd5c 100644 --- a/core/unittest/flusher/SLSClientManagerUnittest.cpp +++ b/core/unittest/flusher/SLSClientManagerUnittest.cpp @@ -1334,7 +1334,7 @@ void SLSClientManagerUnittest::TestAccessKeyManagement() { APSARA_TEST_EQUAL(STRING_FLAG(default_access_key), accessKeySecret); } -void SLSClientManagerUnittest::TestGetCandidateHostsInfoOpen() { +void SLSClientManagerUnittest::TestGetCandidateHostsInfo() { const string project = "project"; const string endpoint = "endpoint"; CandidateHostsInfo* infoPtr = nullptr; From 6c48535a0a6fcc9095e20ae4daae1f1e36a7ea13 Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Tue, 10 Dec 2024 09:47:38 +0000 Subject: [PATCH 07/41] polish --- core/pipeline/plugin/interface/HttpFlusher.h | 2 +- core/plugin/flusher/sls/FlusherSLS.cpp | 17 +- core/plugin/flusher/sls/FlusherSLS.h | 2 +- core/plugin/flusher/sls/SLSClientManager.cpp | 4 + core/plugin/flusher/sls/SLSClientManager.h | 3 + .../flusher/SLSClientManagerUnittest.cpp | 402 +++++++++--------- core/unittest/plugin/PluginMock.h | 2 +- .../how-to-write-native-flusher-plugins.md | 2 +- 8 files changed, 228 insertions(+), 206 deletions(-) diff --git a/core/pipeline/plugin/interface/HttpFlusher.h b/core/pipeline/plugin/interface/HttpFlusher.h index a8bba21296..377a075b31 100644 --- a/core/pipeline/plugin/interface/HttpFlusher.h +++ b/core/pipeline/plugin/interface/HttpFlusher.h @@ -27,7 +27,7 @@ class HttpFlusher : public Flusher { public: virtual ~HttpFlusher() = default; - virtual bool BuildRequest(SenderQueueItem* item, std::unique_ptr& req, bool* keepItem) const = 0; + virtual bool BuildRequest(SenderQueueItem* item, std::unique_ptr& req, bool* keepItem) = 0; virtual void OnSendDone(const HttpResponse& response, SenderQueueItem* item) = 0; virtual SinkType GetSinkType() override { return SinkType::HTTP; } diff --git a/core/plugin/flusher/sls/FlusherSLS.cpp b/core/plugin/flusher/sls/FlusherSLS.cpp index a13dfa62c8..5e0c47d8fd 100644 --- a/core/plugin/flusher/sls/FlusherSLS.cpp +++ b/core/plugin/flusher/sls/FlusherSLS.cpp @@ -581,7 +581,7 @@ bool FlusherSLS::FlushAll() { return SerializeAndPush(std::move(res)); } -bool FlusherSLS::BuildRequest(SenderQueueItem* item, unique_ptr& req, bool* keepItem) const { +bool FlusherSLS::BuildRequest(SenderQueueItem* item, unique_ptr& req, bool* keepItem) { if (mSendCnt) { mSendCnt->Add(1); } @@ -593,25 +593,38 @@ bool FlusherSLS::BuildRequest(SenderQueueItem* item, unique_ptr return false; } + auto data = static_cast(item); #ifdef __ENTERPRISE__ if (BOOL_FLAG(send_prefer_real_ip)) { data->mCurrentHost = EnterpriseSLSClientManager::GetInstance()->GetRealIp(mRegion); if (data->mCurrentHost.empty()) { + auto info + = EnterpriseSLSClientManager::GetInstance()->GetCandidateHostsInfo(mRegion, mProject, mEndpointMode); + if (mCandidateHostsInfo.get() != info.get()) { + mCandidateHostsInfo = info; + } data->mCurrentHost = mCandidateHostsInfo->GetCurrentHost(); data->mRealIpFlag = false; } else { data->mRealIpFlag = true; } } else { + // in case local region endpoint mode is changed, we should always check before sending + auto info = EnterpriseSLSClientManager::GetInstance()->GetCandidateHostsInfo(mRegion, mProject, mEndpointMode); + if (mCandidateHostsInfo.get() != info.get()) { + mCandidateHostsInfo = info; + } #endif data->mCurrentHost = mCandidateHostsInfo->GetCurrentHost(); #ifdef __ENTERPRISE__ } #endif if (data->mCurrentHost.empty()) { + if (mCandidateHostsInfo->IsInitialized()) { + GetRegionConcurrencyLimiter(mRegion)->OnFail(); + } *keepItem = true; - GetRegionConcurrencyLimiter(mRegion)->OnFail(); return false; } diff --git a/core/plugin/flusher/sls/FlusherSLS.h b/core/plugin/flusher/sls/FlusherSLS.h index 61f3010415..01fa082a85 100644 --- a/core/plugin/flusher/sls/FlusherSLS.h +++ b/core/plugin/flusher/sls/FlusherSLS.h @@ -63,7 +63,7 @@ class FlusherSLS : public HttpFlusher { bool Send(PipelineEventGroup&& g) override; bool Flush(size_t key) override; bool FlushAll() override; - bool BuildRequest(SenderQueueItem* item, std::unique_ptr& req, bool* keepItem) const override; + bool BuildRequest(SenderQueueItem* item, std::unique_ptr& req, bool* keepItem) override; void OnSendDone(const HttpResponse& response, SenderQueueItem* item) override; CompressType GetCompressType() const { return mCompressor ? mCompressor->GetCompressType() : CompressType::NONE; } diff --git a/core/plugin/flusher/sls/SLSClientManager.cpp b/core/plugin/flusher/sls/SLSClientManager.cpp index 343e8525b9..59f3aae789 100644 --- a/core/plugin/flusher/sls/SLSClientManager.cpp +++ b/core/plugin/flusher/sls/SLSClientManager.cpp @@ -644,6 +644,10 @@ shared_ptr SLSClientManager::GetCandidateHostsInfo(const str } auto info = make_shared(project, "", EndpointMode::CUSTOM); info->UpdateHosts({EndpointMode::CUSTOM, {standardEndpoint}}); + // manually set the endpoint to be available + info->UpdateHostInfo(info->GetFirstHost(), chrono::milliseconds(10)); + info->SelectBestHost(); + info->SetInitialized(); { lock_guard lock(mCandidateHostsInfosMapMux); mProjectCandidateHostsInfosMap[project].emplace_back(info); diff --git a/core/plugin/flusher/sls/SLSClientManager.h b/core/plugin/flusher/sls/SLSClientManager.h index d72cfd5e62..7afee13026 100644 --- a/core/plugin/flusher/sls/SLSClientManager.h +++ b/core/plugin/flusher/sls/SLSClientManager.h @@ -86,6 +86,8 @@ class CandidateHostsInfo { const std::string& GetProject() const { return mProject; } const std::string& GetRegion() const { return mRegion; } EndpointMode GetMode() const { return mMode; } + bool IsInitialized() const { return mInitialized.load(); } + void SetInitialized() { mInitialized = true; } private: bool HasValidHost() const; @@ -95,6 +97,7 @@ class CandidateHostsInfo { const std::string mProject; const std::string mRegion; const EndpointMode mMode; + std::atomic_bool mInitialized = false; mutable std::mutex mCurrentHostMux; std::string mCurrentHost; diff --git a/core/unittest/flusher/SLSClientManagerUnittest.cpp b/core/unittest/flusher/SLSClientManagerUnittest.cpp index d28a53cd5c..bcdebccdf0 100644 --- a/core/unittest/flusher/SLSClientManagerUnittest.cpp +++ b/core/unittest/flusher/SLSClientManagerUnittest.cpp @@ -572,10 +572,10 @@ class SLSClientManagerUnittest : public ::testing::Test { // SLSClientManager::GetInstance()->mGetEndpointRealIp = GetRealIpMock::GetEndpointRealIpMock; // } - // void TearDown() override { mManager->Clear(); } + // void TearDown() override { mManager.Clear(); } private: - SLSClientManager* mManager = SLSClientManager::GetInstance(); + SLSClientManager mManager; }; // void SLSClientManagerUnittest::TestLocalRegionEndpoints() { @@ -583,80 +583,80 @@ class SLSClientManagerUnittest : public ::testing::Test { // { // // default // const string endpoint = region + ".log.aliyuncs.com"; -// mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint}); +// mManager.UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint}); -// auto& item = mManager->mRegionCandidateEndpointsMap[region]; +// auto& item = mManager.mRegionCandidateEndpointsMap[region]; // APSARA_TEST_EQUAL(EndpointMode::DEFAULT, item.mMode); // APSARA_TEST_EQUAL(1U, item.mLocalEndpoints.size()); // APSARA_TEST_EQUAL(endpoint, item.mLocalEndpoints[0]); -// APSARA_TEST_FALSE(mManager->UsingHttps(region)); +// APSARA_TEST_FALSE(mManager.UsingHttps(region)); // } // { // // accelerate (+ default) // const string endpoint1 = region + "-intranet.log.aliyuncs.com"; // const string endpoint2 = kAccelerationDataEndpoint; -// mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); +// mManager.UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); -// auto& item = mManager->mRegionCandidateEndpointsMap[region]; +// auto& item = mManager.mRegionCandidateEndpointsMap[region]; // APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, item.mMode); // APSARA_TEST_EQUAL(0U, item.mLocalEndpoints.size()); -// APSARA_TEST_FALSE(mManager->UsingHttps(region)); +// APSARA_TEST_FALSE(mManager.UsingHttps(region)); // } // { // // custom (+ default) // const string endpoint1 = region + "-intranet.log.aliyuncs.com"; // const string endpoint2 = "custom.endpoint"; -// mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); +// mManager.UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); -// auto& item = mManager->mRegionCandidateEndpointsMap[region]; +// auto& item = mManager.mRegionCandidateEndpointsMap[region]; // APSARA_TEST_EQUAL(EndpointMode::CUSTOM, item.mMode); // APSARA_TEST_EQUAL(1U, item.mLocalEndpoints.size()); // APSARA_TEST_EQUAL(endpoint2, item.mLocalEndpoints[0]); -// APSARA_TEST_FALSE(mManager->UsingHttps(region)); +// APSARA_TEST_FALSE(mManager.UsingHttps(region)); // } // { // // accelerate + custom (+ default) // const string endpoint1 = "custom.endpoint"; // const string endpoint2 = region + "-intranet.log.aliyuncs.com"; // const string endpoint3 = kAccelerationDataEndpoint; -// mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2, endpoint3}); +// mManager.UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2, endpoint3}); -// auto& item = mManager->mRegionCandidateEndpointsMap[region]; +// auto& item = mManager.mRegionCandidateEndpointsMap[region]; // APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, item.mMode); // APSARA_TEST_EQUAL(0U, item.mLocalEndpoints.size()); -// APSARA_TEST_FALSE(mManager->UsingHttps(region)); +// APSARA_TEST_FALSE(mManager.UsingHttps(region)); // } // { // // http -> https -> https // const string endpoint1 = region + "-intranet.log.aliyuncs.com"; // const string endpoint2 = "https://" + kAccelerationDataEndpoint; -// mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); -// APSARA_TEST_TRUE(mManager->UsingHttps(region)); +// mManager.UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); +// APSARA_TEST_TRUE(mManager.UsingHttps(region)); -// mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); -// APSARA_TEST_TRUE(mManager->UsingHttps(region)); +// mManager.UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); +// APSARA_TEST_TRUE(mManager.UsingHttps(region)); // } // { // // https -> http -> http // const string endpoint1 = region + "-intranet.log.aliyuncs.com"; // const string endpoint2 = kAccelerationDataEndpoint; -// mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); -// APSARA_TEST_FALSE(mManager->UsingHttps(region)); +// mManager.UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); +// APSARA_TEST_FALSE(mManager.UsingHttps(region)); -// mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); -// APSARA_TEST_FALSE(mManager->UsingHttps(region)); +// mManager.UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); +// APSARA_TEST_FALSE(mManager.UsingHttps(region)); // } // } // void SLSClientManagerUnittest::TestRemoteRegionEndpoints() { // { -// auto info1 = mManager->GetCandidateHostsInfo("region_1", "project_1", EndpointMode::DEFAULT); -// auto info2 = mManager->GetCandidateHostsInfo("region_1", "project_2", EndpointMode::DEFAULT); -// auto info3 = mManager->GetCandidateHostsInfo("region_2", "project_3", EndpointMode::DEFAULT); +// auto info1 = mManager.GetCandidateHostsInfo("region_1", "project_1", EndpointMode::DEFAULT); +// auto info2 = mManager.GetCandidateHostsInfo("region_1", "project_2", EndpointMode::DEFAULT); +// auto info3 = mManager.GetCandidateHostsInfo("region_2", "project_3", EndpointMode::DEFAULT); // APSARA_TEST_EQUAL(0U, info3->mCandidateHosts.size()); // // create remote info -// mManager->UpdateRemoteRegionEndpoints("region_1", {"endpoint_1", "endpoint_2"}); +// mManager.UpdateRemoteRegionEndpoints("region_1", {"endpoint_1", "endpoint_2"}); // APSARA_TEST_EQUAL(2U, info1->mCandidateHosts.size()); // APSARA_TEST_EQUAL(2U, info2->mCandidateHosts.size()); // APSARA_TEST_EQUAL(0U, info3->mCandidateHosts.size()); @@ -670,7 +670,7 @@ class SLSClientManagerUnittest : public ::testing::Test { // APSARA_TEST_EQUAL("project_2.endpoint_2", info2->mCandidateHosts[1][0].GetHostname()); // // update remote info with overwrite -// mManager->UpdateRemoteRegionEndpoints("region_1", {"endpoint_2", "endpoint_3"}); +// mManager.UpdateRemoteRegionEndpoints("region_1", {"endpoint_2", "endpoint_3"}); // APSARA_TEST_EQUAL(2U, info1->mCandidateHosts.size()); // APSARA_TEST_EQUAL(2U, info2->mCandidateHosts.size()); // APSARA_TEST_EQUAL(0U, info3->mCandidateHosts.size()); @@ -684,7 +684,7 @@ class SLSClientManagerUnittest : public ::testing::Test { // APSARA_TEST_EQUAL("project_2.endpoint_3", info2->mCandidateHosts[1][0].GetHostname()); // // update remote info with create -// mManager->UpdateRemoteRegionEndpoints( +// mManager.UpdateRemoteRegionEndpoints( // "region_1", {"endpoint_4"}, SLSClientManager::RemoteEndpointUpdateAction::CREATE); // APSARA_TEST_EQUAL(2U, info1->mCandidateHosts.size()); // APSARA_TEST_EQUAL(2U, info2->mCandidateHosts.size()); @@ -699,7 +699,7 @@ class SLSClientManagerUnittest : public ::testing::Test { // APSARA_TEST_EQUAL("project_2.endpoint_3", info2->mCandidateHosts[1][0].GetHostname()); // // update remote info with append -// mManager->UpdateRemoteRegionEndpoints( +// mManager.UpdateRemoteRegionEndpoints( // "region_1", {"endpoint_3", "endpoint_1"}, SLSClientManager::RemoteEndpointUpdateAction::APPEND); // APSARA_TEST_EQUAL(3U, info1->mCandidateHosts.size()); // APSARA_TEST_EQUAL(3U, info2->mCandidateHosts.size()); @@ -719,7 +719,7 @@ class SLSClientManagerUnittest : public ::testing::Test { // } // { // // get candidate host info after remote info is updated -// auto info = mManager->GetCandidateHostsInfo("region_1", "project_1", EndpointMode::DEFAULT); +// auto info = mManager.GetCandidateHostsInfo("region_1", "project_1", EndpointMode::DEFAULT); // APSARA_TEST_EQUAL(3U, info->mCandidateHosts.size()); // APSARA_TEST_EQUAL(1U, info->mCandidateHosts[0].size()); // APSARA_TEST_EQUAL(1U, info->mCandidateHosts[1].size()); @@ -747,33 +747,33 @@ class SLSClientManagerUnittest : public ::testing::Test { // // no candidate host info && no region endpoints // { // // default mode -// auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); +// auto info = mManager.GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); // APSARA_TEST_EQUAL(project, info->GetProject()); // APSARA_TEST_EQUAL(region, info->GetRegion()); // APSARA_TEST_EQUAL(EndpointMode::DEFAULT, info->GetMode()); // APSARA_TEST_EQUAL(0U, info->mCandidateHosts.size()); // { -// APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); -// auto& infos = mManager->mProjectCandidateHostsInfosMap[project]; +// APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap.size()); +// auto& infos = mManager.mProjectCandidateHostsInfosMap[project]; // APSARA_TEST_EQUAL(1U, infos.size()); // auto& weakInfo = *infos.begin(); // APSARA_TEST_FALSE(weakInfo.expired()); // APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); // } // { -// APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap.size()); -// auto& infos = mManager->mRegionCandidateHostsInfosMap[region]; +// APSARA_TEST_EQUAL(1U, mManager.mRegionCandidateHostsInfosMap.size()); +// auto& infos = mManager.mRegionCandidateHostsInfosMap[region]; // APSARA_TEST_EQUAL(1U, infos.size()); // auto& weakInfo = *infos.begin(); // APSARA_TEST_FALSE(weakInfo.expired()); // APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); // } -// APSARA_TEST_EQUAL(1U, mManager->mUnInitializedCandidateHostsInfos.size()); -// APSARA_TEST_EQUAL(info.get(), mManager->mUnInitializedCandidateHostsInfos[0].lock().get()); +// APSARA_TEST_EQUAL(1U, mManager.mUnInitializedCandidateHostsInfos.size()); +// APSARA_TEST_EQUAL(info.get(), mManager.mUnInitializedCandidateHostsInfos[0].lock().get()); // } // { // // accelerate mode -// auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); +// auto info = mManager.GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); // APSARA_TEST_EQUAL(project, info->GetProject()); // APSARA_TEST_EQUAL(region, info->GetRegion()); // APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, info->GetMode()); @@ -782,8 +782,8 @@ class SLSClientManagerUnittest : public ::testing::Test { // APSARA_TEST_EQUAL(globalHost, info->mCandidateHosts[0][0].GetHostname()); // APSARA_TEST_EQUAL(chrono::milliseconds::max(), info->mCandidateHosts[0][0].GetLatency()); // { -// APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); -// auto& infos = mManager->mProjectCandidateHostsInfosMap[project]; +// APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap.size()); +// auto& infos = mManager.mProjectCandidateHostsInfosMap[project]; // APSARA_TEST_EQUAL(2U, infos.size()); // auto it = infos.begin(); // APSARA_TEST_TRUE(it->expired()); @@ -792,8 +792,8 @@ class SLSClientManagerUnittest : public ::testing::Test { // APSARA_TEST_EQUAL(info.get(), it->lock().get()); // } // { -// APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap.size()); -// auto& infos = mManager->mRegionCandidateHostsInfosMap[region]; +// APSARA_TEST_EQUAL(1U, mManager.mRegionCandidateHostsInfosMap.size()); +// auto& infos = mManager.mRegionCandidateHostsInfosMap[region]; // APSARA_TEST_EQUAL(2U, infos.size()); // auto it = infos.begin(); // APSARA_TEST_TRUE(it->expired()); @@ -801,17 +801,17 @@ class SLSClientManagerUnittest : public ::testing::Test { // APSARA_TEST_FALSE(it->expired()); // APSARA_TEST_EQUAL(info.get(), it->lock().get()); // } -// APSARA_TEST_EQUAL(2U, mManager->mUnInitializedCandidateHostsInfos.size()); -// APSARA_TEST_EQUAL(info.get(), mManager->mUnInitializedCandidateHostsInfos[1].lock().get()); +// APSARA_TEST_EQUAL(2U, mManager.mUnInitializedCandidateHostsInfos.size()); +// APSARA_TEST_EQUAL(info.get(), mManager.mUnInitializedCandidateHostsInfos[1].lock().get()); // } -// mManager->Clear(); +// mManager.Clear(); // } // { // // no candidate host info && with region endpoints && region endpoint mode is default -// mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {privateEndpoint}); +// mManager.UpdateLocalRegionEndpointsAndHttpsInfo(region, {privateEndpoint}); // { // // default mode -// auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); +// auto info = mManager.GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); // APSARA_TEST_EQUAL(project, info->GetProject()); // APSARA_TEST_EQUAL(region, info->GetRegion()); // APSARA_TEST_EQUAL(EndpointMode::DEFAULT, info->GetMode()); @@ -819,27 +819,27 @@ class SLSClientManagerUnittest : public ::testing::Test { // APSARA_TEST_EQUAL(1U, info->mCandidateHosts[0].size()); // APSARA_TEST_EQUAL(privateHost, info->mCandidateHosts[0][0].GetHostname()); // { -// APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); -// auto& infos = mManager->mProjectCandidateHostsInfosMap[project]; +// APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap.size()); +// auto& infos = mManager.mProjectCandidateHostsInfosMap[project]; // APSARA_TEST_EQUAL(1U, infos.size()); // auto& weakInfo = *infos.begin(); // APSARA_TEST_FALSE(weakInfo.expired()); // APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); // } // { -// APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap.size()); -// auto& infos = mManager->mRegionCandidateHostsInfosMap[region]; +// APSARA_TEST_EQUAL(1U, mManager.mRegionCandidateHostsInfosMap.size()); +// auto& infos = mManager.mRegionCandidateHostsInfosMap[region]; // APSARA_TEST_EQUAL(1U, infos.size()); // auto& weakInfo = *infos.begin(); // APSARA_TEST_FALSE(weakInfo.expired()); // APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); // } -// APSARA_TEST_EQUAL(1U, mManager->mUnInitializedCandidateHostsInfos.size()); -// APSARA_TEST_EQUAL(info.get(), mManager->mUnInitializedCandidateHostsInfos[0].lock().get()); +// APSARA_TEST_EQUAL(1U, mManager.mUnInitializedCandidateHostsInfos.size()); +// APSARA_TEST_EQUAL(info.get(), mManager.mUnInitializedCandidateHostsInfos[0].lock().get()); // } // { // // accelerate mode -// auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); +// auto info = mManager.GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); // APSARA_TEST_EQUAL(project, info->GetProject()); // APSARA_TEST_EQUAL(region, info->GetRegion()); // APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, info->GetMode()); @@ -848,8 +848,8 @@ class SLSClientManagerUnittest : public ::testing::Test { // APSARA_TEST_EQUAL(globalHost, info->mCandidateHosts[0][0].GetHostname()); // APSARA_TEST_EQUAL(chrono::milliseconds::max(), info->mCandidateHosts[0][0].GetLatency()); // { -// APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); -// auto& infos = mManager->mProjectCandidateHostsInfosMap[project]; +// APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap.size()); +// auto& infos = mManager.mProjectCandidateHostsInfosMap[project]; // APSARA_TEST_EQUAL(2U, infos.size()); // auto it = infos.begin(); // APSARA_TEST_TRUE(it->expired()); @@ -858,8 +858,8 @@ class SLSClientManagerUnittest : public ::testing::Test { // APSARA_TEST_EQUAL(info.get(), it->lock().get()); // } // { -// APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap.size()); -// auto& infos = mManager->mRegionCandidateHostsInfosMap[region]; +// APSARA_TEST_EQUAL(1U, mManager.mRegionCandidateHostsInfosMap.size()); +// auto& infos = mManager.mRegionCandidateHostsInfosMap[region]; // APSARA_TEST_EQUAL(2U, infos.size()); // auto it = infos.begin(); // APSARA_TEST_TRUE(it->expired()); @@ -867,17 +867,17 @@ class SLSClientManagerUnittest : public ::testing::Test { // APSARA_TEST_FALSE(it->expired()); // APSARA_TEST_EQUAL(info.get(), it->lock().get()); // } -// APSARA_TEST_EQUAL(2U, mManager->mUnInitializedCandidateHostsInfos.size()); -// APSARA_TEST_EQUAL(info.get(), mManager->mUnInitializedCandidateHostsInfos[1].lock().get()); +// APSARA_TEST_EQUAL(2U, mManager.mUnInitializedCandidateHostsInfos.size()); +// APSARA_TEST_EQUAL(info.get(), mManager.mUnInitializedCandidateHostsInfos[1].lock().get()); // } -// mManager->Clear(); +// mManager.Clear(); // } // { // // no candidate host info && with region endpoints && region endpoint mode is accelerate -// mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {globalEndpoint}); +// mManager.UpdateLocalRegionEndpointsAndHttpsInfo(region, {globalEndpoint}); // { // // default mode -// auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); +// auto info = mManager.GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); // APSARA_TEST_EQUAL(project, info->GetProject()); // APSARA_TEST_EQUAL(region, info->GetRegion()); // APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, info->GetMode()); @@ -886,27 +886,27 @@ class SLSClientManagerUnittest : public ::testing::Test { // APSARA_TEST_EQUAL(globalHost, info->mCandidateHosts[0][0].GetHostname()); // APSARA_TEST_EQUAL(chrono::milliseconds::max(), info->mCandidateHosts[0][0].GetLatency()); // { -// APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); -// auto& infos = mManager->mProjectCandidateHostsInfosMap[project]; +// APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap.size()); +// auto& infos = mManager.mProjectCandidateHostsInfosMap[project]; // APSARA_TEST_EQUAL(1U, infos.size()); // auto& weakInfo = *infos.begin(); // APSARA_TEST_FALSE(weakInfo.expired()); // APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); // } // { -// APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap.size()); -// auto& infos = mManager->mRegionCandidateHostsInfosMap[region]; +// APSARA_TEST_EQUAL(1U, mManager.mRegionCandidateHostsInfosMap.size()); +// auto& infos = mManager.mRegionCandidateHostsInfosMap[region]; // APSARA_TEST_EQUAL(1U, infos.size()); // auto& weakInfo = *infos.begin(); // APSARA_TEST_FALSE(weakInfo.expired()); // APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); // } -// APSARA_TEST_EQUAL(1U, mManager->mUnInitializedCandidateHostsInfos.size()); -// APSARA_TEST_EQUAL(info.get(), mManager->mUnInitializedCandidateHostsInfos[0].lock().get()); +// APSARA_TEST_EQUAL(1U, mManager.mUnInitializedCandidateHostsInfos.size()); +// APSARA_TEST_EQUAL(info.get(), mManager.mUnInitializedCandidateHostsInfos[0].lock().get()); // } // { // // accelerate mode -// auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); +// auto info = mManager.GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); // APSARA_TEST_EQUAL(project, info->GetProject()); // APSARA_TEST_EQUAL(region, info->GetRegion()); // APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, info->GetMode()); @@ -915,8 +915,8 @@ class SLSClientManagerUnittest : public ::testing::Test { // APSARA_TEST_EQUAL(globalHost, info->mCandidateHosts[0][0].GetHostname()); // APSARA_TEST_EQUAL(chrono::milliseconds::max(), info->mCandidateHosts[0][0].GetLatency()); // { -// APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); -// auto& infos = mManager->mProjectCandidateHostsInfosMap[project]; +// APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap.size()); +// auto& infos = mManager.mProjectCandidateHostsInfosMap[project]; // APSARA_TEST_EQUAL(2U, infos.size()); // auto it = infos.begin(); // APSARA_TEST_TRUE(it->expired()); @@ -925,8 +925,8 @@ class SLSClientManagerUnittest : public ::testing::Test { // APSARA_TEST_EQUAL(info.get(), it->lock().get()); // } // { -// APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap.size()); -// auto& infos = mManager->mRegionCandidateHostsInfosMap[region]; +// APSARA_TEST_EQUAL(1U, mManager.mRegionCandidateHostsInfosMap.size()); +// auto& infos = mManager.mRegionCandidateHostsInfosMap[region]; // APSARA_TEST_EQUAL(2U, infos.size()); // auto it = infos.begin(); // APSARA_TEST_TRUE(it->expired()); @@ -934,17 +934,17 @@ class SLSClientManagerUnittest : public ::testing::Test { // APSARA_TEST_FALSE(it->expired()); // APSARA_TEST_EQUAL(info.get(), it->lock().get()); // } -// APSARA_TEST_EQUAL(2U, mManager->mUnInitializedCandidateHostsInfos.size()); -// APSARA_TEST_EQUAL(info.get(), mManager->mUnInitializedCandidateHostsInfos[1].lock().get()); +// APSARA_TEST_EQUAL(2U, mManager.mUnInitializedCandidateHostsInfos.size()); +// APSARA_TEST_EQUAL(info.get(), mManager.mUnInitializedCandidateHostsInfos[1].lock().get()); // } -// mManager->Clear(); +// mManager.Clear(); // } // { // // no candidate host info && with region endpoints && region endpoint mode is custom -// mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {customEndpoint}); +// mManager.UpdateLocalRegionEndpointsAndHttpsInfo(region, {customEndpoint}); // { // // default mode -// auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); +// auto info = mManager.GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); // APSARA_TEST_EQUAL(project, info->GetProject()); // APSARA_TEST_EQUAL(region, info->GetRegion()); // APSARA_TEST_EQUAL(EndpointMode::CUSTOM, info->GetMode()); @@ -953,27 +953,27 @@ class SLSClientManagerUnittest : public ::testing::Test { // APSARA_TEST_EQUAL(customHost, info->mCandidateHosts[0][0].GetHostname()); // APSARA_TEST_EQUAL(chrono::milliseconds::max(), info->mCandidateHosts[0][0].GetLatency()); // { -// APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); -// auto& infos = mManager->mProjectCandidateHostsInfosMap[project]; +// APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap.size()); +// auto& infos = mManager.mProjectCandidateHostsInfosMap[project]; // APSARA_TEST_EQUAL(1U, infos.size()); // auto& weakInfo = *infos.begin(); // APSARA_TEST_FALSE(weakInfo.expired()); // APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); // } // { -// APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap.size()); -// auto& infos = mManager->mRegionCandidateHostsInfosMap[region]; +// APSARA_TEST_EQUAL(1U, mManager.mRegionCandidateHostsInfosMap.size()); +// auto& infos = mManager.mRegionCandidateHostsInfosMap[region]; // APSARA_TEST_EQUAL(1U, infos.size()); // auto& weakInfo = *infos.begin(); // APSARA_TEST_FALSE(weakInfo.expired()); // APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); // } -// APSARA_TEST_EQUAL(1U, mManager->mUnInitializedCandidateHostsInfos.size()); -// APSARA_TEST_EQUAL(info.get(), mManager->mUnInitializedCandidateHostsInfos[0].lock().get()); +// APSARA_TEST_EQUAL(1U, mManager.mUnInitializedCandidateHostsInfos.size()); +// APSARA_TEST_EQUAL(info.get(), mManager.mUnInitializedCandidateHostsInfos[0].lock().get()); // } // { // // accelerate mode -// auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); +// auto info = mManager.GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); // APSARA_TEST_EQUAL(project, info->GetProject()); // APSARA_TEST_EQUAL(region, info->GetRegion()); // APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, info->GetMode()); @@ -982,107 +982,107 @@ class SLSClientManagerUnittest : public ::testing::Test { // APSARA_TEST_EQUAL(globalHost, info->mCandidateHosts[0][0].GetHostname()); // APSARA_TEST_EQUAL(chrono::milliseconds::max(), info->mCandidateHosts[0][0].GetLatency()); // { -// APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); -// auto& infos = mManager->mProjectCandidateHostsInfosMap[project]; +// APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap.size()); +// auto& infos = mManager.mProjectCandidateHostsInfosMap[project]; // APSARA_TEST_EQUAL(2U, infos.size()); // auto& weakInfo = *(++infos.begin()); // APSARA_TEST_FALSE(weakInfo.expired()); // APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); // } // { -// APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap.size()); -// auto& infos = mManager->mRegionCandidateHostsInfosMap[region]; +// APSARA_TEST_EQUAL(1U, mManager.mRegionCandidateHostsInfosMap.size()); +// auto& infos = mManager.mRegionCandidateHostsInfosMap[region]; // APSARA_TEST_EQUAL(2U, infos.size()); // auto& weakInfo = *(++infos.begin()); // APSARA_TEST_FALSE(weakInfo.expired()); // APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); // } -// APSARA_TEST_EQUAL(2U, mManager->mUnInitializedCandidateHostsInfos.size()); -// APSARA_TEST_EQUAL(info.get(), mManager->mUnInitializedCandidateHostsInfos[1].lock().get()); +// APSARA_TEST_EQUAL(2U, mManager.mUnInitializedCandidateHostsInfos.size()); +// APSARA_TEST_EQUAL(info.get(), mManager.mUnInitializedCandidateHostsInfos[1].lock().get()); // } -// mManager->Clear(); +// mManager.Clear(); // } // { // // candidate host info exists && no region endpoints -// auto info1 = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); -// auto info2 = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); +// auto info1 = mManager.GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); +// auto info2 = mManager.GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); // APSARA_TEST_EQUAL(info1.get(), info2.get()); -// APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); -// APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap[project].size()); -// APSARA_TEST_EQUAL(1U, mManager->mUnInitializedCandidateHostsInfos.size()); -// APSARA_TEST_EQUAL(info1.get(), mManager->mUnInitializedCandidateHostsInfos[0].lock().get()); +// APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap.size()); +// APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap[project].size()); +// APSARA_TEST_EQUAL(1U, mManager.mUnInitializedCandidateHostsInfos.size()); +// APSARA_TEST_EQUAL(info1.get(), mManager.mUnInitializedCandidateHostsInfos[0].lock().get()); -// mManager->Clear(); +// mManager.Clear(); // } // { // // candidate host info exists && with region endpoints && region endpoint mode is default -// mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {privateEndpoint}); -// auto infoDefault = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); -// auto infoAcc = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); -// APSARA_TEST_EQUAL(2U, mManager->mUnInitializedCandidateHostsInfos.size()); -// APSARA_TEST_EQUAL(infoDefault.get(), mManager->mUnInitializedCandidateHostsInfos[0].lock().get()); -// APSARA_TEST_EQUAL(infoAcc.get(), mManager->mUnInitializedCandidateHostsInfos[1].lock().get()); +// mManager.UpdateLocalRegionEndpointsAndHttpsInfo(region, {privateEndpoint}); +// auto infoDefault = mManager.GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); +// auto infoAcc = mManager.GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); +// APSARA_TEST_EQUAL(2U, mManager.mUnInitializedCandidateHostsInfos.size()); +// APSARA_TEST_EQUAL(infoDefault.get(), mManager.mUnInitializedCandidateHostsInfos[0].lock().get()); +// APSARA_TEST_EQUAL(infoAcc.get(), mManager.mUnInitializedCandidateHostsInfos[1].lock().get()); // { // // default mode -// auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); +// auto info = mManager.GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); // APSARA_TEST_EQUAL(infoDefault.get(), info.get()); -// APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); -// APSARA_TEST_EQUAL(2U, mManager->mProjectCandidateHostsInfosMap[project].size()); +// APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap.size()); +// APSARA_TEST_EQUAL(2U, mManager.mProjectCandidateHostsInfosMap[project].size()); // } // { // // accelerate mode -// auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); +// auto info = mManager.GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); // APSARA_TEST_EQUAL(infoAcc.get(), info.get()); -// APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); -// APSARA_TEST_EQUAL(2U, mManager->mProjectCandidateHostsInfosMap[project].size()); +// APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap.size()); +// APSARA_TEST_EQUAL(2U, mManager.mProjectCandidateHostsInfosMap[project].size()); // } -// mManager->Clear(); +// mManager.Clear(); // } // { // // candidate host info exists && with region endpoints && region endpoint mode is accelerate -// mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {globalEndpoint}); -// auto infoAcc = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); -// APSARA_TEST_EQUAL(1U, mManager->mUnInitializedCandidateHostsInfos.size()); -// APSARA_TEST_EQUAL(infoAcc.get(), mManager->mUnInitializedCandidateHostsInfos[0].lock().get()); +// mManager.UpdateLocalRegionEndpointsAndHttpsInfo(region, {globalEndpoint}); +// auto infoAcc = mManager.GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); +// APSARA_TEST_EQUAL(1U, mManager.mUnInitializedCandidateHostsInfos.size()); +// APSARA_TEST_EQUAL(infoAcc.get(), mManager.mUnInitializedCandidateHostsInfos[0].lock().get()); // { // // default mode -// auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); +// auto info = mManager.GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); // APSARA_TEST_EQUAL(infoAcc.get(), info.get()); -// APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); -// APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap[project].size()); +// APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap.size()); +// APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap[project].size()); // } // { // // accelerate mode -// auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); +// auto info = mManager.GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); // APSARA_TEST_EQUAL(infoAcc.get(), info.get()); -// APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); -// APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap[project].size()); +// APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap.size()); +// APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap[project].size()); // } -// mManager->Clear(); +// mManager.Clear(); // } // { // // candidate host info exists && with region endpoints && region endpoint mode is custom -// mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {customEndpoint}); -// auto infoCustom = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); -// auto infoAcc = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); -// APSARA_TEST_EQUAL(2U, mManager->mUnInitializedCandidateHostsInfos.size()); -// APSARA_TEST_EQUAL(infoCustom.get(), mManager->mUnInitializedCandidateHostsInfos[0].lock().get()); -// APSARA_TEST_EQUAL(infoAcc.get(), mManager->mUnInitializedCandidateHostsInfos[1].lock().get()); +// mManager.UpdateLocalRegionEndpointsAndHttpsInfo(region, {customEndpoint}); +// auto infoCustom = mManager.GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); +// auto infoAcc = mManager.GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); +// APSARA_TEST_EQUAL(2U, mManager.mUnInitializedCandidateHostsInfos.size()); +// APSARA_TEST_EQUAL(infoCustom.get(), mManager.mUnInitializedCandidateHostsInfos[0].lock().get()); +// APSARA_TEST_EQUAL(infoAcc.get(), mManager.mUnInitializedCandidateHostsInfos[1].lock().get()); // { // // default mode -// auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); +// auto info = mManager.GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); // APSARA_TEST_EQUAL(infoCustom.get(), info.get()); -// APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); -// APSARA_TEST_EQUAL(2U, mManager->mProjectCandidateHostsInfosMap[project].size()); +// APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap.size()); +// APSARA_TEST_EQUAL(2U, mManager.mProjectCandidateHostsInfosMap[project].size()); // } // { // // accelerate mode -// auto info = mManager->GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); +// auto info = mManager.GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); // APSARA_TEST_EQUAL(infoAcc.get(), info.get()); -// APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); -// APSARA_TEST_EQUAL(2U, mManager->mProjectCandidateHostsInfosMap[project].size()); +// APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap.size()); +// APSARA_TEST_EQUAL(2U, mManager.mProjectCandidateHostsInfosMap[project].size()); // } -// mManager->Clear(); +// mManager.Clear(); // } // } @@ -1094,29 +1094,29 @@ class SLSClientManagerUnittest : public ::testing::Test { // const string endpoint2 = region + "-intranet.log.aliyuncs.com"; // const string host1 = project + "." + endpoint1; // const string host2 = project + "." + endpoint2; -// mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); +// mManager.UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); // { -// auto info = mManager->GetCandidateHostsInfo(region, project, mode); +// auto info = mManager.GetCandidateHostsInfo(region, project, mode); -// APSARA_TEST_TRUE(mManager->UpdateHostInfo(project, mode, host1, chrono::milliseconds(100))); +// APSARA_TEST_TRUE(mManager.UpdateHostInfo(project, mode, host1, chrono::milliseconds(100))); // APSARA_TEST_EQUAL(chrono::milliseconds(100), info->mCandidateHosts[0][0].GetLatency()); // } // // expired info -// APSARA_TEST_FALSE(mManager->UpdateHostInfo(project, mode, host1, chrono::milliseconds(50))); +// APSARA_TEST_FALSE(mManager.UpdateHostInfo(project, mode, host1, chrono::milliseconds(50))); // // unknown project -// APSARA_TEST_FALSE(mManager->UpdateHostInfo("unknown_project", mode, host1, chrono::milliseconds(50))); +// APSARA_TEST_FALSE(mManager.UpdateHostInfo("unknown_project", mode, host1, chrono::milliseconds(50))); // } // void SLSClientManagerUnittest::TestUsingHttps() { // const string region = "region"; // const string host = region + ".log.aliyuncs.com"; -// mManager->UpdateLocalRegionEndpointsAndHttpsInfo(region, {host}); -// APSARA_TEST_FALSE(mManager->UsingHttps(region)); +// mManager.UpdateLocalRegionEndpointsAndHttpsInfo(region, {host}); +// APSARA_TEST_FALSE(mManager.UsingHttps(region)); // AppConfig::GetInstance()->mSendDataPort = 443; -// APSARA_TEST_TRUE(mManager->UsingHttps(region)); +// APSARA_TEST_TRUE(mManager.UsingHttps(region)); // } // void SLSClientManagerUnittest::TestProbeNetwork() { @@ -1132,23 +1132,23 @@ class SLSClientManagerUnittest : public ::testing::Test { // // ! means not available // vector> infos; -// infos.push_back(mManager->GetCandidateHostsInfo( +// infos.push_back(mManager.GetCandidateHostsInfo( // ProbeNetworkMock::region1, ProbeNetworkMock::project1, EndpointMode::DEFAULT)); // public !internal !private -// infos.push_back(mManager->GetCandidateHostsInfo( +// infos.push_back(mManager.GetCandidateHostsInfo( // ProbeNetworkMock::region1, ProbeNetworkMock::project1, EndpointMode::ACCELERATE)); // !accelerate public -// infos.push_back(mManager->GetCandidateHostsInfo( +// infos.push_back(mManager.GetCandidateHostsInfo( // ProbeNetworkMock::region2, ProbeNetworkMock::project2, EndpointMode::DEFAULT)); // accelerate public -// infos.push_back(mManager->GetCandidateHostsInfo( +// infos.push_back(mManager.GetCandidateHostsInfo( // ProbeNetworkMock::region3, ProbeNetworkMock::project3, EndpointMode::DEFAULT)); // custom -// infos.push_back(mManager->GetCandidateHostsInfo( +// infos.push_back(mManager.GetCandidateHostsInfo( // ProbeNetworkMock::region3, ProbeNetworkMock::project3, EndpointMode::ACCELERATE)); // !accelerate // // probe uninitialized host -// mManager->DoProbeUnInitializedHost(); -// APSARA_TEST_TRUE(mManager->mUnInitializedCandidateHostsInfos.empty()); -// APSARA_TEST_EQUAL(infos.size(), mManager->mPartiallyInitializedCandidateHostsInfos.size()); +// mManager.DoProbeUnInitializedHost(); +// APSARA_TEST_TRUE(mManager.mUnInitializedCandidateHostsInfos.empty()); +// APSARA_TEST_EQUAL(infos.size(), mManager.mPartiallyInitializedCandidateHostsInfos.size()); // for (size_t i = 0; i < infos.size(); ++i) { -// APSARA_TEST_EQUAL(infos[i].get(), mManager->mPartiallyInitializedCandidateHostsInfos[i].lock().get()); +// APSARA_TEST_EQUAL(infos[i].get(), mManager.mPartiallyInitializedCandidateHostsInfos[i].lock().get()); // } // APSARA_TEST_EQUAL(publicHost1, infos[0]->GetCurrentHost()); @@ -1164,8 +1164,8 @@ class SLSClientManagerUnittest : public ::testing::Test { // } // // probe partially initialized host -// mManager->DoProbeHost(); -// APSARA_TEST_TRUE(mManager->mPartiallyInitializedCandidateHostsInfos.empty()); +// mManager.DoProbeHost(); +// APSARA_TEST_TRUE(mManager.mPartiallyInitializedCandidateHostsInfos.empty()); // APSARA_TEST_EQUAL(publicHost1, infos[0]->GetCurrentHost()); // APSARA_TEST_EQUAL(publicHost1, infos[1]->GetCurrentHost()); @@ -1191,7 +1191,7 @@ class SLSClientManagerUnittest : public ::testing::Test { // // probe all avaialble hosts // INT32_FLAG(sls_hosts_probe_interval_sec) = 0; -// mManager->DoProbeHost(); +// mManager.DoProbeHost(); // APSARA_TEST_EQUAL(publicHost1, infos[0]->GetCurrentHost()); // APSARA_TEST_EQUAL(publicHost1, infos[1]->GetCurrentHost()); @@ -1229,7 +1229,7 @@ class SLSClientManagerUnittest : public ::testing::Test { // // probe all hosts // INT32_FLAG(sls_all_hosts_probe_interval_sec) = 0; // ProbeNetworkMock::enableUnavailableHosts = true; -// mManager->DoProbeHost(); +// mManager.DoProbeHost(); // APSARA_TEST_EQUAL(publicHost1, infos[0]->GetCurrentHost()); // APSARA_TEST_EQUAL(globalHost1, infos[1]->GetCurrentHost()); @@ -1249,86 +1249,86 @@ class SLSClientManagerUnittest : public ::testing::Test { // // expired info // infos[2].reset(); // infos[4].reset(); -// mManager->DoProbeHost(); -// APSARA_TEST_EQUAL(2U, mManager->mProjectCandidateHostsInfosMap.size()); -// APSARA_TEST_EQUAL(2U, mManager->mProjectCandidateHostsInfosMap[ProbeNetworkMock::project1].size()); -// APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap[ProbeNetworkMock::project3].size()); -// APSARA_TEST_EQUAL(2U, mManager->mRegionCandidateHostsInfosMap.size()); -// APSARA_TEST_EQUAL(2U, mManager->mRegionCandidateHostsInfosMap[ProbeNetworkMock::region1].size()); -// APSARA_TEST_EQUAL(1U, mManager->mRegionCandidateHostsInfosMap[ProbeNetworkMock::region3].size()); +// mManager.DoProbeHost(); +// APSARA_TEST_EQUAL(2U, mManager.mProjectCandidateHostsInfosMap.size()); +// APSARA_TEST_EQUAL(2U, mManager.mProjectCandidateHostsInfosMap[ProbeNetworkMock::project1].size()); +// APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap[ProbeNetworkMock::project3].size()); +// APSARA_TEST_EQUAL(2U, mManager.mRegionCandidateHostsInfosMap.size()); +// APSARA_TEST_EQUAL(2U, mManager.mRegionCandidateHostsInfosMap[ProbeNetworkMock::region1].size()); +// APSARA_TEST_EQUAL(1U, mManager.mRegionCandidateHostsInfosMap[ProbeNetworkMock::region3].size()); // } // void SLSClientManagerUnittest::TestRealIp() { // BOOL_FLAG(send_prefer_real_ip) = true; -// mManager->mDoProbeNetwork = GetRealIpMock::DoProbeNetwork; +// mManager.mDoProbeNetwork = GetRealIpMock::DoProbeNetwork; // const string unknownRegion = "unknown_region"; // const string normalRegionEndpoint1 = GetRealIpMock::normalRegion + "-intranet.log.aliyuncs.com"; // const string normalRegionEndpoint2 = GetRealIpMock::normalRegion + ".log.aliyuncs.com"; // const string unavailableRegionEndpoint = GetRealIpMock::unavailableRegion + ".log.aliyuncs.com"; // const string unknownRegionEndpoint = unknownRegion + ".log.aliyuncs.com"; -// mManager->UpdateRemoteRegionEndpoints(GetRealIpMock::normalRegion, {normalRegionEndpoint1, -// normalRegionEndpoint2}); mManager->UpdateRemoteRegionEndpoints(GetRealIpMock::unavailableRegion, -// {unavailableRegionEndpoint}); mManager->UpdateRemoteRegionEndpoints(unknownRegion, {unknownRegionEndpoint}); -// APSARA_TEST_EQUAL(3U, mManager->mRegionRealIpCandidateHostsInfosMap.size()); +// mManager.UpdateRemoteRegionEndpoints(GetRealIpMock::normalRegion, {normalRegionEndpoint1, +// normalRegionEndpoint2}); mManager.UpdateRemoteRegionEndpoints(GetRealIpMock::unavailableRegion, +// {unavailableRegionEndpoint}); mManager.UpdateRemoteRegionEndpoints(unknownRegion, {unknownRegionEndpoint}); +// APSARA_TEST_EQUAL(3U, mManager.mRegionRealIpCandidateHostsInfosMap.size()); // { // vector hosts; -// mManager->mRegionRealIpCandidateHostsInfosMap.at(GetRealIpMock::normalRegion).GetAllHosts(hosts); +// mManager.mRegionRealIpCandidateHostsInfosMap.at(GetRealIpMock::normalRegion).GetAllHosts(hosts); // APSARA_TEST_EQUAL(2U, hosts.size()); // APSARA_TEST_EQUAL(normalRegionEndpoint1, hosts[0]); // APSARA_TEST_EQUAL(normalRegionEndpoint2, hosts[1]); // } // { // vector hosts; -// mManager->mRegionRealIpCandidateHostsInfosMap.at(GetRealIpMock::unavailableRegion).GetAllHosts(hosts); +// mManager.mRegionRealIpCandidateHostsInfosMap.at(GetRealIpMock::unavailableRegion).GetAllHosts(hosts); // APSARA_TEST_EQUAL(1U, hosts.size()); // APSARA_TEST_EQUAL(unavailableRegionEndpoint, hosts[0]); // } // { // vector hosts; -// mManager->mRegionRealIpCandidateHostsInfosMap.at(unknownRegion).GetAllHosts(hosts); +// mManager.mRegionRealIpCandidateHostsInfosMap.at(unknownRegion).GetAllHosts(hosts); // APSARA_TEST_EQUAL(1U, hosts.size()); // APSARA_TEST_EQUAL(unknownRegionEndpoint, hosts[0]); // } // // no endpoint available -// mManager->DoUpdateRealIp(); -// APSARA_TEST_EQUAL(0U, mManager->mRegionRealIpMap.size()); +// mManager.DoUpdateRealIp(); +// APSARA_TEST_EQUAL(0U, mManager.mRegionRealIpMap.size()); // // probe host // INT32_FLAG(sls_all_hosts_probe_interval_sec) = 0; -// mManager->DoProbeHost(); +// mManager.DoProbeHost(); // INT32_FLAG(sls_all_hosts_probe_interval_sec) = 300; -// for (const auto& item : mManager->mRegionRealIpCandidateHostsInfosMap) { +// for (const auto& item : mManager.mRegionRealIpCandidateHostsInfosMap) { // APSARA_TEST_NOT_EQUAL("", item.second.GetCurrentHost()); // } // // update all regions // INT32_FLAG(send_switch_real_ip_interval) = 0; -// mManager->UpdateOutdatedRealIpRegions(GetRealIpMock::unavailableRegion); -// mManager->DoUpdateRealIp(); -// APSARA_TEST_TRUE(mManager->mOutdatedRealIpRegions.empty()); -// APSARA_TEST_EQUAL(2U, mManager->mRegionRealIpMap.size()); -// APSARA_TEST_EQUAL(GetRealIpMock::normalRegionIps[0], mManager->GetRealIp(GetRealIpMock::normalRegion)); -// APSARA_TEST_EQUAL(GetRealIpMock::unavailableRegionIps[0], mManager->GetRealIp(GetRealIpMock::unavailableRegion)); +// mManager.UpdateOutdatedRealIpRegions(GetRealIpMock::unavailableRegion); +// mManager.DoUpdateRealIp(); +// APSARA_TEST_TRUE(mManager.mOutdatedRealIpRegions.empty()); +// APSARA_TEST_EQUAL(2U, mManager.mRegionRealIpMap.size()); +// APSARA_TEST_EQUAL(GetRealIpMock::normalRegionIps[0], mManager.GetRealIp(GetRealIpMock::normalRegion)); +// APSARA_TEST_EQUAL(GetRealIpMock::unavailableRegionIps[0], mManager.GetRealIp(GetRealIpMock::unavailableRegion)); // INT32_FLAG(send_switch_real_ip_interval) = 60; // // update only outdated regions -// mManager->UpdateOutdatedRealIpRegions(GetRealIpMock::normalRegion); -// mManager->DoUpdateRealIp(); -// APSARA_TEST_TRUE(mManager->mOutdatedRealIpRegions.empty()); -// APSARA_TEST_EQUAL(2U, mManager->mRegionRealIpMap.size()); -// APSARA_TEST_EQUAL(GetRealIpMock::normalRegionIps[1], mManager->GetRealIp(GetRealIpMock::normalRegion)); -// APSARA_TEST_EQUAL(GetRealIpMock::unavailableRegionIps[0], mManager->GetRealIp(GetRealIpMock::unavailableRegion)); - -// mManager->mDoProbeNetwork = ProbeNetworkMock::DoProbeNetwork; +// mManager.UpdateOutdatedRealIpRegions(GetRealIpMock::normalRegion); +// mManager.DoUpdateRealIp(); +// APSARA_TEST_TRUE(mManager.mOutdatedRealIpRegions.empty()); +// APSARA_TEST_EQUAL(2U, mManager.mRegionRealIpMap.size()); +// APSARA_TEST_EQUAL(GetRealIpMock::normalRegionIps[1], mManager.GetRealIp(GetRealIpMock::normalRegion)); +// APSARA_TEST_EQUAL(GetRealIpMock::unavailableRegionIps[0], mManager.GetRealIp(GetRealIpMock::unavailableRegion)); + +// mManager.mDoProbeNetwork = ProbeNetworkMock::DoProbeNetwork; // BOOL_FLAG(send_prefer_real_ip) = false; // } void SLSClientManagerUnittest::TestAccessKeyManagement() { string accessKeyId, accessKeySecret; SLSClientManager::AuthType type; - mManager->GetAccessKey("", type, accessKeyId, accessKeySecret); + mManager.GetAccessKey("", type, accessKeyId, accessKeySecret); APSARA_TEST_EQUAL(SLSClientManager::AuthType::AK, type); APSARA_TEST_EQUAL(STRING_FLAG(default_access_key_id), accessKeyId); APSARA_TEST_EQUAL(STRING_FLAG(default_access_key), accessKeySecret); @@ -1339,29 +1339,31 @@ void SLSClientManagerUnittest::TestGetCandidateHostsInfo() { const string endpoint = "endpoint"; CandidateHostsInfo* infoPtr = nullptr; { - auto info1 = mManager->GetCandidateHostsInfo(project, endpoint); - auto info2 = mManager->GetCandidateHostsInfo(project, endpoint); + auto info1 = mManager.GetCandidateHostsInfo(project, endpoint); + auto info2 = mManager.GetCandidateHostsInfo(project, endpoint); infoPtr = info1.get(); APSARA_TEST_EQUAL(info1.get(), info2.get()); APSARA_TEST_EQUAL(project, info1->GetProject()); APSARA_TEST_EQUAL("", info1->GetRegion()); APSARA_TEST_EQUAL(EndpointMode::CUSTOM, info1->GetMode()); + APSARA_TEST_TRUE(info1->IsInitialized()); + APSARA_TEST_EQUAL(project + "." + endpoint, info1->GetCurrentHost()); APSARA_TEST_EQUAL(1U, info1->mCandidateHosts.size()); APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[0].size()); APSARA_TEST_EQUAL(project + "." + endpoint, info1->mCandidateHosts[0][0].GetHostname()); - APSARA_TEST_EQUAL(chrono::milliseconds::max(), info1->mCandidateHosts[0][0].GetLatency()); - APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); - auto& infos = mManager->mProjectCandidateHostsInfosMap[project]; + APSARA_TEST_EQUAL(chrono::milliseconds(10), info1->mCandidateHosts[0][0].GetLatency()); + APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap.size()); + auto& infos = mManager.mProjectCandidateHostsInfosMap[project]; APSARA_TEST_EQUAL(1U, infos.size()); auto& weakInfo = *infos.begin(); APSARA_TEST_FALSE(weakInfo.expired()); APSARA_TEST_EQUAL(info1.get(), weakInfo.lock().get()); } { - auto info = mManager->GetCandidateHostsInfo(project, endpoint); + auto info = mManager.GetCandidateHostsInfo(project, endpoint); APSARA_TEST_NOT_EQUAL(infoPtr, info.get()); - APSARA_TEST_EQUAL(1U, mManager->mProjectCandidateHostsInfosMap.size()); - APSARA_TEST_EQUAL(2U, mManager->mProjectCandidateHostsInfosMap[project].size()); + APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap.size()); + APSARA_TEST_EQUAL(2U, mManager.mProjectCandidateHostsInfosMap[project].size()); } } diff --git a/core/unittest/plugin/PluginMock.h b/core/unittest/plugin/PluginMock.h index 14c95c8ce3..a20e9fdce4 100644 --- a/core/unittest/plugin/PluginMock.h +++ b/core/unittest/plugin/PluginMock.h @@ -128,7 +128,7 @@ class FlusherHttpMock : public HttpFlusher { return true; } bool FlushAll() override { return mIsValid; } - bool BuildRequest(SenderQueueItem* item, std::unique_ptr& req, bool* keepItem) const override { + bool BuildRequest(SenderQueueItem* item, std::unique_ptr& req, bool* keepItem) override { if (item->mData == "invalid_keep") { *keepItem = true; return false; diff --git a/docs/cn/developer-guide/plugin-development/native-plugins/how-to-write-native-flusher-plugins.md b/docs/cn/developer-guide/plugin-development/native-plugins/how-to-write-native-flusher-plugins.md index 031b1ed76c..048a1d1386 100644 --- a/docs/cn/developer-guide/plugin-development/native-plugins/how-to-write-native-flusher-plugins.md +++ b/docs/cn/developer-guide/plugin-development/native-plugins/how-to-write-native-flusher-plugins.md @@ -24,7 +24,7 @@ public: class HttpFlusher : public Flusher { public: // 用于将待发送数据打包成http请求 - virtual bool BuildRequest(SenderQueueItem* item, std::unique_ptr& req, bool* keepItem) const = 0; + virtual bool BuildRequest(SenderQueueItem* item, std::unique_ptr& req, bool* keepItem) = 0; // 用于发送完成后进行记录和处理 virtual void OnSendDone(const HttpResponse& response, SenderQueueItem* item) = 0; }; From 1977793d5a683f8e25010a6828b06ac27ea1f3a7 Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Tue, 10 Dec 2024 11:34:03 +0000 Subject: [PATCH 08/41] polish --- core/plugin/flusher/sls/DiskBufferWriter.cpp | 4 +++- core/plugin/flusher/sls/FlusherSLS.h | 2 ++ core/plugin/flusher/sls/SLSClientManager.cpp | 13 ------------- core/plugin/flusher/sls/SLSClientManager.h | 2 -- 4 files changed, 5 insertions(+), 16 deletions(-) diff --git a/core/plugin/flusher/sls/DiskBufferWriter.cpp b/core/plugin/flusher/sls/DiskBufferWriter.cpp index e3c46e86c9..f91478f79e 100644 --- a/core/plugin/flusher/sls/DiskBufferWriter.cpp +++ b/core/plugin/flusher/sls/DiskBufferWriter.cpp @@ -798,10 +798,12 @@ SLSResponse DiskBufferWriter::SendBufferFileData(const sls_logs::LogtailBufferMe std::string& host) { RateLimiter::FlowControl(bufferMeta.rawsize(), mSendLastTime, mSendLastByte, false); string region = bufferMeta.region(); +#ifdef __ENTERPRISE__ // old buffer file which record the endpoint if (region.find("http://") == 0) { - region = SLSClientManager::GetInstance()->GetRegionFromEndpoint(region); + region = EnterpriseSLSClientManager::GetInstance()->GetRegionFromEndpoint(region); } +#endif SLSClientManager::AuthType type; string accessKeyId, accessKeySecret; diff --git a/core/plugin/flusher/sls/FlusherSLS.h b/core/plugin/flusher/sls/FlusherSLS.h index 01fa082a85..e742445a3d 100644 --- a/core/plugin/flusher/sls/FlusherSLS.h +++ b/core/plugin/flusher/sls/FlusherSLS.h @@ -128,6 +128,8 @@ class FlusherSLS : public HttpFlusher { Batcher mBatcher; std::unique_ptr mGroupSerializer; std::unique_ptr>> mGroupListSerializer; + // This may not be cached. However, this provides a simple way to control the lifetime of a CandidateHostsInfo. + // Otherwise, timeout machanisim must be emplyed to clean up unused CandidateHostsInfo. std::shared_ptr mCandidateHostsInfo; CounterPtr mSendCnt; diff --git a/core/plugin/flusher/sls/SLSClientManager.cpp b/core/plugin/flusher/sls/SLSClientManager.cpp index 59f3aae789..e6e382edb6 100644 --- a/core/plugin/flusher/sls/SLSClientManager.cpp +++ b/core/plugin/flusher/sls/SLSClientManager.cpp @@ -661,19 +661,6 @@ bool SLSClientManager::UsingHttps(const string& region) const { // return mHttpsRegions.find(region) != mHttpsRegions.end(); } -string SLSClientManager::GetRegionFromEndpoint(const string& endpoint) { - // lock_guard lock(mRegionEndpointEntryMapLock); - // for (auto iter = mRegionEndpointEntryMap.begin(); iter != mRegionEndpointEntryMap.end(); ++iter) { - // for (auto epIter = ((iter->second).mEndpointInfoMap).begin(); epIter != - // ((iter->second).mEndpointInfoMap).end(); - // ++epIter) { - // if (epIter->first == endpoint) - // return iter->first; - // } - // } - return STRING_FLAG(default_region_name); -} - // void SLSClientManager::UnInitializedHostProbeThread() { // LOG_INFO(sLogger, ("sls uninitialized host probe", "started")); // unique_lock lock(mUnInitializedHostProbeThreadRunningMux); diff --git a/core/plugin/flusher/sls/SLSClientManager.h b/core/plugin/flusher/sls/SLSClientManager.h index 7afee13026..1e87509031 100644 --- a/core/plugin/flusher/sls/SLSClientManager.h +++ b/core/plugin/flusher/sls/SLSClientManager.h @@ -159,8 +159,6 @@ class SLSClientManager { // void UpdateOutdatedRealIpRegions(const std::string& region); // std::string GetRealIp(const std::string& region) const; - std::string GetRegionFromEndpoint(const std::string& endpoint); // for backward compatibility - #ifdef APSARA_UNIT_TEST_MAIN virtual void Clear(); #endif From 4324e6424f372d9458ae67f55dd3f5573c6a2c01 Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Wed, 11 Dec 2024 11:20:00 +0000 Subject: [PATCH 09/41] polish --- core/plugin/flusher/sls/FlusherSLS.cpp | 7 +- core/plugin/flusher/sls/SLSClientManager.cpp | 2 +- core/unittest/flusher/CMakeLists.txt | 6 +- core/unittest/flusher/FlusherSLSUnittest.cpp | 617 ++++++++++++++++-- .../flusher/SLSClientManagerUnittest.cpp | 2 + 5 files changed, 569 insertions(+), 65 deletions(-) diff --git a/core/plugin/flusher/sls/FlusherSLS.cpp b/core/plugin/flusher/sls/FlusherSLS.cpp index 5e0c47d8fd..6d082e3eb2 100644 --- a/core/plugin/flusher/sls/FlusherSLS.cpp +++ b/core/plugin/flusher/sls/FlusherSLS.cpp @@ -593,7 +593,6 @@ bool FlusherSLS::BuildRequest(SenderQueueItem* item, unique_ptr return false; } - auto data = static_cast(item); #ifdef __ENTERPRISE__ if (BOOL_FLAG(send_prefer_real_ip)) { @@ -629,12 +628,12 @@ bool FlusherSLS::BuildRequest(SenderQueueItem* item, unique_ptr } switch (mTelemetryType) { - case sls_logs::SLS_TELEMETRY_TYPE_METRICS: - req = CreatePostMetricStoreLogsRequest(accessKeyId, accessKeySecret, type, data); - break; case sls_logs::SLS_TELEMETRY_TYPE_LOGS: req = CreatePostLogStoreLogsRequest(accessKeyId, accessKeySecret, type, data); break; + case sls_logs::SLS_TELEMETRY_TYPE_METRICS: + req = CreatePostMetricStoreLogsRequest(accessKeyId, accessKeySecret, type, data); + break; default: break; } diff --git a/core/plugin/flusher/sls/SLSClientManager.cpp b/core/plugin/flusher/sls/SLSClientManager.cpp index e6e382edb6..61dc6a87f7 100644 --- a/core/plugin/flusher/sls/SLSClientManager.cpp +++ b/core/plugin/flusher/sls/SLSClientManager.cpp @@ -1115,7 +1115,7 @@ void PreparePostLogStoreLogsRequest(const string& accessKeyId, map parameterList; if (!shardHashKey.empty()) { parameterList["key"] = shardHashKey; - if (!seqId.has_value()) { + if (seqId.has_value()) { parameterList["seqid"] = to_string(seqId.value()); } } diff --git a/core/unittest/flusher/CMakeLists.txt b/core/unittest/flusher/CMakeLists.txt index 5c63884d14..177a86fcc1 100644 --- a/core/unittest/flusher/CMakeLists.txt +++ b/core/unittest/flusher/CMakeLists.txt @@ -16,6 +16,9 @@ cmake_minimum_required(VERSION 3.22) project(flusher_unittest) add_executable(flusher_sls_unittest FlusherSLSUnittest.cpp) +if (ENABLE_ENTERPRISE) + target_sources(flusher_sls_unittest PRIVATE SLSNetworkRequestMock.cpp) +endif () target_link_libraries(flusher_sls_unittest ${UT_BASE_TARGET}) add_executable(pack_id_manager_unittest PackIdManagerUnittest.cpp) @@ -25,8 +28,9 @@ add_executable(sls_client_manager_unittest SLSClientManagerUnittest.cpp) target_link_libraries(sls_client_manager_unittest ${UT_BASE_TARGET}) if (ENABLE_ENTERPRISE) - add_executable(enterprise_sls_client_manager_unittest EnterpriseSLSClientManagerUnittest.cpp) + add_executable(enterprise_sls_client_manager_unittest EnterpriseSLSClientManagerUnittest.cpp SLSNetworkRequestMock.cpp) target_link_libraries(enterprise_sls_client_manager_unittest ${UT_BASE_TARGET}) + add_executable(enterprise_flusher_sls_monitor_unittest EnterpriseFlusherSLSMonitorUnittest.cpp) target_link_libraries(enterprise_flusher_sls_monitor_unittest ${UT_BASE_TARGET}) endif () diff --git a/core/unittest/flusher/FlusherSLSUnittest.cpp b/core/unittest/flusher/FlusherSLSUnittest.cpp index 0b34e257a8..cdb02dab1d 100644 --- a/core/unittest/flusher/FlusherSLSUnittest.cpp +++ b/core/unittest/flusher/FlusherSLSUnittest.cpp @@ -17,8 +17,10 @@ #include #include +#include "app_config/AppConfig.h" #include "common/JsonUtil.h" #include "common/LogtailCommonFlags.h" +#include "common/http/Constant.h" #ifdef __ENTERPRISE__ #include "config/provider/EnterpriseConfigProvider.h" #endif @@ -36,13 +38,20 @@ #include "plugin/flusher/sls/FlusherSLS.h" #include "plugin/flusher/sls/PackIdManager.h" #include "plugin/flusher/sls/SLSClientManager.h" +#include "plugin/flusher/sls/SLSConstant.h" #include "unittest/Unittest.h" +#ifdef __ENTERPRISE__ +#include "unittest/flusher/SLSNetworkRequestMock.h" +#endif DECLARE_FLAG_INT32(batch_send_interval); DECLARE_FLAG_INT32(merge_log_count_limit); DECLARE_FLAG_INT32(batch_send_metric_size); DECLARE_FLAG_INT32(max_send_log_group_size); DECLARE_FLAG_DOUBLE(sls_serialize_size_expansion_ratio); +DECLARE_FLAG_BOOL(send_prefer_real_ip); +DECLARE_FLAG_STRING(default_access_key_id); +DECLARE_FLAG_STRING(default_access_key); using namespace std; @@ -53,6 +62,7 @@ class FlusherSLSUnittest : public testing::Test { void OnSuccessfulInit(); void OnFailedInit(); void OnPipelineUpdate(); + void TestBuildRequest(); void TestSend(); void TestFlush(); void TestFlushAll(); @@ -60,6 +70,14 @@ class FlusherSLSUnittest : public testing::Test { void OnGoPipelineSend(); protected: + static void SetUpTestCase() { +#ifdef __ENTERPRISE__ + EnterpriseSLSClientManager::GetInstance()->mDoProbeNetwork = ProbeNetworkMock::DoProbeNetwork; + EnterpriseSLSClientManager::GetInstance()->mGetEndpointRealIp = GetRealIpMock::GetEndpointRealIp; + EnterpriseSLSClientManager::GetInstance()->mGetAccessKeyFromSLS = GetAccessKeyMock::DoGetAccessKey; +#endif + } + void SetUp() override { ctx.SetConfigName("test_config"); ctx.SetPipeline(pipeline); @@ -70,6 +88,7 @@ class FlusherSLSUnittest : public testing::Test { QueueKeyManager::GetInstance()->Clear(); SenderQueueManager::GetInstance()->Clear(); ExactlyOnceQueueManager::GetInstance()->Clear(); + SLSClientManager::GetInstance()->Clear(); } private: @@ -92,7 +111,7 @@ void FlusherSLSUnittest::OnSuccessfulInit() { )"; APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); #ifndef __ENTERPRISE__ - configJson["Endpoint"] = "cn-hangzhou.log.aliyuncs.com"; + configJson["Endpoint"] = "test_region.log.aliyuncs.com"; #endif flusher.reset(new FlusherSLS()); flusher->SetContext(ctx); @@ -103,8 +122,8 @@ void FlusherSLSUnittest::OnSuccessfulInit() { APSARA_TEST_EQUAL("test_logstore", flusher->mLogstore); APSARA_TEST_EQUAL(STRING_FLAG(default_region_name), flusher->mRegion); #ifndef __ENTERPRISE__ - APSARA_TEST_EQUAL("cn-hangzhou.log.aliyuncs.com", flusher->mEndpoint); - APSARA_TEST_EQUAL("test_project.cn-hangzhou.log.aliyuncs.com", flusher->mCandidateHostsInfo->GetFirstHost()); + APSARA_TEST_EQUAL("test_region.log.aliyuncs.com", flusher->mEndpoint); + APSARA_TEST_EQUAL("test_project.test_region.log.aliyuncs.com", flusher->mCandidateHostsInfo->GetFirstHost()); #endif APSARA_TEST_EQUAL("", flusher->mAliuid); APSARA_TEST_EQUAL(sls_logs::SlsTelemetryType::SLS_TELEMETRY_TYPE_LOGS, flusher->mTelemetryType); @@ -139,8 +158,8 @@ void FlusherSLSUnittest::OnSuccessfulInit() { "Type": "flusher_sls", "Project": "test_project", "Logstore": "test_logstore", - "Region": "cn-hangzhou", - "Endpoint": "cn-hangzhou.log.aliyuncs.com", + "Region": "test_region", + "Endpoint": "test_region.log.aliyuncs.com", "Aliuid": "123456789", "TelemetryType": "metrics", "ShardHashKeys": [ @@ -156,14 +175,14 @@ void FlusherSLSUnittest::OnSuccessfulInit() { flusher->SetContext(ctx); flusher->SetMetricsRecordRef(FlusherSLS::sName, "1"); APSARA_TEST_TRUE(flusher->Init(configJson, optionalGoPipeline)); - APSARA_TEST_EQUAL("cn-hangzhou", flusher->mRegion); + APSARA_TEST_EQUAL("test_region", flusher->mRegion); #ifdef __ENTERPRISE__ APSARA_TEST_EQUAL("123456789", flusher->mAliuid); APSARA_TEST_EQUAL(EndpointMode::DEFAULT, flusher->mEndpointMode); #else APSARA_TEST_EQUAL("", flusher->mAliuid); #endif - APSARA_TEST_EQUAL("cn-hangzhou.log.aliyuncs.com", flusher->mEndpoint); + APSARA_TEST_EQUAL("test_region.log.aliyuncs.com", flusher->mEndpoint); APSARA_TEST_EQUAL(sls_logs::SlsTelemetryType::SLS_TELEMETRY_TYPE_METRICS, flusher->mTelemetryType); APSARA_TEST_EQUAL(1U, flusher->mShardHashKeys.size()); APSARA_TEST_EQUAL("__source__", flusher->mShardHashKeys[0]); @@ -186,7 +205,7 @@ void FlusherSLSUnittest::OnSuccessfulInit() { configJson["EndpointMode"] = true; configJson["Endpoint"] = true; #else - configJson["Endpoint"] = "cn-hangzhou.log.aliyuncs.com"; + configJson["Endpoint"] = "test_region.log.aliyuncs.com"; #endif flusher.reset(new FlusherSLS()); flusher->SetContext(ctx); @@ -210,8 +229,8 @@ void FlusherSLSUnittest::OnSuccessfulInit() { "Type": "flusher_sls", "Project": "test_project", "Logstore": "test_logstore", - "Region": "cn-hangzhou", - "Endpoint": "cn-hangzhou.log.aliyuncs.com" + "Region": "test_region", + "Endpoint": "test_region.log.aliyuncs.com" } )"; APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); @@ -233,8 +252,9 @@ void FlusherSLSUnittest::OnSuccessfulInit() { "Type": "flusher_sls", "Project": "test_project", "Logstore": "test_logstore", + "Region": "test_region", "EndpointMode": "accelerate", - "Endpoint": " cn-hangzhou.log.aliyuncs.com " + "Endpoint": " test_region.log.aliyuncs.com " } )"; APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); @@ -243,9 +263,8 @@ void FlusherSLSUnittest::OnSuccessfulInit() { flusher->SetMetricsRecordRef(FlusherSLS::sName, "1"); APSARA_TEST_TRUE(flusher->Init(configJson, optionalGoPipeline)); APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, flusher->mEndpointMode); - APSARA_TEST_TRUE(EnterpriseSLSClientManager::GetInstance() - ->mRegionCandidateEndpointsMap["cn-hangzhou"] - .mRemoteEndpoints.empty()); + APSARA_TEST_EQUAL(EnterpriseSLSClientManager::GetInstance()->mRegionCandidateEndpointsMap.end(), + EnterpriseSLSClientManager::GetInstance()->mRegionCandidateEndpointsMap.find("test_region")); APSARA_TEST_EQUAL(flusher->mProject, flusher->mCandidateHostsInfo->GetProject()); APSARA_TEST_EQUAL(flusher->mRegion, flusher->mCandidateHostsInfo->GetRegion()); APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, flusher->mCandidateHostsInfo->GetMode()); @@ -253,12 +272,13 @@ void FlusherSLSUnittest::OnSuccessfulInit() { // Endpoint should be added to region remote endpoints if not existed configStr = R"( - { + { "Type": "flusher_sls", "Project": "test_project", "Logstore": "test_logstore", + "Region": "test_region", "EndpointMode": "unknown", - "Endpoint": " cn-hangzhou.log.aliyuncs.com " + "Endpoint": " test_region.log.aliyuncs.com " } )"; APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); @@ -268,9 +288,9 @@ void FlusherSLSUnittest::OnSuccessfulInit() { APSARA_TEST_TRUE(flusher->Init(configJson, optionalGoPipeline)); APSARA_TEST_EQUAL(EndpointMode::DEFAULT, flusher->mEndpointMode); auto& endpoints - = EnterpriseSLSClientManager::GetInstance()->mRegionCandidateEndpointsMap["cn-hangzhou"].mRemoteEndpoints; + = EnterpriseSLSClientManager::GetInstance()->mRegionCandidateEndpointsMap["test_region"].mRemoteEndpoints; APSARA_TEST_EQUAL(1U, endpoints.size()); - APSARA_TEST_EQUAL("cn-hangzhou.log.aliyuncs.com", endpoints[0]); + APSARA_TEST_EQUAL("test_region.log.aliyuncs.com", endpoints[0]); APSARA_TEST_EQUAL(flusher->mProject, flusher->mCandidateHostsInfo->GetProject()); APSARA_TEST_EQUAL(flusher->mRegion, flusher->mCandidateHostsInfo->GetRegion()); APSARA_TEST_EQUAL(EndpointMode::DEFAULT, flusher->mCandidateHostsInfo->GetMode()); @@ -278,12 +298,13 @@ void FlusherSLSUnittest::OnSuccessfulInit() { // Endpoint should be ignored when region remote endpoints existed configStr = R"( - { + { "Type": "flusher_sls", "Project": "test_project", "Logstore": "test_logstore", + "Region": "test_region", "EndpointMode": "default", - "Endpoint": " cn-hangzhou-intranet.log.aliyuncs.com " + "Endpoint": " test_region-intranet.log.aliyuncs.com " } )"; APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); @@ -292,9 +313,9 @@ void FlusherSLSUnittest::OnSuccessfulInit() { flusher->SetMetricsRecordRef(FlusherSLS::sName, "1"); APSARA_TEST_TRUE(flusher->Init(configJson, optionalGoPipeline)); APSARA_TEST_EQUAL(EndpointMode::DEFAULT, flusher->mEndpointMode); - endpoints = EnterpriseSLSClientManager::GetInstance()->mRegionCandidateEndpointsMap["cn-hangzhou"].mRemoteEndpoints; + endpoints = EnterpriseSLSClientManager::GetInstance()->mRegionCandidateEndpointsMap["test_region"].mRemoteEndpoints; APSARA_TEST_EQUAL(1U, endpoints.size()); - APSARA_TEST_EQUAL("cn-hangzhou.log.aliyuncs.com", endpoints[0]); + APSARA_TEST_EQUAL("test_region.log.aliyuncs.com", endpoints[0]); APSARA_TEST_EQUAL(flusher->mProject, flusher->mCandidateHostsInfo->GetProject()); APSARA_TEST_EQUAL(flusher->mRegion, flusher->mCandidateHostsInfo->GetRegion()); APSARA_TEST_EQUAL(EndpointMode::DEFAULT, flusher->mCandidateHostsInfo->GetMode()); @@ -308,8 +329,8 @@ void FlusherSLSUnittest::OnSuccessfulInit() { "Type": "flusher_sls", "Project": "test_project", "Logstore": "test_logstore", - "Region": "cn-hangzhou", - "Endpoint": " cn-hangzhou.log.aliyuncs.com " + "Region": "test_region", + "Endpoint": " test_region.log.aliyuncs.com " } )"; APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); @@ -317,7 +338,7 @@ void FlusherSLSUnittest::OnSuccessfulInit() { flusher->SetContext(ctx); flusher->SetMetricsRecordRef(FlusherSLS::sName, "1"); APSARA_TEST_TRUE(flusher->Init(configJson, optionalGoPipeline)); - APSARA_TEST_EQUAL("test_project.cn-hangzhou.log.aliyuncs.com", flusher->mCandidateHostsInfo->GetFirstHost()); + APSARA_TEST_EQUAL("test_project.test_region.log.aliyuncs.com", flusher->mCandidateHostsInfo->GetFirstHost()); SenderQueueManager::GetInstance()->Clear(); #endif @@ -327,8 +348,8 @@ void FlusherSLSUnittest::OnSuccessfulInit() { "Type": "flusher_sls", "Project": "test_project", "Logstore": "test_logstore", - "Region": "cn-hangzhou", - "Endpoint": "cn-hangzhou.log.aliyuncs.com", + "Region": "test_region", + "Endpoint": "test_region.log.aliyuncs.com", "TelemetryType": "metrics" } )"; @@ -345,8 +366,8 @@ void FlusherSLSUnittest::OnSuccessfulInit() { "Type": "flusher_sls", "Project": "test_project", "Logstore": "test_logstore", - "Region": "cn-hangzhou", - "Endpoint": "cn-hangzhou.log.aliyuncs.com", + "Region": "test_region", + "Endpoint": "test_region.log.aliyuncs.com", "TelemetryType": "unknown" } )"; @@ -364,8 +385,8 @@ void FlusherSLSUnittest::OnSuccessfulInit() { "Type": "flusher_sls", "Project": "test_project", "Logstore": "test_logstore", - "Region": "cn-hangzhou", - "Endpoint": "cn-hangzhou.log.aliyuncs.com", + "Region": "test_region", + "Endpoint": "test_region.log.aliyuncs.com", "ShardHashKeys": [ "__source__" ] @@ -387,8 +408,8 @@ void FlusherSLSUnittest::OnSuccessfulInit() { "Type": "flusher_sls", "Project": "test_project", "Logstore": "test_logstore", - "Region": "cn-hangzhou", - "Endpoint": "cn-hangzhou.log.aliyuncs.com", + "Region": "test_region", + "Endpoint": "test_region.log.aliyuncs.com", "ShardHashKeys": [ "__source__" ] @@ -407,8 +428,8 @@ void FlusherSLSUnittest::OnSuccessfulInit() { "Type": "flusher_sls", "Project": "test_project", "Logstore": "test_logstore", - "Region": "cn-hangzhou", - "Endpoint": "cn-hangzhou.log.aliyuncs.com" + "Region": "test_region", + "Endpoint": "test_region.log.aliyuncs.com" } )"; APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); @@ -428,8 +449,8 @@ void FlusherSLSUnittest::OnSuccessfulInit() { "Type": "flusher_sls", "Project": "test_project", "Logstore": "test_logstore", - "Region": "cn-hangzhou", - "Endpoint": "cn-hangzhou.log.aliyuncs.com", + "Region": "test_region", + "Endpoint": "test_region.log.aliyuncs.com", "EnableShardHash": false } )"; @@ -466,7 +487,7 @@ void FlusherSLSUnittest::OnFailedInit() { { "Type": "flusher_sls", "Logstore": "test_logstore", - "Endpoint": "cn-hangzhou.log.aliyuncs.com" + "Endpoint": "test_region.log.aliyuncs.com" } )"; APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); @@ -480,7 +501,7 @@ void FlusherSLSUnittest::OnFailedInit() { "Type": "flusher_sls", "Project": true, "Logstore": "test_logstore", - "Endpoint": "cn-hangzhou.log.aliyuncs.com" + "Endpoint": "test_region.log.aliyuncs.com" } )"; APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); @@ -494,7 +515,7 @@ void FlusherSLSUnittest::OnFailedInit() { { "Type": "flusher_sls", "Project": "test_project", - "Endpoint": "cn-hangzhou.log.aliyuncs.com" + "Endpoint": "test_region.log.aliyuncs.com" } )"; APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); @@ -508,7 +529,7 @@ void FlusherSLSUnittest::OnFailedInit() { "Type": "flusher_sls", "Project": "test_project", "Logstore": true, - "Endpoint": "cn-hangzhou.log.aliyuncs.com" + "Endpoint": "test_region.log.aliyuncs.com" } )"; APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); @@ -563,8 +584,8 @@ void FlusherSLSUnittest::OnPipelineUpdate() { "Type": "flusher_sls", "Project": "test_project", "Logstore": "test_logstore", - "Region": "cn-hangzhou", - "Endpoint": "cn-hangzhou.log.aliyuncs.com" + "Region": "test_region", + "Endpoint": "test_region.log.aliyuncs.com" } )"; APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); @@ -583,8 +604,8 @@ void FlusherSLSUnittest::OnPipelineUpdate() { "Type": "flusher_sls", "Project": "test_project_2", "Logstore": "test_logstore_2", - "Region": "cn-hangzhou", - "Endpoint": "cn-hangzhou.log.aliyuncs.com", + "Region": "test_region", + "Endpoint": "test_region.log.aliyuncs.com", "Aliuid": "123456789" } )"; @@ -613,8 +634,8 @@ void FlusherSLSUnittest::OnPipelineUpdate() { "Type": "flusher_sls", "Project": "test_project", "Logstore": "test_logstore", - "Region": "cn-hangzhou", - "Endpoint": "cn-hangzhou.log.aliyuncs.com", + "Region": "test_region", + "Endpoint": "test_region.log.aliyuncs.com", "Aliuid": "123456789" } )"; @@ -631,6 +652,483 @@ void FlusherSLSUnittest::OnPipelineUpdate() { } } +void FlusherSLSUnittest::TestBuildRequest() { +#ifdef __ENTERPRISE__ + EnterpriseSLSClientManager::GetInstance()->UpdateLocalRegionEndpointsAndHttpsInfo("test_region", + {kAccelerationDataEndpoint}); + EnterpriseSLSClientManager::GetInstance()->UpdateRemoteRegionEndpoints( + "test_region", {"test_region-intranet.log.aliyuncs.com", "test_region.log.aliyuncs.com"}); + EnterpriseSLSClientManager::GetInstance()->UpdateRemoteRegionEndpoints( + "test_region-b", {"test_region-b-intranet.log.aliyuncs.com", "test_region-b.log.aliyuncs.com"}); +#endif + Json::Value configJson, optionalGoPipeline; + string errorMsg; + string configStr = R"( + { + "Type": "flusher_sls", + "Project": "test_project", + "Logstore": "test_logstore", + "Region": "test_region-b", + "Aliuid": "1234567890", + "Endpoint": "test_endpoint" + } + )"; + APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); + FlusherSLS flusher; + flusher.SetContext(ctx); + flusher.SetMetricsRecordRef(FlusherSLS::sName, "1"); + APSARA_TEST_TRUE(flusher.Init(configJson, optionalGoPipeline)); + + string body = "hello, world!"; + string bodyLenStr = to_string(body.size()); + uint32_t rawSize = 100; + string rawSizeStr = "100"; + + SLSSenderQueueItem item("hello, world!", rawSize, &flusher, flusher.GetQueueKey(), flusher.mLogstore); + unique_ptr req; + bool keepItem = false; +#ifdef __ENTERPRISE__ + { + // empty ak + APSARA_TEST_FALSE(flusher.BuildRequest(&item, req, &keepItem)); + APSARA_TEST_EQUAL(nullptr, req); + APSARA_TEST_TRUE(keepItem); + } + EnterpriseSLSClientManager::GetInstance()->SetAccessKey( + "1234567890", SLSClientManager::AuthType::ANONYMOUS, "test_ak", "test_sk"); + { + // no available host, uninitialized + APSARA_TEST_FALSE(flusher.BuildRequest(&item, req, &keepItem)); + APSARA_TEST_EQUAL(nullptr, req); + APSARA_TEST_TRUE(keepItem); + APSARA_TEST_EQUAL(static_cast(AppConfig::GetInstance()->GetSendRequestConcurrency()), + FlusherSLS::GetRegionConcurrencyLimiter(flusher.mRegion)->GetCurrentLimit()); + } + { + // no available host, initialized + flusher.mCandidateHostsInfo->SetInitialized(); + APSARA_TEST_FALSE(flusher.BuildRequest(&item, req, &keepItem)); + APSARA_TEST_EQUAL(nullptr, req); + APSARA_TEST_TRUE(keepItem); + APSARA_TEST_EQUAL(static_cast(AppConfig::GetInstance()->GetSendRequestConcurrency()) / 2, + FlusherSLS::GetRegionConcurrencyLimiter(flusher.mRegion)->GetCurrentLimit()); + } + EnterpriseSLSClientManager::GetInstance()->UpdateHostInfo("test_project", + EndpointMode::DEFAULT, + "test_project.test_region-b.log.aliyuncs.com", + chrono::milliseconds(100)); + flusher.mCandidateHostsInfo->SelectBestHost(); +#endif + // log telemetry type + { + // normal + SLSSenderQueueItem item("hello, world!", rawSize, &flusher, flusher.GetQueueKey(), flusher.mLogstore); + APSARA_TEST_TRUE(flusher.BuildRequest(&item, req, &keepItem)); + APSARA_TEST_EQUAL(HTTP_POST, req->mMethod); +#ifdef __ENTERPRISE__ + APSARA_TEST_FALSE(req->mHTTPSFlag); +#else + APSARA_TEST_TRUE(req->mHTTPSFlag); +#endif + APSARA_TEST_EQUAL("/logstores/test_logstore/shards/lb", req->mUrl); + APSARA_TEST_EQUAL("", req->mQueryString); +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL(12U, req->mHeader.size()); +#else + APSARA_TEST_EQUAL(11U, req->mHeader.size()); +#endif +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL("test_project.test_region-b.log.aliyuncs.com", req->mHeader[HOST]); +#else + APSARA_TEST_EQUAL("test_project.test_endpoint", req->mHeader[HOST]); +#endif + APSARA_TEST_EQUAL(SLSClientManager::GetInstance()->GetUserAgent(), req->mHeader[USER_AGENT]); + APSARA_TEST_FALSE(req->mHeader[DATE].empty()); + APSARA_TEST_EQUAL(TYPE_LOG_PROTOBUF, req->mHeader[CONTENT_TYPE]); + APSARA_TEST_EQUAL(bodyLenStr, req->mHeader[CONTENT_LENGTH]); + APSARA_TEST_EQUAL(CalcMD5(req->mBody), req->mHeader[CONTENT_MD5]); + APSARA_TEST_EQUAL(LOG_API_VERSION, req->mHeader[X_LOG_APIVERSION]); + APSARA_TEST_EQUAL(HMAC_SHA1, req->mHeader[X_LOG_SIGNATUREMETHOD]); + APSARA_TEST_EQUAL("lz4", req->mHeader[X_LOG_COMPRESSTYPE]); + APSARA_TEST_EQUAL(rawSizeStr, req->mHeader[X_LOG_BODYRAWSIZE]); +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL(MD5_SHA1_SALT_KEYPROVIDER, req->mHeader[X_LOG_KEYPROVIDER]); +#endif + APSARA_TEST_FALSE(req->mHeader[AUTHORIZATION].empty()); + APSARA_TEST_EQUAL(body, req->mBody); +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL("test_project.test_region-b.log.aliyuncs.com", req->mHost); +#else + APSARA_TEST_EQUAL("test_project.test_endpoint", req->mHost); +#endif +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL(80, req->mPort); +#else + APSARA_TEST_EQUAL(443, req->mPort); +#endif + APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_timeout_secs)), req->mTimeout); + APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_max_try_cnt)), req->mMaxTryCnt); + APSARA_TEST_FALSE(req->mFollowRedirects); + APSARA_TEST_EQUAL(&item, req->mItem); + APSARA_TEST_FALSE(item.mRealIpFlag); +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL("test_project.test_region-b.log.aliyuncs.com", item.mCurrentHost); +#else + APSARA_TEST_EQUAL("test_project.test_endpoint", item.mCurrentHost); +#endif + } + { + // event group list + SLSSenderQueueItem item("hello, world!", + rawSize, + &flusher, + flusher.GetQueueKey(), + flusher.mLogstore, + RawDataType::EVENT_GROUP_LIST); + APSARA_TEST_TRUE(flusher.BuildRequest(&item, req, &keepItem)); + APSARA_TEST_EQUAL(HTTP_POST, req->mMethod); +#ifdef __ENTERPRISE__ + APSARA_TEST_FALSE(req->mHTTPSFlag); +#else + APSARA_TEST_TRUE(req->mHTTPSFlag); +#endif + APSARA_TEST_EQUAL("/logstores/test_logstore/shards/lb", req->mUrl); + APSARA_TEST_EQUAL("", req->mQueryString); +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL(13U, req->mHeader.size()); +#else + APSARA_TEST_EQUAL(12U, req->mHeader.size()); +#endif +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL("test_project.test_region-b.log.aliyuncs.com", req->mHeader[HOST]); +#else + APSARA_TEST_EQUAL("test_project.test_endpoint", req->mHeader[HOST]); +#endif + APSARA_TEST_EQUAL(SLSClientManager::GetInstance()->GetUserAgent(), req->mHeader[USER_AGENT]); + APSARA_TEST_FALSE(req->mHeader[DATE].empty()); + APSARA_TEST_EQUAL(TYPE_LOG_PROTOBUF, req->mHeader[CONTENT_TYPE]); + APSARA_TEST_EQUAL(bodyLenStr, req->mHeader[CONTENT_LENGTH]); + APSARA_TEST_EQUAL(CalcMD5(req->mBody), req->mHeader[CONTENT_MD5]); + APSARA_TEST_EQUAL(LOG_API_VERSION, req->mHeader[X_LOG_APIVERSION]); + APSARA_TEST_EQUAL(HMAC_SHA1, req->mHeader[X_LOG_SIGNATUREMETHOD]); + APSARA_TEST_EQUAL("lz4", req->mHeader[X_LOG_COMPRESSTYPE]); + APSARA_TEST_EQUAL(bodyLenStr, req->mHeader[X_LOG_BODYRAWSIZE]); + APSARA_TEST_EQUAL(LOG_MODE_BATCH_GROUP, req->mHeader[X_LOG_MODE]); +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL(MD5_SHA1_SALT_KEYPROVIDER, req->mHeader[X_LOG_KEYPROVIDER]); +#endif + APSARA_TEST_FALSE(req->mHeader[AUTHORIZATION].empty()); + APSARA_TEST_EQUAL(body, req->mBody); +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL("test_project.test_region-b.log.aliyuncs.com", req->mHost); +#else + APSARA_TEST_EQUAL("test_project.test_endpoint", req->mHost); +#endif +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL(80, req->mPort); +#else + APSARA_TEST_EQUAL(443, req->mPort); +#endif + APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_timeout_secs)), req->mTimeout); + APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_max_try_cnt)), req->mMaxTryCnt); + APSARA_TEST_FALSE(req->mFollowRedirects); + APSARA_TEST_EQUAL(&item, req->mItem); +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL("test_project.test_region-b.log.aliyuncs.com", item.mCurrentHost); +#else + APSARA_TEST_EQUAL("test_project.test_endpoint", item.mCurrentHost); +#endif + } + { + // shard hash + SLSSenderQueueItem item("hello, world!", + rawSize, + &flusher, + flusher.GetQueueKey(), + flusher.mLogstore, + RawDataType::EVENT_GROUP, + "hash_key"); + APSARA_TEST_TRUE(flusher.BuildRequest(&item, req, &keepItem)); + APSARA_TEST_EQUAL(HTTP_POST, req->mMethod); +#ifdef __ENTERPRISE__ + APSARA_TEST_FALSE(req->mHTTPSFlag); +#else + APSARA_TEST_TRUE(req->mHTTPSFlag); +#endif + APSARA_TEST_EQUAL("/logstores/test_logstore/shards/route", req->mUrl); + map params{{"key", "hash_key"}}; + APSARA_TEST_EQUAL(GetQueryString(params), req->mQueryString); +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL(12U, req->mHeader.size()); +#else + APSARA_TEST_EQUAL(11U, req->mHeader.size()); +#endif +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL("test_project.test_region-b.log.aliyuncs.com", req->mHeader[HOST]); +#else + APSARA_TEST_EQUAL("test_project.test_endpoint", req->mHeader[HOST]); +#endif + APSARA_TEST_EQUAL(SLSClientManager::GetInstance()->GetUserAgent(), req->mHeader[USER_AGENT]); + APSARA_TEST_FALSE(req->mHeader[DATE].empty()); + APSARA_TEST_EQUAL(TYPE_LOG_PROTOBUF, req->mHeader[CONTENT_TYPE]); + APSARA_TEST_EQUAL(bodyLenStr, req->mHeader[CONTENT_LENGTH]); + APSARA_TEST_EQUAL(CalcMD5(req->mBody), req->mHeader[CONTENT_MD5]); + APSARA_TEST_EQUAL(LOG_API_VERSION, req->mHeader[X_LOG_APIVERSION]); + APSARA_TEST_EQUAL(HMAC_SHA1, req->mHeader[X_LOG_SIGNATUREMETHOD]); + APSARA_TEST_EQUAL("lz4", req->mHeader[X_LOG_COMPRESSTYPE]); + APSARA_TEST_EQUAL(rawSizeStr, req->mHeader[X_LOG_BODYRAWSIZE]); +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL(MD5_SHA1_SALT_KEYPROVIDER, req->mHeader[X_LOG_KEYPROVIDER]); +#endif + APSARA_TEST_FALSE(req->mHeader[AUTHORIZATION].empty()); + APSARA_TEST_EQUAL(body, req->mBody); +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL("test_project.test_region-b.log.aliyuncs.com", req->mHost); +#else + APSARA_TEST_EQUAL("test_project.test_endpoint", req->mHost); +#endif +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL(80, req->mPort); +#else + APSARA_TEST_EQUAL(443, req->mPort); +#endif + APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_timeout_secs)), req->mTimeout); + APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_max_try_cnt)), req->mMaxTryCnt); + APSARA_TEST_FALSE(req->mFollowRedirects); + APSARA_TEST_EQUAL(&item, req->mItem); + APSARA_TEST_FALSE(item.mRealIpFlag); +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL("test_project.test_region-b.log.aliyuncs.com", item.mCurrentHost); +#else + APSARA_TEST_EQUAL("test_project.test_endpoint", item.mCurrentHost); +#endif + } + { + // exactly once + auto cpt = make_shared(); + cpt->index = 0; + cpt->data.set_hash_key("hash_key_0"); + cpt->data.set_sequence_id(1); + SLSSenderQueueItem item("hello, world!", + rawSize, + &flusher, + flusher.GetQueueKey(), + flusher.mLogstore, + RawDataType::EVENT_GROUP, + "hash_key_0", + std::move(cpt)); + APSARA_TEST_TRUE(flusher.BuildRequest(&item, req, &keepItem)); + APSARA_TEST_EQUAL(HTTP_POST, req->mMethod); +#ifdef __ENTERPRISE__ + APSARA_TEST_FALSE(req->mHTTPSFlag); +#else + APSARA_TEST_TRUE(req->mHTTPSFlag); +#endif + APSARA_TEST_EQUAL("/logstores/test_logstore/shards/route", req->mUrl); + map params{{"key", "hash_key_0"}, {"seqid", "1"}}; + APSARA_TEST_EQUAL(GetQueryString(params), req->mQueryString); +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL(12U, req->mHeader.size()); +#else + APSARA_TEST_EQUAL(11U, req->mHeader.size()); +#endif +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL("test_project.test_region-b.log.aliyuncs.com", req->mHeader[HOST]); +#else + APSARA_TEST_EQUAL("test_project.test_endpoint", req->mHeader[HOST]); +#endif + APSARA_TEST_EQUAL(SLSClientManager::GetInstance()->GetUserAgent(), req->mHeader[USER_AGENT]); + APSARA_TEST_FALSE(req->mHeader[DATE].empty()); + APSARA_TEST_EQUAL(TYPE_LOG_PROTOBUF, req->mHeader[CONTENT_TYPE]); + APSARA_TEST_EQUAL(bodyLenStr, req->mHeader[CONTENT_LENGTH]); + APSARA_TEST_EQUAL(CalcMD5(req->mBody), req->mHeader[CONTENT_MD5]); + APSARA_TEST_EQUAL(LOG_API_VERSION, req->mHeader[X_LOG_APIVERSION]); + APSARA_TEST_EQUAL(HMAC_SHA1, req->mHeader[X_LOG_SIGNATUREMETHOD]); + APSARA_TEST_EQUAL("lz4", req->mHeader[X_LOG_COMPRESSTYPE]); + APSARA_TEST_EQUAL(rawSizeStr, req->mHeader[X_LOG_BODYRAWSIZE]); +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL(MD5_SHA1_SALT_KEYPROVIDER, req->mHeader[X_LOG_KEYPROVIDER]); +#endif + APSARA_TEST_FALSE(req->mHeader[AUTHORIZATION].empty()); + APSARA_TEST_EQUAL(body, req->mBody); +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL("test_project.test_region-b.log.aliyuncs.com", req->mHost); +#else + APSARA_TEST_EQUAL("test_project.test_endpoint", req->mHost); +#endif +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL(80, req->mPort); +#else + APSARA_TEST_EQUAL(443, req->mPort); +#endif + APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_timeout_secs)), req->mTimeout); + APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_max_try_cnt)), req->mMaxTryCnt); + APSARA_TEST_FALSE(req->mFollowRedirects); + APSARA_TEST_EQUAL(&item, req->mItem); + APSARA_TEST_FALSE(item.mRealIpFlag); +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL("test_project.test_region-b.log.aliyuncs.com", item.mCurrentHost); +#else + APSARA_TEST_EQUAL("test_project.test_endpoint", item.mCurrentHost); +#endif + } + // metric telemtery type + flusher.mTelemetryType = sls_logs::SlsTelemetryType::SLS_TELEMETRY_TYPE_METRICS; + { + SLSSenderQueueItem item("hello, world!", rawSize, &flusher, flusher.GetQueueKey(), flusher.mLogstore); + APSARA_TEST_TRUE(flusher.BuildRequest(&item, req, &keepItem)); + APSARA_TEST_EQUAL(HTTP_POST, req->mMethod); +#ifdef __ENTERPRISE__ + APSARA_TEST_FALSE(req->mHTTPSFlag); +#else + APSARA_TEST_TRUE(req->mHTTPSFlag); +#endif + APSARA_TEST_EQUAL("/prometheus/test_project/test_logstore/api/v1/write", req->mUrl); + APSARA_TEST_EQUAL("", req->mQueryString); +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL(12U, req->mHeader.size()); +#else + APSARA_TEST_EQUAL(11U, req->mHeader.size()); +#endif +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL("test_project.test_region-b.log.aliyuncs.com", req->mHeader[HOST]); +#else + APSARA_TEST_EQUAL("test_project.test_endpoint", req->mHeader[HOST]); +#endif + APSARA_TEST_EQUAL(SLSClientManager::GetInstance()->GetUserAgent(), req->mHeader[USER_AGENT]); + APSARA_TEST_FALSE(req->mHeader[DATE].empty()); + APSARA_TEST_EQUAL(TYPE_LOG_PROTOBUF, req->mHeader[CONTENT_TYPE]); + APSARA_TEST_EQUAL(bodyLenStr, req->mHeader[CONTENT_LENGTH]); + APSARA_TEST_EQUAL(CalcMD5(req->mBody), req->mHeader[CONTENT_MD5]); + APSARA_TEST_EQUAL(LOG_API_VERSION, req->mHeader[X_LOG_APIVERSION]); + APSARA_TEST_EQUAL(HMAC_SHA1, req->mHeader[X_LOG_SIGNATUREMETHOD]); + APSARA_TEST_EQUAL("lz4", req->mHeader[X_LOG_COMPRESSTYPE]); + APSARA_TEST_EQUAL(rawSizeStr, req->mHeader[X_LOG_BODYRAWSIZE]); +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL(MD5_SHA1_SALT_KEYPROVIDER, req->mHeader[X_LOG_KEYPROVIDER]); +#endif + APSARA_TEST_FALSE(req->mHeader[AUTHORIZATION].empty()); + APSARA_TEST_EQUAL(body, req->mBody); +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL("test_project.test_region-b.log.aliyuncs.com", req->mHost); +#else + APSARA_TEST_EQUAL("test_project.test_endpoint", req->mHost); +#endif +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL(80, req->mPort); +#else + APSARA_TEST_EQUAL(443, req->mPort); +#endif + APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_timeout_secs)), req->mTimeout); + APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_max_try_cnt)), req->mMaxTryCnt); + APSARA_TEST_FALSE(req->mFollowRedirects); + APSARA_TEST_EQUAL(&item, req->mItem); + APSARA_TEST_FALSE(item.mRealIpFlag); +#ifdef __ENTERPRISE__ + APSARA_TEST_EQUAL("test_project.test_region-b.log.aliyuncs.com", item.mCurrentHost); +#else + APSARA_TEST_EQUAL("test_project.test_endpoint", item.mCurrentHost); +#endif + } + flusher.mTelemetryType = sls_logs::SlsTelemetryType::SLS_TELEMETRY_TYPE_LOGS; +#ifdef __ENTERPRISE__ + { + // region mode changed + EnterpriseSLSClientManager::GetInstance()->CopyLocalRegionEndpointsAndHttpsInfoIfNotExisted("test_region", + "test_region-b"); + auto old = flusher.mCandidateHostsInfo.get(); + APSARA_TEST_FALSE(flusher.BuildRequest(&item, req, &keepItem)); + APSARA_TEST_NOT_EQUAL(old, flusher.mCandidateHostsInfo.get()); + + EnterpriseSLSClientManager::GetInstance()->UpdateHostInfo("test_project", + EndpointMode::ACCELERATE, + "test_project." + kAccelerationDataEndpoint, + chrono::milliseconds(10)); + flusher.mCandidateHostsInfo->SelectBestHost(); + APSARA_TEST_TRUE(flusher.BuildRequest(&item, req, &keepItem)); + APSARA_TEST_EQUAL("test_project." + kAccelerationDataEndpoint, req->mHost); + } + // real ip + BOOL_FLAG(send_prefer_real_ip) = true; + { + // ip not empty + EnterpriseSLSClientManager::GetInstance()->SetRealIp("test_region-b", "192.168.0.1"); + SLSSenderQueueItem item("hello, world!", rawSize, &flusher, flusher.GetQueueKey(), flusher.mLogstore); + APSARA_TEST_TRUE(flusher.BuildRequest(&item, req, &keepItem)); + APSARA_TEST_EQUAL(HTTP_POST, req->mMethod); + APSARA_TEST_FALSE(req->mHTTPSFlag); + APSARA_TEST_EQUAL("/logstores/test_logstore/shards/lb", req->mUrl); + APSARA_TEST_EQUAL("", req->mQueryString); + APSARA_TEST_EQUAL(12U, req->mHeader.size()); + APSARA_TEST_EQUAL("test_project.192.168.0.1", req->mHeader[HOST]); + APSARA_TEST_EQUAL(SLSClientManager::GetInstance()->GetUserAgent(), req->mHeader[USER_AGENT]); + APSARA_TEST_FALSE(req->mHeader[DATE].empty()); + APSARA_TEST_EQUAL(TYPE_LOG_PROTOBUF, req->mHeader[CONTENT_TYPE]); + APSARA_TEST_EQUAL(bodyLenStr, req->mHeader[CONTENT_LENGTH]); + APSARA_TEST_EQUAL(CalcMD5(req->mBody), req->mHeader[CONTENT_MD5]); + APSARA_TEST_EQUAL(LOG_API_VERSION, req->mHeader[X_LOG_APIVERSION]); + APSARA_TEST_EQUAL(HMAC_SHA1, req->mHeader[X_LOG_SIGNATUREMETHOD]); + APSARA_TEST_EQUAL("lz4", req->mHeader[X_LOG_COMPRESSTYPE]); + APSARA_TEST_EQUAL(rawSizeStr, req->mHeader[X_LOG_BODYRAWSIZE]); + APSARA_TEST_EQUAL(MD5_SHA1_SALT_KEYPROVIDER, req->mHeader[X_LOG_KEYPROVIDER]); + APSARA_TEST_FALSE(req->mHeader[AUTHORIZATION].empty()); + APSARA_TEST_EQUAL(body, req->mBody); + APSARA_TEST_EQUAL("192.168.0.1", req->mHost); + APSARA_TEST_EQUAL(80, req->mPort); + APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_timeout_secs)), req->mTimeout); + APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_max_try_cnt)), req->mMaxTryCnt); + APSARA_TEST_FALSE(req->mFollowRedirects); + APSARA_TEST_EQUAL(&item, req->mItem); + APSARA_TEST_TRUE(item.mRealIpFlag); + APSARA_TEST_EQUAL("192.168.0.1", item.mCurrentHost); + } + { + // ip empty + EnterpriseSLSClientManager::GetInstance()->SetRealIp("test_region-b", ""); + SLSSenderQueueItem item("hello, world!", rawSize, &flusher, flusher.GetQueueKey(), flusher.mLogstore); + APSARA_TEST_TRUE(flusher.BuildRequest(&item, req, &keepItem)); + APSARA_TEST_EQUAL("test_project." + kAccelerationDataEndpoint, req->mHeader[HOST]); + APSARA_TEST_EQUAL(SLSClientManager::GetInstance()->GetUserAgent(), req->mHeader[USER_AGENT]); + APSARA_TEST_FALSE(req->mHeader[DATE].empty()); + APSARA_TEST_EQUAL(TYPE_LOG_PROTOBUF, req->mHeader[CONTENT_TYPE]); + APSARA_TEST_EQUAL(bodyLenStr, req->mHeader[CONTENT_LENGTH]); + APSARA_TEST_EQUAL(CalcMD5(req->mBody), req->mHeader[CONTENT_MD5]); + APSARA_TEST_EQUAL(LOG_API_VERSION, req->mHeader[X_LOG_APIVERSION]); + APSARA_TEST_EQUAL(HMAC_SHA1, req->mHeader[X_LOG_SIGNATUREMETHOD]); + APSARA_TEST_EQUAL("lz4", req->mHeader[X_LOG_COMPRESSTYPE]); + APSARA_TEST_EQUAL(rawSizeStr, req->mHeader[X_LOG_BODYRAWSIZE]); + APSARA_TEST_EQUAL(MD5_SHA1_SALT_KEYPROVIDER, req->mHeader[X_LOG_KEYPROVIDER]); + APSARA_TEST_FALSE(req->mHeader[AUTHORIZATION].empty()); + APSARA_TEST_EQUAL(body, req->mBody); + APSARA_TEST_EQUAL("test_project." + kAccelerationDataEndpoint, req->mHost); + APSARA_TEST_EQUAL(80, req->mPort); + APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_timeout_secs)), req->mTimeout); + APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_max_try_cnt)), req->mMaxTryCnt); + APSARA_TEST_FALSE(req->mFollowRedirects); + APSARA_TEST_EQUAL(&item, req->mItem); + APSARA_TEST_FALSE(item.mRealIpFlag); + APSARA_TEST_EQUAL("test_project." + kAccelerationDataEndpoint, item.mCurrentHost); + } + { + // ip empty, and region mode changed + auto& endpoints = EnterpriseSLSClientManager::GetInstance()->mRegionCandidateEndpointsMap["test_region-b"]; + endpoints.mMode = EndpointMode::CUSTOM; + endpoints.mLocalEndpoints = {"custom.endpoint"}; + + auto old = flusher.mCandidateHostsInfo.get(); + APSARA_TEST_FALSE(flusher.BuildRequest(&item, req, &keepItem)); + APSARA_TEST_NOT_EQUAL(old, flusher.mCandidateHostsInfo.get()); + + EnterpriseSLSClientManager::GetInstance()->UpdateHostInfo( + "test_project", EndpointMode::CUSTOM, "test_project.custom.endpoint", chrono::milliseconds(10)); + flusher.mCandidateHostsInfo->SelectBestHost(); + APSARA_TEST_TRUE(flusher.BuildRequest(&item, req, &keepItem)); + APSARA_TEST_EQUAL("test_project.custom.endpoint", req->mHost); + } + BOOL_FLAG(send_prefer_real_ip) = false; +#endif +} + void FlusherSLSUnittest::TestSend() { { // exactly once enabled @@ -642,8 +1140,8 @@ void FlusherSLSUnittest::TestSend() { "Type": "flusher_sls", "Project": "test_project", "Logstore": "test_logstore", - "Region": "cn-hangzhou", - "Endpoint": "cn-hangzhou.log.aliyuncs.com", + "Region": "test_region", + "Endpoint": "test_region.log.aliyuncs.com", "Aliuid": "123456789" } )"; @@ -789,8 +1287,8 @@ void FlusherSLSUnittest::TestSend() { "Type": "flusher_sls", "Project": "test_project", "Logstore": "test_logstore", - "Region": "cn-hangzhou", - "Endpoint": "cn-hangzhou.log.aliyuncs.com", + "Region": "test_region", + "Endpoint": "test_region.log.aliyuncs.com", "Aliuid": "123456789", "ShardHashKeys": [ "tag_key" @@ -884,8 +1382,8 @@ void FlusherSLSUnittest::TestSend() { "Type": "flusher_sls", "Project": "test_project", "Logstore": "test_logstore", - "Region": "cn-hangzhou", - "Endpoint": "cn-hangzhou.log.aliyuncs.com", + "Region": "test_region", + "Endpoint": "test_region.log.aliyuncs.com", "Aliuid": "123456789" } )"; @@ -986,8 +1484,8 @@ void FlusherSLSUnittest::TestFlush() { "Type": "flusher_sls", "Project": "test_project", "Logstore": "test_logstore", - "Region": "cn-hangzhou", - "Endpoint": "cn-hangzhou.log.aliyuncs.com", + "Region": "test_region", + "Endpoint": "test_region.log.aliyuncs.com", "Aliuid": "123456789" } )"; @@ -1030,8 +1528,8 @@ void FlusherSLSUnittest::TestFlushAll() { "Type": "flusher_sls", "Project": "test_project", "Logstore": "test_logstore", - "Region": "cn-hangzhou", - "Endpoint": "cn-hangzhou.log.aliyuncs.com", + "Region": "test_region", + "Endpoint": "test_region.log.aliyuncs.com", "Aliuid": "123456789" } )"; @@ -1082,8 +1580,8 @@ void FlusherSLSUnittest::OnGoPipelineSend() { "Type": "flusher_sls", "Project": "test_project", "Logstore": "test_logstore", - "Region": "cn-hangzhou", - "Endpoint": "cn-hangzhou.log.aliyuncs.com", + "Region": "test_region", + "Endpoint": "test_region.log.aliyuncs.com", "Aliuid": "123456789" } )"; @@ -1163,6 +1661,7 @@ void FlusherSLSUnittest::OnGoPipelineSend() { UNIT_TEST_CASE(FlusherSLSUnittest, OnSuccessfulInit) UNIT_TEST_CASE(FlusherSLSUnittest, OnFailedInit) UNIT_TEST_CASE(FlusherSLSUnittest, OnPipelineUpdate) +UNIT_TEST_CASE(FlusherSLSUnittest, TestBuildRequest) UNIT_TEST_CASE(FlusherSLSUnittest, TestSend) UNIT_TEST_CASE(FlusherSLSUnittest, TestFlush) UNIT_TEST_CASE(FlusherSLSUnittest, TestFlushAll) diff --git a/core/unittest/flusher/SLSClientManagerUnittest.cpp b/core/unittest/flusher/SLSClientManagerUnittest.cpp index bcdebccdf0..83fb8f1c26 100644 --- a/core/unittest/flusher/SLSClientManagerUnittest.cpp +++ b/core/unittest/flusher/SLSClientManagerUnittest.cpp @@ -241,6 +241,7 @@ void CandidateHostsInfoUnittest::TestUpdateHosts() { const string internalHost = mProject + "." + internalEndpoint; const string globalHost = mProject + "." + globalEndpoint; const string customHost = mProject + "." + customEndpoint; +#ifdef __ENTERPRISE__ { // default mode CandidateHostsInfo info("project", region, EndpointMode::DEFAULT); @@ -391,6 +392,7 @@ void CandidateHostsInfoUnittest::TestUpdateHosts() { APSARA_TEST_EQUAL(chrono::milliseconds(150), info.mCandidateHosts[0][1].GetLatency()); } } +#endif { // custom mode CandidateHostsInfo info("project", region, EndpointMode::CUSTOM); From 297eaea79bd3746d0d4aa5e88965be805b07590c Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Thu, 12 Dec 2024 02:12:33 +0000 Subject: [PATCH 10/41] polish --- core/plugin/flusher/sls/DiskBufferWriter.cpp | 15 +- core/plugin/flusher/sls/DiskBufferWriter.h | 13 +- core/plugin/flusher/sls/FlusherSLS.cpp | 8 +- core/plugin/flusher/sls/FlusherSLS.h | 5 + core/plugin/flusher/sls/SLSClientManager.cpp | 526 +++++------ core/plugin/flusher/sls/SLSClientManager.h | 156 ++-- core/unittest/flusher/FlusherSLSUnittest.cpp | 9 +- .../flusher/SLSClientManagerUnittest.cpp | 852 +++++++++--------- 8 files changed, 799 insertions(+), 785 deletions(-) diff --git a/core/plugin/flusher/sls/DiskBufferWriter.cpp b/core/plugin/flusher/sls/DiskBufferWriter.cpp index f91478f79e..9ed4af09d6 100644 --- a/core/plugin/flusher/sls/DiskBufferWriter.cpp +++ b/core/plugin/flusher/sls/DiskBufferWriter.cpp @@ -33,6 +33,7 @@ #endif #include "plugin/flusher/sls/SLSClientManager.h" #include "plugin/flusher/sls/SLSConstant.h" +#include "plugin/flusher/sls/SendResult.h" #include "protobuf/sls/sls_logs.pb.h" #include "provider/Provider.h" @@ -239,7 +240,9 @@ void DiskBufferWriter::BufferSenderThread() { "check header of buffer file failed, delete file: " + fileName); } } +#ifdef __ENTERPRISE__ mCandidateHostsInfos.clear(); +#endif // mIsSendingBuffer = false; lock.lock(); if (mStopCV.wait_for(lock, chrono::seconds(mCheckPeriod), [this]() { return !mIsSendBufferThreadRunning; })) { @@ -821,15 +824,6 @@ SLSResponse DiskBufferWriter::SendBufferFileData(const sls_logs::LogtailBufferMe } auto info = EnterpriseSLSClientManager::GetInstance()->GetCandidateHostsInfo( region, bufferMeta.project(), GetEndpointMode(bufferMeta.endpointmode())); -#else - auto info = SLSClientManager::GetInstance()->GetCandidateHostsInfo(bufferMeta.project(), bufferMeta.endpoint()); - if (info == nullptr) { - SLSResponse response; - response.mErrorCode = LOGE_REQUEST_ERROR; - response.mErrorMsg = "can not get available host"; - return response; - } -#endif mCandidateHostsInfos.insert(info); host = info->GetCurrentHost(); @@ -839,6 +833,9 @@ SLSResponse DiskBufferWriter::SendBufferFileData(const sls_logs::LogtailBufferMe response.mErrorMsg = "can not get available host"; return response; } +#else + host = bufferMeta.project() + "." + bufferMeta.endpoint(); +#endif bool httpsFlag = SLSClientManager::GetInstance()->UsingHttps(region); diff --git a/core/plugin/flusher/sls/DiskBufferWriter.h b/core/plugin/flusher/sls/DiskBufferWriter.h index abfbf49396..6075ffa763 100644 --- a/core/plugin/flusher/sls/DiskBufferWriter.h +++ b/core/plugin/flusher/sls/DiskBufferWriter.h @@ -16,17 +16,26 @@ #pragma once +#include #include #include +#include #include +#include #include #include +#ifdef __ENTERPRIRSE__ +#include +#endif #include #include "common/SafeQueue.h" #include "pipeline/queue/SenderQueueItem.h" +#ifdef __ENTERPRISE__ +#include "plugin/flusher/sls/EnterpriseSLSClientManager.h" +#endif #include "plugin/flusher/sls/SLSClientManager.h" -#include "plugin/flusher/sls/SendResult.h" +#include "plugin/flusher/sls/SLSResponse.h" #include "protobuf/sls/logtail_buffer_meta.pb.h" namespace logtail { @@ -93,6 +102,7 @@ class DiskBufferWriter { bool mIsSendBufferThreadRunning = true; mutable std::condition_variable mStopCV; +#ifdef __ENTERPRISE__ struct PointerHash { std::size_t operator()(const std::shared_ptr& ptr) const { return std::hash()(ptr.get()); @@ -107,6 +117,7 @@ class DiskBufferWriter { }; std::unordered_set, PointerHash, PointerEqual> mCandidateHostsInfos; +#endif mutable std::mutex mBufferFileLock; std::string mBufferFilePath; diff --git a/core/plugin/flusher/sls/FlusherSLS.cpp b/core/plugin/flusher/sls/FlusherSLS.cpp index 6d082e3eb2..6e5d5b029b 100644 --- a/core/plugin/flusher/sls/FlusherSLS.cpp +++ b/core/plugin/flusher/sls/FlusherSLS.cpp @@ -397,7 +397,6 @@ bool FlusherSLS::Init(const Json::Value& config, Json::Value& optionalGoPipeline mContext->GetLogstoreName(), mContext->GetRegion()); } - mCandidateHostsInfo = SLSClientManager::GetInstance()->GetCandidateHostsInfo(mProject, mEndpoint); #endif // TelemetryType @@ -614,11 +613,8 @@ bool FlusherSLS::BuildRequest(SenderQueueItem* item, unique_ptr if (mCandidateHostsInfo.get() != info.get()) { mCandidateHostsInfo = info; } -#endif data->mCurrentHost = mCandidateHostsInfo->GetCurrentHost(); -#ifdef __ENTERPRISE__ } -#endif if (data->mCurrentHost.empty()) { if (mCandidateHostsInfo->IsInitialized()) { GetRegionConcurrencyLimiter(mRegion)->OnFail(); @@ -626,6 +622,10 @@ bool FlusherSLS::BuildRequest(SenderQueueItem* item, unique_ptr *keepItem = true; return false; } +#else + static string host = mProject + "." + mEndpoint; + data->mCurrentHost = host; +#endif switch (mTelemetryType) { case sls_logs::SLS_TELEMETRY_TYPE_LOGS: diff --git a/core/plugin/flusher/sls/FlusherSLS.h b/core/plugin/flusher/sls/FlusherSLS.h index e742445a3d..b3d8292576 100644 --- a/core/plugin/flusher/sls/FlusherSLS.h +++ b/core/plugin/flusher/sls/FlusherSLS.h @@ -32,6 +32,9 @@ #include "pipeline/plugin/interface/HttpFlusher.h" #include "pipeline/queue/SLSSenderQueueItem.h" #include "pipeline/serializer/SLSSerializer.h" +#ifdef __ENTERPRISE__ +#include "plugin/flusher/sls/EnterpriseSLSClientManager.h" +#endif #include "plugin/flusher/sls/SLSClientManager.h" #include "protobuf/sls/sls_logs.pb.h" @@ -128,9 +131,11 @@ class FlusherSLS : public HttpFlusher { Batcher mBatcher; std::unique_ptr mGroupSerializer; std::unique_ptr>> mGroupListSerializer; +#ifdef __ENTERPRISE__ // This may not be cached. However, this provides a simple way to control the lifetime of a CandidateHostsInfo. // Otherwise, timeout machanisim must be emplyed to clean up unused CandidateHostsInfo. std::shared_ptr mCandidateHostsInfo; +#endif CounterPtr mSendCnt; CounterPtr mSendDoneCnt; diff --git a/core/plugin/flusher/sls/SLSClientManager.cpp b/core/plugin/flusher/sls/SLSClientManager.cpp index 61dc6a87f7..939cd87adb 100644 --- a/core/plugin/flusher/sls/SLSClientManager.cpp +++ b/core/plugin/flusher/sls/SLSClientManager.cpp @@ -86,238 +86,238 @@ namespace logtail { // static const string kAccelerationDataEndpoint = "log-global.aliyuncs.com"; -const string& EndpointModeToString(EndpointMode mode) { - switch (mode) { - case EndpointMode::CUSTOM: - static string customStr = "custom"; - return customStr; - case EndpointMode::ACCELERATE: - static string accelerateStr = "accelerate"; - return accelerateStr; - case EndpointMode::DEFAULT: - static string defaultStr = "default"; - return defaultStr; - default: - static string unknownStr = "unknown"; - return unknownStr; - } -} +// const string& EndpointModeToString(EndpointMode mode) { +// switch (mode) { +// case EndpointMode::CUSTOM: +// static string customStr = "custom"; +// return customStr; +// case EndpointMode::ACCELERATE: +// static string accelerateStr = "accelerate"; +// return accelerateStr; +// case EndpointMode::DEFAULT: +// static string defaultStr = "default"; +// return defaultStr; +// default: +// static string unknownStr = "unknown"; +// return unknownStr; +// } +// } -chrono::milliseconds HostInfo::GetLatency() const { - lock_guard lock(mLatencyMux); - return mLatency; -} +// chrono::milliseconds HostInfo::GetLatency() const { +// lock_guard lock(mLatencyMux); +// return mLatency; +// } -void HostInfo::SetLatency(const chrono::milliseconds& latency) { - lock_guard lock(mLatencyMux); - mLatency = latency; -} +// void HostInfo::SetLatency(const chrono::milliseconds& latency) { +// lock_guard lock(mLatencyMux); +// mLatency = latency; +// } -void HostInfo::SetForbidden() { - lock_guard lock(mLatencyMux); - mLatency = chrono::milliseconds::max(); -} +// void HostInfo::SetForbidden() { +// lock_guard lock(mLatencyMux); +// mLatency = chrono::milliseconds::max(); +// } -bool HostInfo::IsForbidden() const { - lock_guard lock(mLatencyMux); - return mLatency == chrono::milliseconds::max(); -} +// bool HostInfo::IsForbidden() const { +// lock_guard lock(mLatencyMux); +// return mLatency == chrono::milliseconds::max(); +// } -void CandidateHostsInfo::UpdateHosts(const CandidateEndpoints& regionEndpoints) { - lock_guard lock(mCandidateHostsMux); - switch (mMode) { -#ifdef __ENTERPRISE__ - case EndpointMode::DEFAULT: { - vector endpoints(regionEndpoints.mLocalEndpoints); - for (const auto& endpoint : regionEndpoints.mRemoteEndpoints) { - bool found = false; - for (const auto& existedEndpoint : regionEndpoints.mLocalEndpoints) { - if (existedEndpoint == endpoint) { - found = true; - break; - } - } - if (!found) { - endpoints.emplace_back(endpoint); - } - } - vector> infos; - for (const auto& endpoint : endpoints) { - string host = mProject.empty() ? endpoint : mProject + "." + endpoint; - if (mCandidateHosts.empty()) { - infos.emplace_back().emplace_back(host); - } else { - bool found = false; - for (const auto& item : mCandidateHosts) { - if (!item.empty() && item[0].GetHostname() == host) { - found = true; - infos.emplace_back(item); - break; - } - } - if (!found) { - infos.emplace_back().emplace_back(host); - } - } - } - mCandidateHosts.swap(infos); - break; - } - case EndpointMode::ACCELERATE: { - vector endpoints{kAccelerationDataEndpoint}; - for (const auto& item : regionEndpoints.mRemoteEndpoints) { - if (GetEndpointAddressType(item) == EndpointAddressType::PUBLIC) { - endpoints.emplace_back(item); - } - } - vector infos; - for (const auto& endpoint : endpoints) { - string host = mProject.empty() ? endpoint : mProject + "." + endpoint; - if (mCandidateHosts.empty()) { - infos.emplace_back(host); - } else { - bool found = false; - for (const auto& item : mCandidateHosts[0]) { - if (item.GetHostname() == host) { - found = true; - infos.emplace_back(item); - break; - } - } - if (!found) { - infos.emplace_back(host); - } - } - } - if (mCandidateHosts.empty()) { - mCandidateHosts.emplace_back(infos); - } else { - mCandidateHosts[0].swap(infos); - } - break; - } -#endif - case EndpointMode::CUSTOM: { - vector infos; - for (const auto& endpoint : regionEndpoints.mLocalEndpoints) { - string host = mProject.empty() ? endpoint : mProject + "." + endpoint; - if (mCandidateHosts.empty()) { - infos.emplace_back(host); - } else { - bool found = false; - for (const auto& item : mCandidateHosts[0]) { - if (item.GetHostname() == host) { - found = true; - infos.emplace_back(item); - break; - } - } - if (!found) { - infos.emplace_back(host); - } - } - } - if (mCandidateHosts.empty()) { - mCandidateHosts.emplace_back(infos); - } else { - mCandidateHosts[0].swap(infos); - } - break; - } - default: - break; - } -} +// void CandidateHostsInfo::UpdateHosts(const CandidateEndpoints& regionEndpoints) { +// lock_guard lock(mCandidateHostsMux); +// switch (mMode) { +// #ifdef __ENTERPRISE__ +// case EndpointMode::DEFAULT: { +// vector endpoints(regionEndpoints.mLocalEndpoints); +// for (const auto& endpoint : regionEndpoints.mRemoteEndpoints) { +// bool found = false; +// for (const auto& existedEndpoint : regionEndpoints.mLocalEndpoints) { +// if (existedEndpoint == endpoint) { +// found = true; +// break; +// } +// } +// if (!found) { +// endpoints.emplace_back(endpoint); +// } +// } +// vector> infos; +// for (const auto& endpoint : endpoints) { +// string host = mProject.empty() ? endpoint : mProject + "." + endpoint; +// if (mCandidateHosts.empty()) { +// infos.emplace_back().emplace_back(host); +// } else { +// bool found = false; +// for (const auto& item : mCandidateHosts) { +// if (!item.empty() && item[0].GetHostname() == host) { +// found = true; +// infos.emplace_back(item); +// break; +// } +// } +// if (!found) { +// infos.emplace_back().emplace_back(host); +// } +// } +// } +// mCandidateHosts.swap(infos); +// break; +// } +// case EndpointMode::ACCELERATE: { +// vector endpoints{kAccelerationDataEndpoint}; +// for (const auto& item : regionEndpoints.mRemoteEndpoints) { +// if (GetEndpointAddressType(item) == EndpointAddressType::PUBLIC) { +// endpoints.emplace_back(item); +// } +// } +// vector infos; +// for (const auto& endpoint : endpoints) { +// string host = mProject.empty() ? endpoint : mProject + "." + endpoint; +// if (mCandidateHosts.empty()) { +// infos.emplace_back(host); +// } else { +// bool found = false; +// for (const auto& item : mCandidateHosts[0]) { +// if (item.GetHostname() == host) { +// found = true; +// infos.emplace_back(item); +// break; +// } +// } +// if (!found) { +// infos.emplace_back(host); +// } +// } +// } +// if (mCandidateHosts.empty()) { +// mCandidateHosts.emplace_back(infos); +// } else { +// mCandidateHosts[0].swap(infos); +// } +// break; +// } +// #endif +// case EndpointMode::CUSTOM: { +// vector infos; +// for (const auto& endpoint : regionEndpoints.mLocalEndpoints) { +// string host = mProject.empty() ? endpoint : mProject + "." + endpoint; +// if (mCandidateHosts.empty()) { +// infos.emplace_back(host); +// } else { +// bool found = false; +// for (const auto& item : mCandidateHosts[0]) { +// if (item.GetHostname() == host) { +// found = true; +// infos.emplace_back(item); +// break; +// } +// } +// if (!found) { +// infos.emplace_back(host); +// } +// } +// } +// if (mCandidateHosts.empty()) { +// mCandidateHosts.emplace_back(infos); +// } else { +// mCandidateHosts[0].swap(infos); +// } +// break; +// } +// default: +// break; +// } +// } -bool CandidateHostsInfo::UpdateHostInfo(const string& hostname, const chrono::milliseconds& latency) { - lock_guard lock(mCandidateHostsMux); - for (auto& item : mCandidateHosts) { - for (auto& entry : item) { - if (entry.GetHostname() == hostname) { - entry.SetLatency(latency); - return true; - } - } - } - return false; -} +// bool CandidateHostsInfo::UpdateHostInfo(const string& hostname, const chrono::milliseconds& latency) { +// lock_guard lock(mCandidateHostsMux); +// for (auto& item : mCandidateHosts) { +// for (auto& entry : item) { +// if (entry.GetHostname() == hostname) { +// entry.SetLatency(latency); +// return true; +// } +// } +// } +// return false; +// } -void CandidateHostsInfo::GetProbeHosts(vector& hosts) const { - if (!HasValidHost()) { - return GetAllHosts(hosts); - } - { - lock_guard lock(mCandidateHostsMux); - for (const auto& item : mCandidateHosts) { - for (const auto& entry : item) { - if (!entry.IsForbidden()) { - hosts.emplace_back(entry.GetHostname()); - } - } - } - } -} +// void CandidateHostsInfo::GetProbeHosts(vector& hosts) const { +// if (!HasValidHost()) { +// return GetAllHosts(hosts); +// } +// { +// lock_guard lock(mCandidateHostsMux); +// for (const auto& item : mCandidateHosts) { +// for (const auto& entry : item) { +// if (!entry.IsForbidden()) { +// hosts.emplace_back(entry.GetHostname()); +// } +// } +// } +// } +// } -void CandidateHostsInfo::GetAllHosts(vector& hosts) const { - lock_guard lock(mCandidateHostsMux); - for (const auto& item : mCandidateHosts) { - for (const auto& entry : item) { - hosts.emplace_back(entry.GetHostname()); - } - } -} +// void CandidateHostsInfo::GetAllHosts(vector& hosts) const { +// lock_guard lock(mCandidateHostsMux); +// for (const auto& item : mCandidateHosts) { +// for (const auto& entry : item) { +// hosts.emplace_back(entry.GetHostname()); +// } +// } +// } -void CandidateHostsInfo::SelectBestHost() { - lock_guard lock(mCandidateHostsMux); - for (size_t i = 0; i < mCandidateHosts.size(); ++i) { - const auto& hosts = mCandidateHosts[i]; - chrono::milliseconds minLatency = chrono::milliseconds::max(); - size_t minIdx = numeric_limits::max(); - for (size_t j = 0; j < hosts.size(); ++j) { - if (!hosts[j].IsForbidden() && hosts[j].GetLatency() < minLatency) { - minLatency = hosts[j].GetLatency(); - minIdx = j; - } - } - if (minIdx != numeric_limits::max()) { - const auto& hostname = hosts[minIdx].GetHostname(); - if (GetCurrentHost() != hostname) { - SetCurrentHost(hostname); - LOG_INFO(sLogger, - ("switch to the best host", hostname)("latency", minLatency.count())("project", mProject)( - "region", mRegion)("endpoint mode", EndpointModeToString(mMode))); - } - return; - } - } - SetCurrentHost(""); - LOG_INFO(sLogger, - ("no valid host", "stop sending data and retry later")("project", mProject)("region", mRegion)( - "endpoint mode", EndpointModeToString(mMode))); -} +// void CandidateHostsInfo::SelectBestHost() { +// lock_guard lock(mCandidateHostsMux); +// for (size_t i = 0; i < mCandidateHosts.size(); ++i) { +// const auto& hosts = mCandidateHosts[i]; +// chrono::milliseconds minLatency = chrono::milliseconds::max(); +// size_t minIdx = numeric_limits::max(); +// for (size_t j = 0; j < hosts.size(); ++j) { +// if (!hosts[j].IsForbidden() && hosts[j].GetLatency() < minLatency) { +// minLatency = hosts[j].GetLatency(); +// minIdx = j; +// } +// } +// if (minIdx != numeric_limits::max()) { +// const auto& hostname = hosts[minIdx].GetHostname(); +// if (GetCurrentHost() != hostname) { +// SetCurrentHost(hostname); +// LOG_INFO(sLogger, +// ("switch to the best host", hostname)("latency", minLatency.count())("project", mProject)( +// "region", mRegion)("endpoint mode", EndpointModeToString(mMode))); +// } +// return; +// } +// } +// SetCurrentHost(""); +// LOG_INFO(sLogger, +// ("no valid host", "stop sending data and retry later")("project", mProject)("region", mRegion)( +// "endpoint mode", EndpointModeToString(mMode))); +// } -string CandidateHostsInfo::GetCurrentHost() const { - lock_guard lock(mCurrentHostMux); - return mCurrentHost; -} +// string CandidateHostsInfo::GetCurrentHost() const { +// lock_guard lock(mCurrentHostMux); +// return mCurrentHost; +// } -string CandidateHostsInfo::GetFirstHost() const { - lock_guard lock(mCandidateHostsMux); - if (mCandidateHosts.empty() || mCandidateHosts[0].empty()) { - return ""; - } - return mCandidateHosts[0][0].GetHostname(); -} +// string CandidateHostsInfo::GetFirstHost() const { +// lock_guard lock(mCandidateHostsMux); +// if (mCandidateHosts.empty() || mCandidateHosts[0].empty()) { +// return ""; +// } +// return mCandidateHosts[0][0].GetHostname(); +// } -bool CandidateHostsInfo::HasValidHost() const { - lock_guard lock(mCurrentHostMux); - return !mCurrentHost.empty(); -} +// bool CandidateHostsInfo::HasValidHost() const { +// lock_guard lock(mCurrentHostMux); +// return !mCurrentHost.empty(); +// } -void CandidateHostsInfo::SetCurrentHost(const string& host) { - lock_guard lock(mCurrentHostMux); - mCurrentHost = host; -} +// void CandidateHostsInfo::SetCurrentHost(const string& host) { +// lock_guard lock(mCurrentHostMux); +// mCurrentHost = host; +// } // SLSClientManager::ProbeNetworkHttpRequest::ProbeNetworkHttpRequest( // const string& region, const string& project, EndpointMode mode, const string& host, bool httpsFlag) @@ -614,46 +614,46 @@ bool SLSClientManager::GetAccessKey(const string& aliuid, // return it->second.UpdateHostInfo(host, latency); // } -shared_ptr SLSClientManager::GetCandidateHostsInfo(const string& project, const string& endpoint) { - if (endpoint.empty()) { - // this should only occur on first update, where we try find any available info - lock_guard lock(mCandidateHostsInfosMapMux); - auto& hostsInfo = mProjectCandidateHostsInfosMap[project]; - for (auto& item : hostsInfo) { - if (item.expired()) { - continue; - } - return item.lock(); - } - return nullptr; - } +// shared_ptr SLSClientManager::GetCandidateHostsInfo(const string& project, const string& endpoint) { +// if (endpoint.empty()) { +// // this should only occur on first update, where we try find any available info +// lock_guard lock(mCandidateHostsInfosMapMux); +// auto& hostsInfo = mProjectCandidateHostsInfosMap[project]; +// for (auto& item : hostsInfo) { +// if (item.expired()) { +// continue; +// } +// return item.lock(); +// } +// return nullptr; +// } - string standardEndpoint = ExtractEndpoint(endpoint); - { - lock_guard lock(mCandidateHostsInfosMapMux); - auto& hostsInfo = mProjectCandidateHostsInfosMap[project]; - for (auto& item : hostsInfo) { - if (item.expired()) { - continue; - } - auto info = item.lock(); - if (info->GetFirstHost() == project + "." + standardEndpoint) { - return info; - } - } - } - auto info = make_shared(project, "", EndpointMode::CUSTOM); - info->UpdateHosts({EndpointMode::CUSTOM, {standardEndpoint}}); - // manually set the endpoint to be available - info->UpdateHostInfo(info->GetFirstHost(), chrono::milliseconds(10)); - info->SelectBestHost(); - info->SetInitialized(); - { - lock_guard lock(mCandidateHostsInfosMapMux); - mProjectCandidateHostsInfosMap[project].emplace_back(info); - } - return info; -} +// string standardEndpoint = ExtractEndpoint(endpoint); +// { +// lock_guard lock(mCandidateHostsInfosMapMux); +// auto& hostsInfo = mProjectCandidateHostsInfosMap[project]; +// for (auto& item : hostsInfo) { +// if (item.expired()) { +// continue; +// } +// auto info = item.lock(); +// if (info->GetFirstHost() == project + "." + standardEndpoint) { +// return info; +// } +// } +// } +// auto info = make_shared(project, "", EndpointMode::CUSTOM); +// info->UpdateHosts({EndpointMode::CUSTOM, {standardEndpoint}}); +// // manually set the endpoint to be available +// info->UpdateHostInfo(info->GetFirstHost(), chrono::milliseconds(10)); +// info->SelectBestHost(); +// info->SetInitialized(); +// { +// lock_guard lock(mCandidateHostsInfosMapMux); +// mProjectCandidateHostsInfosMap[project].emplace_back(info); +// } +// return info; +// } bool SLSClientManager::UsingHttps(const string& region) const { return true; @@ -1057,11 +1057,11 @@ bool SLSClientManager::PingEndpoint(const string& host, const string& path) { response); } -#ifdef APSARA_UNIT_TEST_MAIN -void SLSClientManager::Clear() { - mProjectCandidateHostsInfosMap.clear(); -} -#endif +// #ifdef APSARA_UNIT_TEST_MAIN +// void SLSClientManager::Clear() { +// mProjectCandidateHostsInfosMap.clear(); +// } +// #endif void PreparePostLogStoreLogsRequest(const string& accessKeyId, const string& accessKeySecret, diff --git a/core/plugin/flusher/sls/SLSClientManager.h b/core/plugin/flusher/sls/SLSClientManager.h index 1e87509031..aa92dc3d65 100644 --- a/core/plugin/flusher/sls/SLSClientManager.h +++ b/core/plugin/flusher/sls/SLSClientManager.h @@ -41,79 +41,79 @@ namespace logtail { -enum class EndpointMode { DEFAULT, ACCELERATE, CUSTOM }; -const std::string& EndpointModeToString(EndpointMode mode); - -struct CandidateEndpoints { - // currently only remote endpoints can be updated - EndpointMode mMode = EndpointMode::DEFAULT; - // when mMode = ACCELERATE, mLocalEndpoints should be empty - std::vector mLocalEndpoints; - std::vector mRemoteEndpoints; -}; - -class HostInfo { -public: - HostInfo(const std::string& hostname) : mHostname(hostname) {} - HostInfo(const HostInfo& rhs) : mHostname(rhs.mHostname) { mLatency = rhs.GetLatency(); } - - const std::string& GetHostname() const { return mHostname; } - std::chrono::milliseconds GetLatency() const; - void SetLatency(const std::chrono::milliseconds& latency); - void SetForbidden(); - bool IsForbidden() const; - -private: - // normally in the form of ., except for the real ip scene, which equals to endpoint - const std::string mHostname; - - mutable std::mutex mLatencyMux; - std::chrono::milliseconds mLatency = std::chrono::milliseconds::max(); -}; - -class CandidateHostsInfo { -public: - CandidateHostsInfo(const std::string& project, const std::string& region, EndpointMode mode) - : mProject(project), mRegion(region), mMode(mode) {} - - void UpdateHosts(const CandidateEndpoints& regionEndpoints); - bool UpdateHostInfo(const std::string& hostname, const std::chrono::milliseconds& latency); - void GetProbeHosts(std::vector& hosts) const; - void GetAllHosts(std::vector& hosts) const; - void SelectBestHost(); - std::string GetCurrentHost() const; - std::string GetFirstHost() const; - const std::string& GetProject() const { return mProject; } - const std::string& GetRegion() const { return mRegion; } - EndpointMode GetMode() const { return mMode; } - bool IsInitialized() const { return mInitialized.load(); } - void SetInitialized() { mInitialized = true; } - -private: - bool HasValidHost() const; - void SetCurrentHost(const std::string& host); - - // for real ip scene, mProject is empty - const std::string mProject; - const std::string mRegion; - const EndpointMode mMode; - std::atomic_bool mInitialized = false; +// enum class EndpointMode { DEFAULT, ACCELERATE, CUSTOM }; +// const std::string& EndpointModeToString(EndpointMode mode); + +// struct CandidateEndpoints { +// // currently only remote endpoints can be updated +// EndpointMode mMode = EndpointMode::DEFAULT; +// // when mMode = ACCELERATE, mLocalEndpoints should be empty +// std::vector mLocalEndpoints; +// std::vector mRemoteEndpoints; +// }; + +// class HostInfo { +// public: +// HostInfo(const std::string& hostname) : mHostname(hostname) {} +// HostInfo(const HostInfo& rhs) : mHostname(rhs.mHostname) { mLatency = rhs.GetLatency(); } + +// const std::string& GetHostname() const { return mHostname; } +// std::chrono::milliseconds GetLatency() const; +// void SetLatency(const std::chrono::milliseconds& latency); +// void SetForbidden(); +// bool IsForbidden() const; + +// private: +// // normally in the form of ., except for the real ip scene, which equals to endpoint +// const std::string mHostname; + +// mutable std::mutex mLatencyMux; +// std::chrono::milliseconds mLatency = std::chrono::milliseconds::max(); +// }; + +// class CandidateHostsInfo { +// public: +// CandidateHostsInfo(const std::string& project, const std::string& region, EndpointMode mode) +// : mProject(project), mRegion(region), mMode(mode) {} + +// void UpdateHosts(const CandidateEndpoints& regionEndpoints); +// bool UpdateHostInfo(const std::string& hostname, const std::chrono::milliseconds& latency); +// void GetProbeHosts(std::vector& hosts) const; +// void GetAllHosts(std::vector& hosts) const; +// void SelectBestHost(); +// std::string GetCurrentHost() const; +// std::string GetFirstHost() const; +// const std::string& GetProject() const { return mProject; } +// const std::string& GetRegion() const { return mRegion; } +// EndpointMode GetMode() const { return mMode; } +// bool IsInitialized() const { return mInitialized.load(); } +// void SetInitialized() { mInitialized = true; } + +// private: +// bool HasValidHost() const; +// void SetCurrentHost(const std::string& host); + +// // for real ip scene, mProject is empty +// const std::string mProject; +// const std::string mRegion; +// const EndpointMode mMode; +// std::atomic_bool mInitialized = false; + +// mutable std::mutex mCurrentHostMux; +// std::string mCurrentHost; + +// // mMode = DEFAULT: each mCandidateHosts element has exactly one element +// // mMode = ACCELERATE: mCandidateHosts.size() == 1 && mCandidateHosts[0].size() == 2 +// // mMode = CUSTOM: mCandidateHosts.size() == 1 && mCandidateHosts[0].size() == 1 +// mutable std::mutex mCandidateHostsMux; +// std::vector> mCandidateHosts; - mutable std::mutex mCurrentHostMux; - std::string mCurrentHost; - - // mMode = DEFAULT: each mCandidateHosts element has exactly one element - // mMode = ACCELERATE: mCandidateHosts.size() == 1 && mCandidateHosts[0].size() == 2 - // mMode = CUSTOM: mCandidateHosts.size() == 1 && mCandidateHosts[0].size() == 1 - mutable std::mutex mCandidateHostsMux; - std::vector> mCandidateHosts; - -#ifdef APSARA_UNIT_TEST_MAIN - friend class CandidateHostsInfoUnittest; - friend class SLSClientManagerUnittest; - friend class EnterpriseSLSClientManagerUnittest; -#endif -}; +// #ifdef APSARA_UNIT_TEST_MAIN +// friend class CandidateHostsInfoUnittest; +// friend class SLSClientManagerUnittest; +// friend class EnterpriseSLSClientManagerUnittest; +// #endif +// }; class SLSClientManager { public: @@ -152,16 +152,16 @@ class SLSClientManager { // latency); // only for open source - std::shared_ptr GetCandidateHostsInfo(const std::string& project, const std::string& endpoint); + // std::shared_ptr GetCandidateHostsInfo(const std::string& project, const std::string& endpoint); virtual bool UsingHttps(const std::string& region) const; // void UpdateOutdatedRealIpRegions(const std::string& region); // std::string GetRealIp(const std::string& region) const; -#ifdef APSARA_UNIT_TEST_MAIN - virtual void Clear(); -#endif +// #ifdef APSARA_UNIT_TEST_MAIN +// virtual void Clear(); +// #endif protected: SLSClientManager() = default; @@ -208,9 +208,9 @@ class SLSClientManager { // mutable std::mutex mHttpsRegionsMux; // std::unordered_set mHttpsRegions; - mutable std::mutex mCandidateHostsInfosMapMux; + // mutable std::mutex mCandidateHostsInfosMapMux; // only custom mode is supported, and one project supports multiple custom endpoints - std::map>> mProjectCandidateHostsInfosMap; + // std::map>> mProjectCandidateHostsInfosMap; // CURLM* mUnInitializedHostProbeClient = nullptr; // mutable std::mutex mUnInitializedCandidateHostsInfosMux; diff --git a/core/unittest/flusher/FlusherSLSUnittest.cpp b/core/unittest/flusher/FlusherSLSUnittest.cpp index cdb02dab1d..f402a093c0 100644 --- a/core/unittest/flusher/FlusherSLSUnittest.cpp +++ b/core/unittest/flusher/FlusherSLSUnittest.cpp @@ -88,7 +88,9 @@ class FlusherSLSUnittest : public testing::Test { QueueKeyManager::GetInstance()->Clear(); SenderQueueManager::GetInstance()->Clear(); ExactlyOnceQueueManager::GetInstance()->Clear(); - SLSClientManager::GetInstance()->Clear(); +#ifdef __ENTERPRISE__ + EnterpriseSLSClientManager::GetInstance()->Clear(); +#endif } private: @@ -123,7 +125,6 @@ void FlusherSLSUnittest::OnSuccessfulInit() { APSARA_TEST_EQUAL(STRING_FLAG(default_region_name), flusher->mRegion); #ifndef __ENTERPRISE__ APSARA_TEST_EQUAL("test_region.log.aliyuncs.com", flusher->mEndpoint); - APSARA_TEST_EQUAL("test_project.test_region.log.aliyuncs.com", flusher->mCandidateHostsInfo->GetFirstHost()); #endif APSARA_TEST_EQUAL("", flusher->mAliuid); APSARA_TEST_EQUAL(sls_logs::SlsTelemetryType::SLS_TELEMETRY_TYPE_LOGS, flusher->mTelemetryType); @@ -245,7 +246,7 @@ void FlusherSLSUnittest::OnSuccessfulInit() { #ifdef __ENTERPRISE__ // EndpointMode && Endpoint - SLSClientManager::GetInstance()->Clear(); + EnterpriseSLSClientManager::GetInstance()->Clear(); // Endpoint ignored in acclerate mode configStr = R"( { @@ -338,7 +339,7 @@ void FlusherSLSUnittest::OnSuccessfulInit() { flusher->SetContext(ctx); flusher->SetMetricsRecordRef(FlusherSLS::sName, "1"); APSARA_TEST_TRUE(flusher->Init(configJson, optionalGoPipeline)); - APSARA_TEST_EQUAL("test_project.test_region.log.aliyuncs.com", flusher->mCandidateHostsInfo->GetFirstHost()); + APSARA_TEST_EQUAL("test_region.log.aliyuncs.com", flusher->mEndpoint); SenderQueueManager::GetInstance()->Clear(); #endif diff --git a/core/unittest/flusher/SLSClientManagerUnittest.cpp b/core/unittest/flusher/SLSClientManagerUnittest.cpp index 83fb8f1c26..01d23abe57 100644 --- a/core/unittest/flusher/SLSClientManagerUnittest.cpp +++ b/core/unittest/flusher/SLSClientManagerUnittest.cpp @@ -27,414 +27,414 @@ using namespace std; namespace logtail { -class HostInfoUnittest : public ::testing::Test { -public: - void TestHostname(); - void TestLatency(); +// class HostInfoUnittest : public ::testing::Test { +// public: +// void TestHostname(); +// void TestLatency(); -private: - const std::string mHostname = "project.endpoint"; -}; +// private: +// const std::string mHostname = "project.endpoint"; +// }; -void HostInfoUnittest::TestHostname() { - HostInfo hostInfo(mHostname); - APSARA_TEST_EQUAL(mHostname, hostInfo.GetHostname()); -} +// void HostInfoUnittest::TestHostname() { +// HostInfo hostInfo(mHostname); +// APSARA_TEST_EQUAL(mHostname, hostInfo.GetHostname()); +// } -void HostInfoUnittest::TestLatency() { - HostInfo hostInfo(mHostname); - APSARA_TEST_TRUE(hostInfo.IsForbidden()); +// void HostInfoUnittest::TestLatency() { +// HostInfo hostInfo(mHostname); +// APSARA_TEST_TRUE(hostInfo.IsForbidden()); - auto latency = chrono::milliseconds(100); - hostInfo.SetLatency(latency); - APSARA_TEST_EQUAL(latency, hostInfo.GetLatency()); - APSARA_TEST_FALSE(hostInfo.IsForbidden()); +// auto latency = chrono::milliseconds(100); +// hostInfo.SetLatency(latency); +// APSARA_TEST_EQUAL(latency, hostInfo.GetLatency()); +// APSARA_TEST_FALSE(hostInfo.IsForbidden()); - hostInfo.SetForbidden(); - APSARA_TEST_EQUAL(chrono::milliseconds::max(), hostInfo.GetLatency()); -} +// hostInfo.SetForbidden(); +// APSARA_TEST_EQUAL(chrono::milliseconds::max(), hostInfo.GetLatency()); +// } -UNIT_TEST_CASE(HostInfoUnittest, TestHostname) -UNIT_TEST_CASE(HostInfoUnittest, TestLatency) +// UNIT_TEST_CASE(HostInfoUnittest, TestHostname) +// UNIT_TEST_CASE(HostInfoUnittest, TestLatency) -class CandidateHostsInfoUnittest : public ::testing::Test { -public: - void TestBasicInfo(); - void TestHostsInfo(); - void TestUpdateHosts(); - void TestFirstHost(); +// class CandidateHostsInfoUnittest : public ::testing::Test { +// public: +// void TestBasicInfo(); +// void TestHostsInfo(); +// void TestUpdateHosts(); +// void TestFirstHost(); + +// private: +// const string mProject = "project"; +// const string mRegion = "region"; +// const EndpointMode mMode = EndpointMode::DEFAULT; +// }; -private: - const string mProject = "project"; - const string mRegion = "region"; - const EndpointMode mMode = EndpointMode::DEFAULT; -}; +// void CandidateHostsInfoUnittest::TestBasicInfo() { +// CandidateHostsInfo info(mProject, mRegion, mMode); +// APSARA_TEST_EQUAL(mProject, info.GetProject()); +// APSARA_TEST_EQUAL(mRegion, info.GetRegion()); +// APSARA_TEST_EQUAL(mMode, info.GetMode()); +// } -void CandidateHostsInfoUnittest::TestBasicInfo() { - CandidateHostsInfo info(mProject, mRegion, mMode); - APSARA_TEST_EQUAL(mProject, info.GetProject()); - APSARA_TEST_EQUAL(mRegion, info.GetRegion()); - APSARA_TEST_EQUAL(mMode, info.GetMode()); -} +// void CandidateHostsInfoUnittest::TestHostsInfo() { +// const string host1 = mProject + ".endpoint_1"; +// const string host21 = mProject + ".endpoint_2_1"; +// const string host22 = mProject + ".endpoint_2_2"; +// const string host3 = mProject + ".endpoint_3"; -void CandidateHostsInfoUnittest::TestHostsInfo() { - const string host1 = mProject + ".endpoint_1"; - const string host21 = mProject + ".endpoint_2_1"; - const string host22 = mProject + ".endpoint_2_2"; - const string host3 = mProject + ".endpoint_3"; - - CandidateHostsInfo info(mProject, mRegion, mMode); - info.mCandidateHosts.push_back({host1}); - info.mCandidateHosts.push_back({host21, host22}); - info.mCandidateHosts.push_back({host3}); - - // initialized - APSARA_TEST_TRUE(info.GetCurrentHost().empty()); - { - vector res; - info.GetAllHosts(res); - APSARA_TEST_EQUAL(4U, res.size()); - APSARA_TEST_EQUAL(host1, res[0]); - APSARA_TEST_EQUAL(host21, res[1]); - APSARA_TEST_EQUAL(host22, res[2]); - APSARA_TEST_EQUAL(host3, res[3]); - } - { - vector res; - info.GetProbeHosts(res); - APSARA_TEST_EQUAL(4U, res.size()); - APSARA_TEST_EQUAL(host1, res[0]); - APSARA_TEST_EQUAL(host21, res[1]); - APSARA_TEST_EQUAL(host22, res[2]); - APSARA_TEST_EQUAL(host3, res[3]); - } - - // no valid host - info.SelectBestHost(); - APSARA_TEST_TRUE(info.GetCurrentHost().empty()); - - // some hosts become valid - info.UpdateHostInfo(host21, chrono::milliseconds(100)); - info.SelectBestHost(); - - APSARA_TEST_EQUAL(host21, info.GetCurrentHost()); - { - vector res; - info.GetProbeHosts(res); - APSARA_TEST_EQUAL(1U, res.size()); - APSARA_TEST_EQUAL(host21, res[0]); - } - { - vector res; - info.GetAllHosts(res); - APSARA_TEST_EQUAL(4U, res.size()); - APSARA_TEST_EQUAL(host1, res[0]); - APSARA_TEST_EQUAL(host21, res[1]); - APSARA_TEST_EQUAL(host22, res[2]); - APSARA_TEST_EQUAL(host3, res[3]); - } - - // host with the same priority as the current one has lower latency - info.UpdateHostInfo(host22, chrono::milliseconds(50)); - info.SelectBestHost(); - - APSARA_TEST_EQUAL(host22, info.GetCurrentHost()); - { - vector res; - info.GetProbeHosts(res); - APSARA_TEST_EQUAL(2U, res.size()); - APSARA_TEST_EQUAL(host21, res[0]); - APSARA_TEST_EQUAL(host22, res[1]); - } - { - vector res; - info.GetAllHosts(res); - APSARA_TEST_EQUAL(4U, res.size()); - APSARA_TEST_EQUAL(host1, res[0]); - APSARA_TEST_EQUAL(host21, res[1]); - APSARA_TEST_EQUAL(host22, res[2]); - APSARA_TEST_EQUAL(host3, res[3]); - } - - // host with higher priority becomes valid - info.UpdateHostInfo(host1, chrono::milliseconds(200)); - info.SelectBestHost(); - - APSARA_TEST_EQUAL(host1, info.GetCurrentHost()); - { - vector res; - info.GetProbeHosts(res); - APSARA_TEST_EQUAL(3U, res.size()); - APSARA_TEST_EQUAL(host1, res[0]); - APSARA_TEST_EQUAL(host21, res[1]); - APSARA_TEST_EQUAL(host22, res[2]); - } - { - vector res; - info.GetAllHosts(res); - APSARA_TEST_EQUAL(4U, res.size()); - APSARA_TEST_EQUAL(host1, res[0]); - APSARA_TEST_EQUAL(host21, res[1]); - APSARA_TEST_EQUAL(host22, res[2]); - APSARA_TEST_EQUAL(host3, res[3]); - } - - // no change - info.SelectBestHost(); - - APSARA_TEST_EQUAL(host1, info.GetCurrentHost()); - { - vector res; - info.GetProbeHosts(res); - APSARA_TEST_EQUAL(3U, res.size()); - APSARA_TEST_EQUAL(host1, res[0]); - APSARA_TEST_EQUAL(host21, res[1]); - APSARA_TEST_EQUAL(host22, res[2]); - } - { - vector res; - info.GetAllHosts(res); - APSARA_TEST_EQUAL(4U, res.size()); - APSARA_TEST_EQUAL(host1, res[0]); - APSARA_TEST_EQUAL(host21, res[1]); - APSARA_TEST_EQUAL(host22, res[2]); - APSARA_TEST_EQUAL(host3, res[3]); - } - - // all hosts becomes invalid - info.UpdateHostInfo(host1, chrono::milliseconds::max()); - info.UpdateHostInfo(host21, chrono::milliseconds::max()); - info.UpdateHostInfo(host22, chrono::milliseconds::max()); - info.UpdateHostInfo(host3, chrono::milliseconds::max()); - info.SelectBestHost(); - - APSARA_TEST_TRUE(info.GetCurrentHost().empty()); - { - vector res; - info.GetProbeHosts(res); - APSARA_TEST_EQUAL(4U, res.size()); - APSARA_TEST_EQUAL(host1, res[0]); - APSARA_TEST_EQUAL(host21, res[1]); - APSARA_TEST_EQUAL(host22, res[2]); - APSARA_TEST_EQUAL(host3, res[3]); - } - { - vector res; - info.GetAllHosts(res); - APSARA_TEST_EQUAL(4U, res.size()); - APSARA_TEST_EQUAL(host1, res[0]); - APSARA_TEST_EQUAL(host21, res[1]); - APSARA_TEST_EQUAL(host22, res[2]); - APSARA_TEST_EQUAL(host3, res[3]); - } -} +// CandidateHostsInfo info(mProject, mRegion, mMode); +// info.mCandidateHosts.push_back({host1}); +// info.mCandidateHosts.push_back({host21, host22}); +// info.mCandidateHosts.push_back({host3}); -void CandidateHostsInfoUnittest::TestUpdateHosts() { - const string region = "region"; - const string publicEndpoint = region + ".log.aliyuncs.com"; - const string privateEndpoint = region + "-intranet.log.aliyuncs.com"; - const string internalEndpoint = region + "-internal.log.aliyuncs.com"; - const string globalEndpoint = "log-global.aliyuncs.com"; - const string customEndpoint = "custom.endpoint"; - const string publicHost = mProject + "." + publicEndpoint; - const string privateHost = mProject + "." + privateEndpoint; - const string internalHost = mProject + "." + internalEndpoint; - const string globalHost = mProject + "." + globalEndpoint; - const string customHost = mProject + "." + customEndpoint; -#ifdef __ENTERPRISE__ - { - // default mode - CandidateHostsInfo info("project", region, EndpointMode::DEFAULT); - - // from no remote endpoints - CandidateEndpoints regionEndpoints{EndpointMode::DEFAULT, {publicEndpoint}, {}}; - info.UpdateHosts(regionEndpoints); - APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); - APSARA_TEST_EQUAL(1U, info.mCandidateHosts[0].size()); - APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][0].GetHostname()); - APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[0][0].GetLatency()); - - info.UpdateHostInfo(publicHost, chrono::milliseconds(100)); - - // to with remote endpoints - regionEndpoints.mRemoteEndpoints = {privateEndpoint, publicEndpoint}; - info.UpdateHosts(regionEndpoints); - APSARA_TEST_EQUAL(2U, info.mCandidateHosts.size()); - APSARA_TEST_EQUAL(1U, info.mCandidateHosts[0].size()); - APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][0].GetHostname()); - APSARA_TEST_EQUAL(chrono::milliseconds(100), info.mCandidateHosts[0][0].GetLatency()); - APSARA_TEST_EQUAL(1U, info.mCandidateHosts[1].size()); - APSARA_TEST_EQUAL(privateHost, info.mCandidateHosts[1][0].GetHostname()); - APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[1][0].GetLatency()); - - info.UpdateHostInfo(publicHost, chrono::milliseconds(50)); - info.UpdateHostInfo(privateHost, chrono::milliseconds(150)); - - // to updated remote endpoints - regionEndpoints.mRemoteEndpoints = {internalEndpoint, privateEndpoint, publicEndpoint}; - info.UpdateHosts(regionEndpoints); - APSARA_TEST_EQUAL(3U, info.mCandidateHosts.size()); - APSARA_TEST_EQUAL(1U, info.mCandidateHosts[0].size()); - APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][0].GetHostname()); - APSARA_TEST_EQUAL(chrono::milliseconds(50), info.mCandidateHosts[0][0].GetLatency()); - APSARA_TEST_EQUAL(1U, info.mCandidateHosts[1].size()); - APSARA_TEST_EQUAL(internalHost, info.mCandidateHosts[1][0].GetHostname()); - APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[1][0].GetLatency()); - APSARA_TEST_EQUAL(1U, info.mCandidateHosts[2].size()); - APSARA_TEST_EQUAL(privateHost, info.mCandidateHosts[2][0].GetHostname()); - APSARA_TEST_EQUAL(chrono::milliseconds(150), info.mCandidateHosts[2][0].GetLatency()); - } - { - // accelerate mode - { - CandidateHostsInfo info("project", region, EndpointMode::ACCELERATE); - // from no remote endpoints - CandidateEndpoints regionEndpoints{EndpointMode::DEFAULT, {publicEndpoint}, {}}; - info.UpdateHosts(regionEndpoints); - APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); - APSARA_TEST_EQUAL(1U, info.mCandidateHosts[0].size()); - APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); - APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[0][0].GetLatency()); - - info.UpdateHostInfo(globalHost, chrono::milliseconds(100)); - - // to with remote endpoints - regionEndpoints.mRemoteEndpoints = {privateEndpoint, publicEndpoint}; - info.UpdateHosts(regionEndpoints); - APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); - APSARA_TEST_EQUAL(2U, info.mCandidateHosts[0].size()); - APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); - APSARA_TEST_EQUAL(chrono::milliseconds(100), info.mCandidateHosts[0][0].GetLatency()); - APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][1].GetHostname()); - APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[0][1].GetLatency()); - - info.UpdateHostInfo(globalHost, chrono::milliseconds(50)); - info.UpdateHostInfo(publicHost, chrono::milliseconds(150)); - - // to updated remote endpoints - regionEndpoints.mRemoteEndpoints = {internalEndpoint, privateEndpoint, publicEndpoint}; - info.UpdateHosts(regionEndpoints); - APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); - APSARA_TEST_EQUAL(2U, info.mCandidateHosts[0].size()); - APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); - APSARA_TEST_EQUAL(chrono::milliseconds(50), info.mCandidateHosts[0][0].GetLatency()); - APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][1].GetHostname()); - APSARA_TEST_EQUAL(chrono::milliseconds(150), info.mCandidateHosts[0][1].GetLatency()); - } - { - CandidateHostsInfo info("project", region, EndpointMode::ACCELERATE); - // from no remote endpoints - CandidateEndpoints regionEndpoints{EndpointMode::ACCELERATE, {globalEndpoint}, {}}; - info.UpdateHosts(regionEndpoints); - APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); - APSARA_TEST_EQUAL(1U, info.mCandidateHosts[0].size()); - APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); - APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[0][0].GetLatency()); - - info.UpdateHostInfo(globalHost, chrono::milliseconds(100)); - - // to with remote endpoints - regionEndpoints.mRemoteEndpoints = {privateEndpoint, publicEndpoint}; - info.UpdateHosts(regionEndpoints); - APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); - APSARA_TEST_EQUAL(2U, info.mCandidateHosts[0].size()); - APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); - APSARA_TEST_EQUAL(chrono::milliseconds(100), info.mCandidateHosts[0][0].GetLatency()); - APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][1].GetHostname()); - APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[0][1].GetLatency()); - - info.UpdateHostInfo(globalHost, chrono::milliseconds(50)); - info.UpdateHostInfo(publicHost, chrono::milliseconds(150)); - - // to updated remote endpoints - regionEndpoints.mRemoteEndpoints = {internalEndpoint, privateEndpoint, publicEndpoint}; - info.UpdateHosts(regionEndpoints); - APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); - APSARA_TEST_EQUAL(2U, info.mCandidateHosts[0].size()); - APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); - APSARA_TEST_EQUAL(chrono::milliseconds(50), info.mCandidateHosts[0][0].GetLatency()); - APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][1].GetHostname()); - APSARA_TEST_EQUAL(chrono::milliseconds(150), info.mCandidateHosts[0][1].GetLatency()); - } - { - CandidateHostsInfo info("project", region, EndpointMode::ACCELERATE); - // from no remote endpoints - CandidateEndpoints regionEndpoints{EndpointMode::CUSTOM, {customEndpoint}, {}}; - info.UpdateHosts(regionEndpoints); - APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); - APSARA_TEST_EQUAL(1U, info.mCandidateHosts[0].size()); - APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); - APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[0][0].GetLatency()); - - info.UpdateHostInfo(globalHost, chrono::milliseconds(100)); - - // to with remote endpoints - regionEndpoints.mRemoteEndpoints = {privateEndpoint, publicEndpoint}; - info.UpdateHosts(regionEndpoints); - APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); - APSARA_TEST_EQUAL(2U, info.mCandidateHosts[0].size()); - APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); - APSARA_TEST_EQUAL(chrono::milliseconds(100), info.mCandidateHosts[0][0].GetLatency()); - APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][1].GetHostname()); - APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[0][1].GetLatency()); - - info.UpdateHostInfo(globalHost, chrono::milliseconds(50)); - info.UpdateHostInfo(publicHost, chrono::milliseconds(150)); - - // to updated remote endpoints - regionEndpoints.mRemoteEndpoints = {internalEndpoint, privateEndpoint, publicEndpoint}; - info.UpdateHosts(regionEndpoints); - APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); - APSARA_TEST_EQUAL(2U, info.mCandidateHosts[0].size()); - APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); - APSARA_TEST_EQUAL(chrono::milliseconds(50), info.mCandidateHosts[0][0].GetLatency()); - APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][1].GetHostname()); - APSARA_TEST_EQUAL(chrono::milliseconds(150), info.mCandidateHosts[0][1].GetLatency()); - } - } -#endif - { - // custom mode - CandidateHostsInfo info("project", region, EndpointMode::CUSTOM); - - // from no remote endpoints - CandidateEndpoints regionEndpoints{EndpointMode::CUSTOM, {customEndpoint}, {}}; - info.UpdateHosts(regionEndpoints); - APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); - APSARA_TEST_EQUAL(1U, info.mCandidateHosts[0].size()); - APSARA_TEST_EQUAL(customHost, info.mCandidateHosts[0][0].GetHostname()); - APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[0][0].GetLatency()); - - info.UpdateHostInfo(customHost, chrono::milliseconds(100)); - - // to with remote endpoints - regionEndpoints.mRemoteEndpoints = {privateEndpoint, publicEndpoint}; - info.UpdateHosts(regionEndpoints); - APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); - APSARA_TEST_EQUAL(1U, info.mCandidateHosts[0].size()); - APSARA_TEST_EQUAL(customHost, info.mCandidateHosts[0][0].GetHostname()); - APSARA_TEST_EQUAL(chrono::milliseconds(100), info.mCandidateHosts[0][0].GetLatency()); - } -} +// // initialized +// APSARA_TEST_TRUE(info.GetCurrentHost().empty()); +// { +// vector res; +// info.GetAllHosts(res); +// APSARA_TEST_EQUAL(4U, res.size()); +// APSARA_TEST_EQUAL(host1, res[0]); +// APSARA_TEST_EQUAL(host21, res[1]); +// APSARA_TEST_EQUAL(host22, res[2]); +// APSARA_TEST_EQUAL(host3, res[3]); +// } +// { +// vector res; +// info.GetProbeHosts(res); +// APSARA_TEST_EQUAL(4U, res.size()); +// APSARA_TEST_EQUAL(host1, res[0]); +// APSARA_TEST_EQUAL(host21, res[1]); +// APSARA_TEST_EQUAL(host22, res[2]); +// APSARA_TEST_EQUAL(host3, res[3]); +// } -void CandidateHostsInfoUnittest::TestFirstHost() { - const string host1 = mProject + ".endpoint_1"; - const string host2 = mProject + ".endpoint_2"; +// // no valid host +// info.SelectBestHost(); +// APSARA_TEST_TRUE(info.GetCurrentHost().empty()); - CandidateHostsInfo info(mProject, mRegion, mMode); - APSARA_TEST_EQUAL("", info.GetFirstHost()); +// // some hosts become valid +// info.UpdateHostInfo(host21, chrono::milliseconds(100)); +// info.SelectBestHost(); - info.mCandidateHosts.push_back({host1, host2}); - APSARA_TEST_EQUAL(host1, info.GetFirstHost()); +// APSARA_TEST_EQUAL(host21, info.GetCurrentHost()); +// { +// vector res; +// info.GetProbeHosts(res); +// APSARA_TEST_EQUAL(1U, res.size()); +// APSARA_TEST_EQUAL(host21, res[0]); +// } +// { +// vector res; +// info.GetAllHosts(res); +// APSARA_TEST_EQUAL(4U, res.size()); +// APSARA_TEST_EQUAL(host1, res[0]); +// APSARA_TEST_EQUAL(host21, res[1]); +// APSARA_TEST_EQUAL(host22, res[2]); +// APSARA_TEST_EQUAL(host3, res[3]); +// } - info.mCandidateHosts[0].clear(); - APSARA_TEST_EQUAL("", info.GetFirstHost()); -} +// // host with the same priority as the current one has lower latency +// info.UpdateHostInfo(host22, chrono::milliseconds(50)); +// info.SelectBestHost(); -UNIT_TEST_CASE(CandidateHostsInfoUnittest, TestBasicInfo) -UNIT_TEST_CASE(CandidateHostsInfoUnittest, TestHostsInfo) -UNIT_TEST_CASE(CandidateHostsInfoUnittest, TestUpdateHosts) -UNIT_TEST_CASE(CandidateHostsInfoUnittest, TestFirstHost) +// APSARA_TEST_EQUAL(host22, info.GetCurrentHost()); +// { +// vector res; +// info.GetProbeHosts(res); +// APSARA_TEST_EQUAL(2U, res.size()); +// APSARA_TEST_EQUAL(host21, res[0]); +// APSARA_TEST_EQUAL(host22, res[1]); +// } +// { +// vector res; +// info.GetAllHosts(res); +// APSARA_TEST_EQUAL(4U, res.size()); +// APSARA_TEST_EQUAL(host1, res[0]); +// APSARA_TEST_EQUAL(host21, res[1]); +// APSARA_TEST_EQUAL(host22, res[2]); +// APSARA_TEST_EQUAL(host3, res[3]); +// } + +// // host with higher priority becomes valid +// info.UpdateHostInfo(host1, chrono::milliseconds(200)); +// info.SelectBestHost(); + +// APSARA_TEST_EQUAL(host1, info.GetCurrentHost()); +// { +// vector res; +// info.GetProbeHosts(res); +// APSARA_TEST_EQUAL(3U, res.size()); +// APSARA_TEST_EQUAL(host1, res[0]); +// APSARA_TEST_EQUAL(host21, res[1]); +// APSARA_TEST_EQUAL(host22, res[2]); +// } +// { +// vector res; +// info.GetAllHosts(res); +// APSARA_TEST_EQUAL(4U, res.size()); +// APSARA_TEST_EQUAL(host1, res[0]); +// APSARA_TEST_EQUAL(host21, res[1]); +// APSARA_TEST_EQUAL(host22, res[2]); +// APSARA_TEST_EQUAL(host3, res[3]); +// } + +// // no change +// info.SelectBestHost(); + +// APSARA_TEST_EQUAL(host1, info.GetCurrentHost()); +// { +// vector res; +// info.GetProbeHosts(res); +// APSARA_TEST_EQUAL(3U, res.size()); +// APSARA_TEST_EQUAL(host1, res[0]); +// APSARA_TEST_EQUAL(host21, res[1]); +// APSARA_TEST_EQUAL(host22, res[2]); +// } +// { +// vector res; +// info.GetAllHosts(res); +// APSARA_TEST_EQUAL(4U, res.size()); +// APSARA_TEST_EQUAL(host1, res[0]); +// APSARA_TEST_EQUAL(host21, res[1]); +// APSARA_TEST_EQUAL(host22, res[2]); +// APSARA_TEST_EQUAL(host3, res[3]); +// } + +// // all hosts becomes invalid +// info.UpdateHostInfo(host1, chrono::milliseconds::max()); +// info.UpdateHostInfo(host21, chrono::milliseconds::max()); +// info.UpdateHostInfo(host22, chrono::milliseconds::max()); +// info.UpdateHostInfo(host3, chrono::milliseconds::max()); +// info.SelectBestHost(); + +// APSARA_TEST_TRUE(info.GetCurrentHost().empty()); +// { +// vector res; +// info.GetProbeHosts(res); +// APSARA_TEST_EQUAL(4U, res.size()); +// APSARA_TEST_EQUAL(host1, res[0]); +// APSARA_TEST_EQUAL(host21, res[1]); +// APSARA_TEST_EQUAL(host22, res[2]); +// APSARA_TEST_EQUAL(host3, res[3]); +// } +// { +// vector res; +// info.GetAllHosts(res); +// APSARA_TEST_EQUAL(4U, res.size()); +// APSARA_TEST_EQUAL(host1, res[0]); +// APSARA_TEST_EQUAL(host21, res[1]); +// APSARA_TEST_EQUAL(host22, res[2]); +// APSARA_TEST_EQUAL(host3, res[3]); +// } +// } + +// void CandidateHostsInfoUnittest::TestUpdateHosts() { +// const string region = "region"; +// const string publicEndpoint = region + ".log.aliyuncs.com"; +// const string privateEndpoint = region + "-intranet.log.aliyuncs.com"; +// const string internalEndpoint = region + "-internal.log.aliyuncs.com"; +// const string globalEndpoint = "log-global.aliyuncs.com"; +// const string customEndpoint = "custom.endpoint"; +// const string publicHost = mProject + "." + publicEndpoint; +// const string privateHost = mProject + "." + privateEndpoint; +// const string internalHost = mProject + "." + internalEndpoint; +// const string globalHost = mProject + "." + globalEndpoint; +// const string customHost = mProject + "." + customEndpoint; +// #ifdef __ENTERPRISE__ +// { +// // default mode +// CandidateHostsInfo info("project", region, EndpointMode::DEFAULT); + +// // from no remote endpoints +// CandidateEndpoints regionEndpoints{EndpointMode::DEFAULT, {publicEndpoint}, {}}; +// info.UpdateHosts(regionEndpoints); +// APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); +// APSARA_TEST_EQUAL(1U, info.mCandidateHosts[0].size()); +// APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][0].GetHostname()); +// APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[0][0].GetLatency()); + +// info.UpdateHostInfo(publicHost, chrono::milliseconds(100)); + +// // to with remote endpoints +// regionEndpoints.mRemoteEndpoints = {privateEndpoint, publicEndpoint}; +// info.UpdateHosts(regionEndpoints); +// APSARA_TEST_EQUAL(2U, info.mCandidateHosts.size()); +// APSARA_TEST_EQUAL(1U, info.mCandidateHosts[0].size()); +// APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][0].GetHostname()); +// APSARA_TEST_EQUAL(chrono::milliseconds(100), info.mCandidateHosts[0][0].GetLatency()); +// APSARA_TEST_EQUAL(1U, info.mCandidateHosts[1].size()); +// APSARA_TEST_EQUAL(privateHost, info.mCandidateHosts[1][0].GetHostname()); +// APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[1][0].GetLatency()); + +// info.UpdateHostInfo(publicHost, chrono::milliseconds(50)); +// info.UpdateHostInfo(privateHost, chrono::milliseconds(150)); + +// // to updated remote endpoints +// regionEndpoints.mRemoteEndpoints = {internalEndpoint, privateEndpoint, publicEndpoint}; +// info.UpdateHosts(regionEndpoints); +// APSARA_TEST_EQUAL(3U, info.mCandidateHosts.size()); +// APSARA_TEST_EQUAL(1U, info.mCandidateHosts[0].size()); +// APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][0].GetHostname()); +// APSARA_TEST_EQUAL(chrono::milliseconds(50), info.mCandidateHosts[0][0].GetLatency()); +// APSARA_TEST_EQUAL(1U, info.mCandidateHosts[1].size()); +// APSARA_TEST_EQUAL(internalHost, info.mCandidateHosts[1][0].GetHostname()); +// APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[1][0].GetLatency()); +// APSARA_TEST_EQUAL(1U, info.mCandidateHosts[2].size()); +// APSARA_TEST_EQUAL(privateHost, info.mCandidateHosts[2][0].GetHostname()); +// APSARA_TEST_EQUAL(chrono::milliseconds(150), info.mCandidateHosts[2][0].GetLatency()); +// } +// { +// // accelerate mode +// { +// CandidateHostsInfo info("project", region, EndpointMode::ACCELERATE); +// // from no remote endpoints +// CandidateEndpoints regionEndpoints{EndpointMode::DEFAULT, {publicEndpoint}, {}}; +// info.UpdateHosts(regionEndpoints); +// APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); +// APSARA_TEST_EQUAL(1U, info.mCandidateHosts[0].size()); +// APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); +// APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[0][0].GetLatency()); + +// info.UpdateHostInfo(globalHost, chrono::milliseconds(100)); + +// // to with remote endpoints +// regionEndpoints.mRemoteEndpoints = {privateEndpoint, publicEndpoint}; +// info.UpdateHosts(regionEndpoints); +// APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); +// APSARA_TEST_EQUAL(2U, info.mCandidateHosts[0].size()); +// APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); +// APSARA_TEST_EQUAL(chrono::milliseconds(100), info.mCandidateHosts[0][0].GetLatency()); +// APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][1].GetHostname()); +// APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[0][1].GetLatency()); + +// info.UpdateHostInfo(globalHost, chrono::milliseconds(50)); +// info.UpdateHostInfo(publicHost, chrono::milliseconds(150)); + +// // to updated remote endpoints +// regionEndpoints.mRemoteEndpoints = {internalEndpoint, privateEndpoint, publicEndpoint}; +// info.UpdateHosts(regionEndpoints); +// APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); +// APSARA_TEST_EQUAL(2U, info.mCandidateHosts[0].size()); +// APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); +// APSARA_TEST_EQUAL(chrono::milliseconds(50), info.mCandidateHosts[0][0].GetLatency()); +// APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][1].GetHostname()); +// APSARA_TEST_EQUAL(chrono::milliseconds(150), info.mCandidateHosts[0][1].GetLatency()); +// } +// { +// CandidateHostsInfo info("project", region, EndpointMode::ACCELERATE); +// // from no remote endpoints +// CandidateEndpoints regionEndpoints{EndpointMode::ACCELERATE, {globalEndpoint}, {}}; +// info.UpdateHosts(regionEndpoints); +// APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); +// APSARA_TEST_EQUAL(1U, info.mCandidateHosts[0].size()); +// APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); +// APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[0][0].GetLatency()); + +// info.UpdateHostInfo(globalHost, chrono::milliseconds(100)); + +// // to with remote endpoints +// regionEndpoints.mRemoteEndpoints = {privateEndpoint, publicEndpoint}; +// info.UpdateHosts(regionEndpoints); +// APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); +// APSARA_TEST_EQUAL(2U, info.mCandidateHosts[0].size()); +// APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); +// APSARA_TEST_EQUAL(chrono::milliseconds(100), info.mCandidateHosts[0][0].GetLatency()); +// APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][1].GetHostname()); +// APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[0][1].GetLatency()); + +// info.UpdateHostInfo(globalHost, chrono::milliseconds(50)); +// info.UpdateHostInfo(publicHost, chrono::milliseconds(150)); + +// // to updated remote endpoints +// regionEndpoints.mRemoteEndpoints = {internalEndpoint, privateEndpoint, publicEndpoint}; +// info.UpdateHosts(regionEndpoints); +// APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); +// APSARA_TEST_EQUAL(2U, info.mCandidateHosts[0].size()); +// APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); +// APSARA_TEST_EQUAL(chrono::milliseconds(50), info.mCandidateHosts[0][0].GetLatency()); +// APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][1].GetHostname()); +// APSARA_TEST_EQUAL(chrono::milliseconds(150), info.mCandidateHosts[0][1].GetLatency()); +// } +// { +// CandidateHostsInfo info("project", region, EndpointMode::ACCELERATE); +// // from no remote endpoints +// CandidateEndpoints regionEndpoints{EndpointMode::CUSTOM, {customEndpoint}, {}}; +// info.UpdateHosts(regionEndpoints); +// APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); +// APSARA_TEST_EQUAL(1U, info.mCandidateHosts[0].size()); +// APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); +// APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[0][0].GetLatency()); + +// info.UpdateHostInfo(globalHost, chrono::milliseconds(100)); + +// // to with remote endpoints +// regionEndpoints.mRemoteEndpoints = {privateEndpoint, publicEndpoint}; +// info.UpdateHosts(regionEndpoints); +// APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); +// APSARA_TEST_EQUAL(2U, info.mCandidateHosts[0].size()); +// APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); +// APSARA_TEST_EQUAL(chrono::milliseconds(100), info.mCandidateHosts[0][0].GetLatency()); +// APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][1].GetHostname()); +// APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[0][1].GetLatency()); + +// info.UpdateHostInfo(globalHost, chrono::milliseconds(50)); +// info.UpdateHostInfo(publicHost, chrono::milliseconds(150)); + +// // to updated remote endpoints +// regionEndpoints.mRemoteEndpoints = {internalEndpoint, privateEndpoint, publicEndpoint}; +// info.UpdateHosts(regionEndpoints); +// APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); +// APSARA_TEST_EQUAL(2U, info.mCandidateHosts[0].size()); +// APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); +// APSARA_TEST_EQUAL(chrono::milliseconds(50), info.mCandidateHosts[0][0].GetLatency()); +// APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][1].GetHostname()); +// APSARA_TEST_EQUAL(chrono::milliseconds(150), info.mCandidateHosts[0][1].GetLatency()); +// } +// } +// #endif +// { +// // custom mode +// CandidateHostsInfo info("project", region, EndpointMode::CUSTOM); + +// // from no remote endpoints +// CandidateEndpoints regionEndpoints{EndpointMode::CUSTOM, {customEndpoint}, {}}; +// info.UpdateHosts(regionEndpoints); +// APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); +// APSARA_TEST_EQUAL(1U, info.mCandidateHosts[0].size()); +// APSARA_TEST_EQUAL(customHost, info.mCandidateHosts[0][0].GetHostname()); +// APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[0][0].GetLatency()); + +// info.UpdateHostInfo(customHost, chrono::milliseconds(100)); + +// // to with remote endpoints +// regionEndpoints.mRemoteEndpoints = {privateEndpoint, publicEndpoint}; +// info.UpdateHosts(regionEndpoints); +// APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); +// APSARA_TEST_EQUAL(1U, info.mCandidateHosts[0].size()); +// APSARA_TEST_EQUAL(customHost, info.mCandidateHosts[0][0].GetHostname()); +// APSARA_TEST_EQUAL(chrono::milliseconds(100), info.mCandidateHosts[0][0].GetLatency()); +// } +// } + +// void CandidateHostsInfoUnittest::TestFirstHost() { +// const string host1 = mProject + ".endpoint_1"; +// const string host2 = mProject + ".endpoint_2"; + +// CandidateHostsInfo info(mProject, mRegion, mMode); +// APSARA_TEST_EQUAL("", info.GetFirstHost()); + +// info.mCandidateHosts.push_back({host1, host2}); +// APSARA_TEST_EQUAL(host1, info.GetFirstHost()); + +// info.mCandidateHosts[0].clear(); +// APSARA_TEST_EQUAL("", info.GetFirstHost()); +// } + +// UNIT_TEST_CASE(CandidateHostsInfoUnittest, TestBasicInfo) +// UNIT_TEST_CASE(CandidateHostsInfoUnittest, TestHostsInfo) +// UNIT_TEST_CASE(CandidateHostsInfoUnittest, TestUpdateHosts) +// UNIT_TEST_CASE(CandidateHostsInfoUnittest, TestFirstHost) // class ProbeNetworkMock { // public: @@ -566,7 +566,7 @@ class SLSClientManagerUnittest : public ::testing::Test { // void TestProbeNetwork(); // void TestRealIp(); void TestAccessKeyManagement(); - void TestGetCandidateHostsInfo(); + // void TestGetCandidateHostsInfo(); protected: // static void SetUpTestCase() { @@ -1336,38 +1336,38 @@ void SLSClientManagerUnittest::TestAccessKeyManagement() { APSARA_TEST_EQUAL(STRING_FLAG(default_access_key), accessKeySecret); } -void SLSClientManagerUnittest::TestGetCandidateHostsInfo() { - const string project = "project"; - const string endpoint = "endpoint"; - CandidateHostsInfo* infoPtr = nullptr; - { - auto info1 = mManager.GetCandidateHostsInfo(project, endpoint); - auto info2 = mManager.GetCandidateHostsInfo(project, endpoint); - infoPtr = info1.get(); - APSARA_TEST_EQUAL(info1.get(), info2.get()); - APSARA_TEST_EQUAL(project, info1->GetProject()); - APSARA_TEST_EQUAL("", info1->GetRegion()); - APSARA_TEST_EQUAL(EndpointMode::CUSTOM, info1->GetMode()); - APSARA_TEST_TRUE(info1->IsInitialized()); - APSARA_TEST_EQUAL(project + "." + endpoint, info1->GetCurrentHost()); - APSARA_TEST_EQUAL(1U, info1->mCandidateHosts.size()); - APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[0].size()); - APSARA_TEST_EQUAL(project + "." + endpoint, info1->mCandidateHosts[0][0].GetHostname()); - APSARA_TEST_EQUAL(chrono::milliseconds(10), info1->mCandidateHosts[0][0].GetLatency()); - APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap.size()); - auto& infos = mManager.mProjectCandidateHostsInfosMap[project]; - APSARA_TEST_EQUAL(1U, infos.size()); - auto& weakInfo = *infos.begin(); - APSARA_TEST_FALSE(weakInfo.expired()); - APSARA_TEST_EQUAL(info1.get(), weakInfo.lock().get()); - } - { - auto info = mManager.GetCandidateHostsInfo(project, endpoint); - APSARA_TEST_NOT_EQUAL(infoPtr, info.get()); - APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap.size()); - APSARA_TEST_EQUAL(2U, mManager.mProjectCandidateHostsInfosMap[project].size()); - } -} +// void SLSClientManagerUnittest::TestGetCandidateHostsInfo() { +// const string project = "project"; +// const string endpoint = "endpoint"; +// CandidateHostsInfo* infoPtr = nullptr; +// { +// auto info1 = mManager.GetCandidateHostsInfo(project, endpoint); +// auto info2 = mManager.GetCandidateHostsInfo(project, endpoint); +// infoPtr = info1.get(); +// APSARA_TEST_EQUAL(info1.get(), info2.get()); +// APSARA_TEST_EQUAL(project, info1->GetProject()); +// APSARA_TEST_EQUAL("", info1->GetRegion()); +// APSARA_TEST_EQUAL(EndpointMode::CUSTOM, info1->GetMode()); +// APSARA_TEST_TRUE(info1->IsInitialized()); +// APSARA_TEST_EQUAL(project + "." + endpoint, info1->GetCurrentHost()); +// APSARA_TEST_EQUAL(1U, info1->mCandidateHosts.size()); +// APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[0].size()); +// APSARA_TEST_EQUAL(project + "." + endpoint, info1->mCandidateHosts[0][0].GetHostname()); +// APSARA_TEST_EQUAL(chrono::milliseconds(10), info1->mCandidateHosts[0][0].GetLatency()); +// APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap.size()); +// auto& infos = mManager.mProjectCandidateHostsInfosMap[project]; +// APSARA_TEST_EQUAL(1U, infos.size()); +// auto& weakInfo = *infos.begin(); +// APSARA_TEST_FALSE(weakInfo.expired()); +// APSARA_TEST_EQUAL(info1.get(), weakInfo.lock().get()); +// } +// { +// auto info = mManager.GetCandidateHostsInfo(project, endpoint); +// APSARA_TEST_NOT_EQUAL(infoPtr, info.get()); +// APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap.size()); +// APSARA_TEST_EQUAL(2U, mManager.mProjectCandidateHostsInfosMap[project].size()); +// } +// } // UNIT_TEST_CASE(SLSClientManagerUnittest, TestLocalRegionEndpoints) // UNIT_TEST_CASE(SLSClientManagerUnittest, TestRemoteRegionEndpoints) @@ -1377,7 +1377,7 @@ void SLSClientManagerUnittest::TestGetCandidateHostsInfo() { // UNIT_TEST_CASE(SLSClientManagerUnittest, TestProbeNetwork) // UNIT_TEST_CASE(SLSClientManagerUnittest, TestRealIp) UNIT_TEST_CASE(SLSClientManagerUnittest, TestAccessKeyManagement) -UNIT_TEST_CASE(SLSClientManagerUnittest, TestGetCandidateHostsInfo) +// UNIT_TEST_CASE(SLSClientManagerUnittest, TestGetCandidateHostsInfo) } // namespace logtail From 5bcff46837dfa029209eb395d390d66da53d993a Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Thu, 12 Dec 2024 02:43:18 +0000 Subject: [PATCH 11/41] polish --- core/plugin/flusher/sls/SLSClientManager.cpp | 980 ------------------- core/plugin/flusher/sls/SLSClientManager.h | 201 +--- core/plugin/flusher/sls/SLSResponse.h | 1 + 3 files changed, 2 insertions(+), 1180 deletions(-) diff --git a/core/plugin/flusher/sls/SLSClientManager.cpp b/core/plugin/flusher/sls/SLSClientManager.cpp index 939cd87adb..0eca46f717 100644 --- a/core/plugin/flusher/sls/SLSClientManager.cpp +++ b/core/plugin/flusher/sls/SLSClientManager.cpp @@ -19,336 +19,28 @@ #endif #include "app_config/AppConfig.h" -#include "common/EndpointUtil.h" -#ifdef __ENTERPRISE__ -#include "common/EnterpriseEndpointUtil.h" -#endif #include "common/Flags.h" #include "common/HashUtil.h" -#include "common/LogtailCommonFlags.h" #include "common/StringTools.h" #include "common/http/Constant.h" #include "common/http/Curl.h" #include "common/version.h" #include "logger/Logger.h" #include "monitor/Monitor.h" -#include "pipeline/queue/SenderQueueItem.h" #ifdef __ENTERPRISE__ #include "plugin/flusher/sls/EnterpriseSLSClientManager.h" #endif -#include "plugin/flusher/sls/PackIdManager.h" #include "plugin/flusher/sls/SLSConstant.h" #include "plugin/flusher/sls/SLSUtil.h" DEFINE_FLAG_STRING(custom_user_agent, "custom user agent appended at the end of the exsiting ones", ""); -// DEFINE_FLAG_INT32(sls_hosts_probe_max_try_cnt, "", 3); -// DEFINE_FLAG_INT32(sls_hosts_probe_timeout, "", 3); -// DEFINE_FLAG_INT32(sls_all_hosts_probe_interval_sec, "seconds", 5 * 60); -// DEFINE_FLAG_INT32(sls_hosts_probe_interval_sec, "", 60); DEFINE_FLAG_STRING(default_access_key_id, "", ""); DEFINE_FLAG_STRING(default_access_key, "", ""); -// DEFINE_FLAG_INT32(send_switch_real_ip_interval, "seconds", 60); -// DEFINE_FLAG_BOOL(send_prefer_real_ip, "use real ip to send data", false); using namespace std; namespace logtail { -// TODO: move endpoint functions to EnterpriseEndpointUtil.h -// bool IsCustomEndpoint(const string& endpoint) { -// if (StartWith(endpoint, "log.") || StartWith(endpoint, "log-intranet.") || StartWith(endpoint, "log-internal.") -// || EndWith(endpoint, ".aliyuncs.com") || EndWith(endpoint, ".aliyun-inc.com")) { -// return false; -// } -// return true; -// } - -// enum class EndpointAddressType { INNER, INTRANET, PUBLIC }; -// const string kLogEndpointSuffix = ".log.aliyuncs.com"; - -// EndpointAddressType GetEndpointAddressType(const string& address) { -// // 一国一云 OXS区访问 & VPC访问 -// if (StartWith(address, "log-intranet.") || StartWith(address, "log-internal.")) { -// return EndpointAddressType::INTRANET; -// } -// // 一国一云 公网访问 -// if (StartWith(address, "log.")) { -// return EndpointAddressType::PUBLIC; -// } -// if (EndWith(address, "-intranet" + kLogEndpointSuffix) || EndWith(address, "-internal" + kLogEndpointSuffix)) { -// return EndpointAddressType::INTRANET; -// } -// if (!EndWith(address, "-share" + kLogEndpointSuffix) && EndWith(address, kLogEndpointSuffix)) { -// return EndpointAddressType::PUBLIC; -// } -// return EndpointAddressType::INNER; -// } - -// static const string kAccelerationDataEndpoint = "log-global.aliyuncs.com"; - -// const string& EndpointModeToString(EndpointMode mode) { -// switch (mode) { -// case EndpointMode::CUSTOM: -// static string customStr = "custom"; -// return customStr; -// case EndpointMode::ACCELERATE: -// static string accelerateStr = "accelerate"; -// return accelerateStr; -// case EndpointMode::DEFAULT: -// static string defaultStr = "default"; -// return defaultStr; -// default: -// static string unknownStr = "unknown"; -// return unknownStr; -// } -// } - -// chrono::milliseconds HostInfo::GetLatency() const { -// lock_guard lock(mLatencyMux); -// return mLatency; -// } - -// void HostInfo::SetLatency(const chrono::milliseconds& latency) { -// lock_guard lock(mLatencyMux); -// mLatency = latency; -// } - -// void HostInfo::SetForbidden() { -// lock_guard lock(mLatencyMux); -// mLatency = chrono::milliseconds::max(); -// } - -// bool HostInfo::IsForbidden() const { -// lock_guard lock(mLatencyMux); -// return mLatency == chrono::milliseconds::max(); -// } - -// void CandidateHostsInfo::UpdateHosts(const CandidateEndpoints& regionEndpoints) { -// lock_guard lock(mCandidateHostsMux); -// switch (mMode) { -// #ifdef __ENTERPRISE__ -// case EndpointMode::DEFAULT: { -// vector endpoints(regionEndpoints.mLocalEndpoints); -// for (const auto& endpoint : regionEndpoints.mRemoteEndpoints) { -// bool found = false; -// for (const auto& existedEndpoint : regionEndpoints.mLocalEndpoints) { -// if (existedEndpoint == endpoint) { -// found = true; -// break; -// } -// } -// if (!found) { -// endpoints.emplace_back(endpoint); -// } -// } -// vector> infos; -// for (const auto& endpoint : endpoints) { -// string host = mProject.empty() ? endpoint : mProject + "." + endpoint; -// if (mCandidateHosts.empty()) { -// infos.emplace_back().emplace_back(host); -// } else { -// bool found = false; -// for (const auto& item : mCandidateHosts) { -// if (!item.empty() && item[0].GetHostname() == host) { -// found = true; -// infos.emplace_back(item); -// break; -// } -// } -// if (!found) { -// infos.emplace_back().emplace_back(host); -// } -// } -// } -// mCandidateHosts.swap(infos); -// break; -// } -// case EndpointMode::ACCELERATE: { -// vector endpoints{kAccelerationDataEndpoint}; -// for (const auto& item : regionEndpoints.mRemoteEndpoints) { -// if (GetEndpointAddressType(item) == EndpointAddressType::PUBLIC) { -// endpoints.emplace_back(item); -// } -// } -// vector infos; -// for (const auto& endpoint : endpoints) { -// string host = mProject.empty() ? endpoint : mProject + "." + endpoint; -// if (mCandidateHosts.empty()) { -// infos.emplace_back(host); -// } else { -// bool found = false; -// for (const auto& item : mCandidateHosts[0]) { -// if (item.GetHostname() == host) { -// found = true; -// infos.emplace_back(item); -// break; -// } -// } -// if (!found) { -// infos.emplace_back(host); -// } -// } -// } -// if (mCandidateHosts.empty()) { -// mCandidateHosts.emplace_back(infos); -// } else { -// mCandidateHosts[0].swap(infos); -// } -// break; -// } -// #endif -// case EndpointMode::CUSTOM: { -// vector infos; -// for (const auto& endpoint : regionEndpoints.mLocalEndpoints) { -// string host = mProject.empty() ? endpoint : mProject + "." + endpoint; -// if (mCandidateHosts.empty()) { -// infos.emplace_back(host); -// } else { -// bool found = false; -// for (const auto& item : mCandidateHosts[0]) { -// if (item.GetHostname() == host) { -// found = true; -// infos.emplace_back(item); -// break; -// } -// } -// if (!found) { -// infos.emplace_back(host); -// } -// } -// } -// if (mCandidateHosts.empty()) { -// mCandidateHosts.emplace_back(infos); -// } else { -// mCandidateHosts[0].swap(infos); -// } -// break; -// } -// default: -// break; -// } -// } - -// bool CandidateHostsInfo::UpdateHostInfo(const string& hostname, const chrono::milliseconds& latency) { -// lock_guard lock(mCandidateHostsMux); -// for (auto& item : mCandidateHosts) { -// for (auto& entry : item) { -// if (entry.GetHostname() == hostname) { -// entry.SetLatency(latency); -// return true; -// } -// } -// } -// return false; -// } - -// void CandidateHostsInfo::GetProbeHosts(vector& hosts) const { -// if (!HasValidHost()) { -// return GetAllHosts(hosts); -// } -// { -// lock_guard lock(mCandidateHostsMux); -// for (const auto& item : mCandidateHosts) { -// for (const auto& entry : item) { -// if (!entry.IsForbidden()) { -// hosts.emplace_back(entry.GetHostname()); -// } -// } -// } -// } -// } - -// void CandidateHostsInfo::GetAllHosts(vector& hosts) const { -// lock_guard lock(mCandidateHostsMux); -// for (const auto& item : mCandidateHosts) { -// for (const auto& entry : item) { -// hosts.emplace_back(entry.GetHostname()); -// } -// } -// } - -// void CandidateHostsInfo::SelectBestHost() { -// lock_guard lock(mCandidateHostsMux); -// for (size_t i = 0; i < mCandidateHosts.size(); ++i) { -// const auto& hosts = mCandidateHosts[i]; -// chrono::milliseconds minLatency = chrono::milliseconds::max(); -// size_t minIdx = numeric_limits::max(); -// for (size_t j = 0; j < hosts.size(); ++j) { -// if (!hosts[j].IsForbidden() && hosts[j].GetLatency() < minLatency) { -// minLatency = hosts[j].GetLatency(); -// minIdx = j; -// } -// } -// if (minIdx != numeric_limits::max()) { -// const auto& hostname = hosts[minIdx].GetHostname(); -// if (GetCurrentHost() != hostname) { -// SetCurrentHost(hostname); -// LOG_INFO(sLogger, -// ("switch to the best host", hostname)("latency", minLatency.count())("project", mProject)( -// "region", mRegion)("endpoint mode", EndpointModeToString(mMode))); -// } -// return; -// } -// } -// SetCurrentHost(""); -// LOG_INFO(sLogger, -// ("no valid host", "stop sending data and retry later")("project", mProject)("region", mRegion)( -// "endpoint mode", EndpointModeToString(mMode))); -// } - -// string CandidateHostsInfo::GetCurrentHost() const { -// lock_guard lock(mCurrentHostMux); -// return mCurrentHost; -// } - -// string CandidateHostsInfo::GetFirstHost() const { -// lock_guard lock(mCandidateHostsMux); -// if (mCandidateHosts.empty() || mCandidateHosts[0].empty()) { -// return ""; -// } -// return mCandidateHosts[0][0].GetHostname(); -// } - -// bool CandidateHostsInfo::HasValidHost() const { -// lock_guard lock(mCurrentHostMux); -// return !mCurrentHost.empty(); -// } - -// void CandidateHostsInfo::SetCurrentHost(const string& host) { -// lock_guard lock(mCurrentHostMux); -// mCurrentHost = host; -// } - -// SLSClientManager::ProbeNetworkHttpRequest::ProbeNetworkHttpRequest( -// const string& region, const string& project, EndpointMode mode, const string& host, bool httpsFlag) -// : AsynHttpRequest(HTTP_GET, -// httpsFlag, -// host, -// httpsFlag ? 443 : 80, -// HEALTH, -// "", -// {}, -// "", -// HttpResponse(), -// INT32_FLAG(sls_hosts_probe_timeout), -// INT32_FLAG(sls_hosts_probe_max_try_cnt)), -// mRegion(region), -// mProject(project), -// mMode(mode), -// mHost(host) { -// } - -// void SLSClientManager::ProbeNetworkHttpRequest::OnSendDone(HttpResponse& response) { -// if (mProject.empty()) { -// SLSClientManager::GetInstance()->UpdateHostInfo(mRegion, mHost, response.GetResponseTime()); -// } else { -// SLSClientManager::GetInstance()->UpdateHostInfo(mProject, mMode, mHost, response.GetResponseTime()); -// } -// } - -// SLSClientManager::SLSClientManager() : mGetEndpointRealIp(GetEndpointRealIp) { -// } - SLSClientManager* SLSClientManager::GetInstance() { #ifdef __ENTERPRISE__ return EnterpriseSLSClientManager::GetInstance(); @@ -360,68 +52,8 @@ SLSClientManager* SLSClientManager::GetInstance() { void SLSClientManager::Init() { GenerateUserAgent(); - - // mUnInitializedHostProbeClient = curl_multi_init(); - // if (mUnInitializedHostProbeClient == nullptr) { - // LOG_ERROR(sLogger, ("failed to init uninitialized host probe", "failed to init curl client")); - // } else { - // mUnInitializedHostProbeThreadRes = async(launch::async, &SLSClientManager::UnInitializedHostProbeThread, - // this); - // } - - // mHostProbeClient = curl_multi_init(); - // if (mHostProbeClient == nullptr) { - // LOG_ERROR(sLogger, ("failed to init host probe", "failed to init curl client")); - // } else { - // mHostProbeThreadRes = async(launch::async, &SLSClientManager::HostProbeThread, this); - // } - - // if (BOOL_FLAG(send_prefer_real_ip)) { - // mUpdateRealIpThreadRes = async(launch::async, &SLSClientManager::UpdateRealIpThread, this); - // } } -// void SLSClientManager::Stop() { -// if (mUnInitializedHostProbeClient) { -// lock_guard lock(mUnInitializedHostProbeThreadRunningMux); -// mIsUnInitializedHostProbeThreadRunning = false; -// } -// if (mHostProbeClient) { -// lock_guard lock(mHostProbeThreadRunningMux); -// mIsHostProbeThreadRunning = false; -// } -// if (BOOL_FLAG(send_prefer_real_ip)) { -// lock_guard lock(mUpdateRealIpThreadRunningMux); -// mIsUpdateRealIpThreadRunning = false; -// } -// mStopCV.notify_all(); - -// if (mUnInitializedHostProbeClient) { -// future_status s = mUnInitializedHostProbeThreadRes.wait_for(chrono::seconds(1)); -// if (s == future_status::ready) { -// LOG_INFO(sLogger, ("sls uninitialized host probe", "stopped successfully")); -// } else { -// LOG_WARNING(sLogger, ("sls uninitialized host probe", "forced to stopped")); -// } -// } -// if (mHostProbeClient) { -// future_status s = mHostProbeThreadRes.wait_for(chrono::seconds(1)); -// if (s == future_status::ready) { -// LOG_INFO(sLogger, ("sls host probe", "stopped successfully")); -// } else { -// LOG_WARNING(sLogger, ("sls host probe", "forced to stopped")); -// } -// } -// if (BOOL_FLAG(send_prefer_real_ip) && mUpdateRealIpThreadRes.valid()) { -// future_status s = mUpdateRealIpThreadRes.wait_for(chrono::seconds(1)); -// if (s == future_status::ready) { -// LOG_INFO(sLogger, ("sls real ip update", "stopped successfully")); -// } else { -// LOG_WARNING(sLogger, ("sls real ip update", "forced to stopped")); -// } -// } -// } - bool SLSClientManager::GetAccessKey(const string& aliuid, AuthType& type, string& accessKeyId, @@ -432,562 +64,6 @@ bool SLSClientManager::GetAccessKey(const string& aliuid, return true; } -// void SLSClientManager::UpdateLocalRegionEndpointsAndHttpsInfo(const string& region, -// const vector& rawEndpoints) { -// vector endpoints; -// for (const auto& item : rawEndpoints) { -// auto tmp = ExtracteEndpoint(item); -// if (!tmp.empty()) { -// endpoints.emplace_back(tmp); -// } -// } - -// lock_guard lock(mRegionCandidateEndpointsMapMux); -// auto& candidate = mRegionCandidateEndpointsMap[region]; -// candidate.mMode = EndpointMode::DEFAULT; -// candidate.mLocalEndpoints.clear(); -// for (const auto& item : endpoints) { -// // if both acclerate and custom endpoints are given, we ignore custom endpoints -// if (item == kAccelerationDataEndpoint) { -// candidate.mMode = EndpointMode::ACCELERATE; -// break; -// } -// if (IsCustomEndpoint(item)) { -// candidate.mMode = EndpointMode::CUSTOM; -// candidate.mLocalEndpoints.emplace_back(item); -// } -// } -// if (candidate.mMode == EndpointMode::ACCELERATE) { -// candidate.mLocalEndpoints.clear(); -// } else if (candidate.mMode == EndpointMode::DEFAULT) { -// candidate.mLocalEndpoints = endpoints; -// } - -// // as long as one endpoint is https, we treat the region as https -// bool isHttps = false; -// for (const auto& item : rawEndpoints) { -// if (IsHttpsEndpoint(item)) { -// isHttps = true; -// break; -// } -// } -// { -// lock_guard lock(mHttpsRegionsMux); -// if (isHttps) { -// mHttpsRegions.insert(region); -// } else { -// mHttpsRegions.erase(region); -// } -// } -// } - -// void SLSClientManager::UpdateRemoteRegionEndpoints(const string& region, -// const vector& rawEndpoints, -// RemoteEndpointUpdateAction action) { -// vector endpoints; -// for (const auto& item : rawEndpoints) { -// auto tmp = ExtractEndpoint(item); -// if (!tmp.empty()) { -// endpoints.emplace_back(tmp); -// } -// } -// lock_guard lock(mRegionCandidateEndpointsMapMux); - -// if (action == RemoteEndpointUpdateAction::CREATE -// && mRegionCandidateEndpointsMap.find(region) != mRegionCandidateEndpointsMap.end()) { -// return; -// } -// auto& candidate = mRegionCandidateEndpointsMap[region]; -// if (action == RemoteEndpointUpdateAction::APPEND) { -// for (const auto& item : endpoints) { -// bool found = false; -// for (const auto& existed : candidate.mRemoteEndpoints) { -// if (existed == item) { -// found = true; -// break; -// } -// } -// if (!found) { -// candidate.mRemoteEndpoints.emplace_back(item); -// } -// } -// } else { -// candidate.mRemoteEndpoints = endpoints; -// } -// { -// lock_guard lock(mCandidateHostsInfosMapMux); -// for (auto& item : mRegionCandidateHostsInfosMap[region]) { -// if (item.expired()) { -// continue; -// } -// auto info = item.lock(); -// info->UpdateHosts(candidate); -// } -// } -// if (BOOL_FLAG(send_prefer_real_ip)) { -// lock_guard lock(mRegionRealIpCandidateHostsInfosMapMux); -// auto it = mRegionRealIpCandidateHostsInfosMap.find(region); -// if (it == mRegionRealIpCandidateHostsInfosMap.end()) { -// it = mRegionRealIpCandidateHostsInfosMap.try_emplace(region, "", region, EndpointMode::DEFAULT).first; -// } -// it->second.UpdateHosts(candidate); -// } -// } - -// shared_ptr -// SLSClientManager::GetCandidateHostsInfo(const string& region, const string& project, EndpointMode mode) { -// if (mode == EndpointMode::DEFAULT) { -// // when flusher does not specify the mode, we use the mode of the region, which is set before flusher init -// and -// // will not change. -// lock_guard lock(mRegionCandidateEndpointsMapMux); -// auto it = mRegionCandidateEndpointsMap.find(region); -// if (it != mRegionCandidateEndpointsMap.end()) { -// mode = it->second.mMode; -// } -// } -// { -// lock_guard lock(mCandidateHostsInfosMapMux); -// auto& hostsInfo = mProjectCandidateHostsInfosMap[project]; -// for (auto& item : hostsInfo) { -// if (item.expired()) { -// continue; -// } -// auto info = item.lock(); -// if (info->GetMode() == mode) { -// return info; -// } -// } -// } - -// auto info = make_shared(project, region, mode); -// CandidateEndpoints endpoints; -// { -// lock_guard lock(mRegionCandidateEndpointsMapMux); -// auto it = mRegionCandidateEndpointsMap.find(region); -// if (it != mRegionCandidateEndpointsMap.end()) { -// endpoints = it->second; -// } -// } -// info->UpdateHosts(endpoints); -// { -// lock_guard lock(mCandidateHostsInfosMapMux); -// mProjectCandidateHostsInfosMap[project].emplace_back(info); -// mRegionCandidateHostsInfosMap[region].emplace_back(info); -// } -// { -// lock_guard lock(mUnInitializedCandidateHostsInfosMux); -// mUnInitializedCandidateHostsInfos.emplace_back(info); -// } -// return info; -// } - -// bool SLSClientManager::UpdateHostInfo(const string& project, -// EndpointMode mode, -// const string& host, -// const chrono::milliseconds& latency) { -// lock_guard lock(mCandidateHostsInfosMapMux); -// auto it = mProjectCandidateHostsInfosMap.find(project); -// if (it == mProjectCandidateHostsInfosMap.end()) { -// return false; -// } -// for (auto& entry : it->second) { -// if (entry.expired()) { -// continue; -// } -// auto info = entry.lock(); -// if (info->GetMode() == mode) { -// info->UpdateHostInfo(host, latency); -// return true; -// } -// } -// return false; -// } - -// bool SLSClientManager::UpdateHostInfo(const string& region, const string& host, const chrono::milliseconds& latency) -// { -// lock_guard lock(mRegionRealIpCandidateHostsInfosMapMux); -// auto it = mRegionRealIpCandidateHostsInfosMap.find(region); -// if (it == mRegionRealIpCandidateHostsInfosMap.end()) { -// return false; -// } -// return it->second.UpdateHostInfo(host, latency); -// } - -// shared_ptr SLSClientManager::GetCandidateHostsInfo(const string& project, const string& endpoint) { -// if (endpoint.empty()) { -// // this should only occur on first update, where we try find any available info -// lock_guard lock(mCandidateHostsInfosMapMux); -// auto& hostsInfo = mProjectCandidateHostsInfosMap[project]; -// for (auto& item : hostsInfo) { -// if (item.expired()) { -// continue; -// } -// return item.lock(); -// } -// return nullptr; -// } - -// string standardEndpoint = ExtractEndpoint(endpoint); -// { -// lock_guard lock(mCandidateHostsInfosMapMux); -// auto& hostsInfo = mProjectCandidateHostsInfosMap[project]; -// for (auto& item : hostsInfo) { -// if (item.expired()) { -// continue; -// } -// auto info = item.lock(); -// if (info->GetFirstHost() == project + "." + standardEndpoint) { -// return info; -// } -// } -// } -// auto info = make_shared(project, "", EndpointMode::CUSTOM); -// info->UpdateHosts({EndpointMode::CUSTOM, {standardEndpoint}}); -// // manually set the endpoint to be available -// info->UpdateHostInfo(info->GetFirstHost(), chrono::milliseconds(10)); -// info->SelectBestHost(); -// info->SetInitialized(); -// { -// lock_guard lock(mCandidateHostsInfosMapMux); -// mProjectCandidateHostsInfosMap[project].emplace_back(info); -// } -// return info; -// } - -bool SLSClientManager::UsingHttps(const string& region) const { - return true; - // lock_guard lock(mHttpsRegionsMux); - // return mHttpsRegions.find(region) != mHttpsRegions.end(); -} - -// void SLSClientManager::UnInitializedHostProbeThread() { -// LOG_INFO(sLogger, ("sls uninitialized host probe", "started")); -// unique_lock lock(mUnInitializedHostProbeThreadRunningMux); -// while (mIsUnInitializedHostProbeThreadRunning) { -// DoProbeUnInitializedHost(); -// if (mStopCV.wait_for( -// lock, chrono::milliseconds(10), [this]() { return !mIsUnInitializedHostProbeThreadRunning; })) { -// return; -// } -// } - -// auto mc = curl_multi_cleanup(mUnInitializedHostProbeClient); -// if (mc != CURLM_OK) { -// LOG_ERROR(sLogger, ("failed to cleanup curl multi handle", "exit anyway")("errMsg", -// curl_multi_strerror(mc))); -// } -// } - -// void SLSClientManager::DoProbeUnInitializedHost() { -// vector> infos; -// { -// lock_guard lock(mUnInitializedCandidateHostsInfosMux); -// infos.swap(mUnInitializedCandidateHostsInfos); -// } -// if (infos.empty()) { -// return; -// } -// #ifndef APSARA_UNIT_TEST_MAIN -// for (auto& item : infos) { -// if (item.expired()) { -// continue; -// } -// auto info = item.lock(); -// bool httpsFlag = UsingHttps(info->GetRegion()); -// auto req = make_unique( -// info->GetRegion(), info->GetProject(), info->GetMode(), info->GetFirstHost(), httpsFlag); -// AddRequestToMultiCurlHandler(mUnInitializedHostProbeClient, std::move(req)); -// } -// SendAsynRequests(mUnInitializedHostProbeClient); -// #else -// for (auto& item : infos) { -// if (item.expired()) { -// continue; -// } -// auto info = item.lock(); -// bool httpsFlag = UsingHttps(info->GetRegion()); -// auto req = make_unique( -// info->GetRegion(), info->GetProject(), info->GetMode(), info->GetFirstHost(), httpsFlag); -// HttpResponse response = mDoProbeNetwork(req); -// req->OnSendDone(response); -// } -// #endif - -// for (auto& item : infos) { -// if (item.expired()) { -// continue; -// } -// item.lock()->SelectBestHost(); -// } -// { -// lock_guard lock(mPartiallyInitializedCandidateHostsInfosMux); -// mPartiallyInitializedCandidateHostsInfos.insert( -// mPartiallyInitializedCandidateHostsInfos.end(), infos.begin(), infos.end()); -// } -// } - -// void SLSClientManager::HostProbeThread() { -// LOG_INFO(sLogger, ("sls host probe", "started")); -// unique_lock lock(mHostProbeThreadRunningMux); -// while (mIsHostProbeThreadRunning) { -// DoProbeHost(); -// if (mStopCV.wait_for(lock, chrono::seconds(1), [this]() { return !mIsHostProbeThreadRunning; })) { -// return; -// } -// } - -// auto mc = curl_multi_cleanup(mHostProbeClient); -// if (mc != CURLM_OK) { -// LOG_ERROR(sLogger, ("failed to cleanup curl multi handle", "exit anyway")("errMsg", -// curl_multi_strerror(mc))); -// } -// } - -// void SLSClientManager::DoProbeHost() { -// static time_t lastProbeAllEndpointsTime = time(nullptr); -// static time_t lastProbeAvailableEndpointsTime = time(nullptr); -// bool shouldTestAllEndpoints = false; -// bool shouldTestAvailableEndpoints = false; -// time_t beginTime = time(nullptr); -// if (beginTime - lastProbeAllEndpointsTime >= INT32_FLAG(sls_all_hosts_probe_interval_sec)) { -// shouldTestAllEndpoints = true; -// lastProbeAllEndpointsTime = beginTime; -// } else if (beginTime - lastProbeAvailableEndpointsTime >= INT32_FLAG(sls_hosts_probe_interval_sec)) { -// shouldTestAvailableEndpoints = true; -// lastProbeAvailableEndpointsTime = beginTime; -// } else if (!HasPartiallyInitializedCandidateHostsInfos()) { -// ClearExpiredCandidateHostsInfos(); -// PackIdManager::GetInstance()->CleanTimeoutEntry(); -// return; -// } - -// // collect hosts to be probed -// map, pair>> projectHostMap; -// map> regionHostsMap; // only for realip -// if (shouldTestAllEndpoints || shouldTestAvailableEndpoints) { -// { -// lock_guard lk(mCandidateHostsInfosMapMux); -// for (const auto& item : mProjectCandidateHostsInfosMap) { -// for (const auto& entry : item.second) { -// if (entry.expired()) { -// continue; -// } -// auto info = entry.lock(); -// auto& hosts = projectHostMap[make_pair(item.first, info->GetMode())]; -// hosts.first = info->GetRegion(); -// if (shouldTestAllEndpoints) { -// info->GetAllHosts(hosts.second); -// } else { -// info->GetProbeHosts(hosts.second); -// } -// } -// } -// } -// if (BOOL_FLAG(send_prefer_real_ip)) { -// lock_guard lk(mRegionRealIpCandidateHostsInfosMapMux); -// for (const auto& item : mRegionRealIpCandidateHostsInfosMap) { -// auto& hosts = regionHostsMap[item.second.GetRegion()]; -// if (shouldTestAllEndpoints) { -// item.second.GetAllHosts(hosts); -// } else { -// item.second.GetProbeHosts(hosts); -// } -// } -// } -// } else { -// lock_guard lk(mPartiallyInitializedCandidateHostsInfosMux); -// for (const auto& item : mPartiallyInitializedCandidateHostsInfos) { -// if (item.expired()) { -// continue; -// } -// auto info = item.lock(); -// auto& hosts = projectHostMap[make_pair(info->GetProject(), info->GetMode())]; -// hosts.first = info->GetRegion(); -// info->GetAllHosts(hosts.second); -// } -// mPartiallyInitializedCandidateHostsInfos.clear(); -// } - -// // do probe and update latency -// for (const auto& item : projectHostMap) { -// bool httpsFlag = UsingHttps(item.second.first); -// #ifndef APSARA_UNIT_TEST_MAIN -// for (const auto& host : item.second.second) { -// auto req = make_unique( -// item.second.first, item.first.first, item.first.second, host, httpsFlag); -// AddRequestToMultiCurlHandler(mHostProbeClient, std::move(req)); -// } -// SendAsynRequests(mHostProbeClient); -// #else -// for (const auto& host : item.second.second) { -// auto req = make_unique( -// item.second.first, item.first.first, item.first.second, host, httpsFlag); -// HttpResponse response = mDoProbeNetwork(req); -// req->OnSendDone(response); -// } -// #endif -// } -// if (BOOL_FLAG(send_prefer_real_ip)) { -// for (const auto& item : regionHostsMap) { -// bool httpsFlag = UsingHttps(item.first); -// #ifndef APSARA_UNIT_TEST_MAIN -// for (const auto& host : item.second) { -// auto req = make_unique(item.first, "", EndpointMode::DEFAULT, host, -// httpsFlag); AddRequestToMultiCurlHandler(mHostProbeClient, std::move(req)); -// } -// SendAsynRequests(mHostProbeClient); -// #else -// for (const auto& host : item.second) { -// auto req = make_unique(item.first, "", EndpointMode::DEFAULT, host, -// httpsFlag); HttpResponse response = mDoProbeNetwork(req); req->OnSendDone(response); -// } -// #endif -// } -// } - -// // update selected host -// { -// lock_guard lk(mCandidateHostsInfosMapMux); -// for (const auto& item : mProjectCandidateHostsInfosMap) { -// for (auto& entry : item.second) { -// if (auto p = entry.lock()) { -// p->SelectBestHost(); -// } -// } -// } -// } -// if (BOOL_FLAG(send_prefer_real_ip) && (shouldTestAllEndpoints || shouldTestAvailableEndpoints)) { -// lock_guard lk(mRegionRealIpCandidateHostsInfosMapMux); -// for (auto& item : mRegionRealIpCandidateHostsInfosMap) { -// item.second.SelectBestHost(); -// } -// } -// } - -// bool SLSClientManager::HasPartiallyInitializedCandidateHostsInfos() const { -// lock_guard lk(mPartiallyInitializedCandidateHostsInfosMux); -// return !mPartiallyInitializedCandidateHostsInfos.empty(); -// } - -// void SLSClientManager::ClearExpiredCandidateHostsInfos() { -// lock_guard lk(mCandidateHostsInfosMapMux); -// for (auto iter = mProjectCandidateHostsInfosMap.begin(); iter != mProjectCandidateHostsInfosMap.end();) { -// for (auto it = iter->second.begin(); it != iter->second.end();) { -// if (it->expired()) { -// it = iter->second.erase(it); -// } else { -// ++it; -// } -// } -// if (iter->second.empty()) { -// iter = mProjectCandidateHostsInfosMap.erase(iter); -// } else { -// ++iter; -// } -// } -// for (auto iter = mRegionCandidateHostsInfosMap.begin(); iter != mRegionCandidateHostsInfosMap.end();) { -// for (auto it = iter->second.begin(); it != iter->second.end();) { -// if (it->expired()) { -// it = iter->second.erase(it); -// } else { -// ++it; -// } -// } -// if (iter->second.empty()) { -// iter = mRegionCandidateHostsInfosMap.erase(iter); -// } else { -// ++iter; -// } -// } -// } - -// void SLSClientManager::UpdateRealIpThread() { -// LOG_INFO(sLogger, ("sls real ip update", "started")); -// unique_lock lock(mUpdateRealIpThreadRunningMux); -// while (mIsUpdateRealIpThreadRunning) { -// DoUpdateRealIp(); -// if (mStopCV.wait_for(lock, chrono::seconds(1), [this]() { return !mIsUpdateRealIpThreadRunning; })) { -// break; -// } -// } -// } - -// void SLSClientManager::DoUpdateRealIp() { -// vector> hosts; -// static time_t lastUpdateRealIpTime = 0; -// if (time(nullptr) - lastUpdateRealIpTime >= INT32_FLAG(send_switch_real_ip_interval)) { -// { -// lock_guard lock(mOutdatedRealIpRegionsMux); -// mOutdatedRealIpRegions.clear(); -// } -// { -// lock_guard lock(mRegionRealIpCandidateHostsInfosMapMux); -// for (const auto& item : mRegionRealIpCandidateHostsInfosMap) { -// hosts.emplace_back(item.first, item.second.GetCurrentHost()); -// } -// } -// lastUpdateRealIpTime = time(nullptr); -// } else if (HasOutdatedRealIpRegions()) { -// vector regions; -// { -// lock_guard lock(mOutdatedRealIpRegionsMux); -// regions.swap(mOutdatedRealIpRegions); -// } -// { -// lock_guard lock(mRegionRealIpCandidateHostsInfosMapMux); -// for (const auto& item : regions) { -// auto it = mRegionRealIpCandidateHostsInfosMap.find(item); -// if (it != mRegionRealIpCandidateHostsInfosMap.end()) { -// hosts.emplace_back(item, it->second.GetCurrentHost()); -// } -// } -// } -// } -// for (const auto& item : hosts) { -// if (item.second.empty()) { -// continue; -// } -// string ip; -// if (!mGetEndpointRealIp(item.second, ip)) { -// LOG_WARNING(sLogger, -// ("failed to get real ip", "retry later")("region", item.first)("endpoint", item.second)); -// continue; -// } -// SetRealIp(item.first, ip); -// } -// } - -// bool SLSClientManager::HasOutdatedRealIpRegions() const { -// lock_guard lock(mOutdatedRealIpRegionsMux); -// return !mOutdatedRealIpRegions.empty(); -// } - -// void SLSClientManager::UpdateOutdatedRealIpRegions(const string& region) { -// lock_guard lock(mOutdatedRealIpRegionsMux); -// mOutdatedRealIpRegions.emplace_back(region); -// } - -// string SLSClientManager::GetRealIp(const string& region) const { -// lock_guard lock(mRegionRealIpMux); -// auto it = mRegionRealIpMap.find(region); -// if (it == mRegionRealIpMap.end()) { -// return ""; -// } -// return it->second; -// } - -// void SLSClientManager::SetRealIp(const string& region, const string& ip) { -// lock_guard lock(mRegionRealIpMux); -// auto it = mRegionRealIpMap.find(region); -// if (it == mRegionRealIpMap.end()) { -// mRegionRealIpMap.emplace(region, ip); -// } else if (it->second != ip) { -// LOG_INFO(sLogger, ("set real ip for region", region)("from", it->second)("to", ip)); -// it->second = ip; -// } -// } - void SLSClientManager::GenerateUserAgent() { string os; #if defined(__linux__) @@ -1057,12 +133,6 @@ bool SLSClientManager::PingEndpoint(const string& host, const string& path) { response); } -// #ifdef APSARA_UNIT_TEST_MAIN -// void SLSClientManager::Clear() { -// mProjectCandidateHostsInfosMap.clear(); -// } -// #endif - void PreparePostLogStoreLogsRequest(const string& accessKeyId, const string& accessKeySecret, SLSClientManager::AuthType type, @@ -1259,54 +329,4 @@ SLSResponse PutWebTracking(const string& host, return ParseHttpResponse(response); } -// bool GetEndpointRealIp(const string& endpoint, string& ip) { -// static string body; -// if (body.empty()) { -// sls_logs::LogGroup logGroup; -// logGroup.set_source(LoongCollectorMonitor::mIpAddr); -// body = logGroup.SerializeAsString(); -// } - -// static string project = "logtail-real-ip-project"; -// static string logstore = "logtail-real-ip-logstore"; - -// SLSClientManager::AuthType type; -// string accessKeyId, accessKeySecret; -// SLSClientManager::GetInstance()->GetAccessKey("", type, accessKeyId, accessKeySecret); - -// string path, query; -// map header; -// string host = project + "." + endpoint; -// PreparePostLogStoreLogsRequest(accessKeyId, -// accessKeySecret, -// type, -// host, -// false, -// project, -// logstore, -// "", -// RawDataType::EVENT_GROUP, -// body, -// body.size(), -// "", -// nullopt, -// path, -// query, -// header); -// HttpResponse response; -// if (!SendHttpRequest(make_unique(HTTP_POST, false, host, 80, path, query, header, body), response)) -// { -// return false; -// } -// if (response.GetStatusCode() != 200) { -// auto it = response.GetHeader().find(X_LOG_HOSTIP); -// if (it != response.GetHeader().end()) { -// ip = it->second; -// } -// return true; -// } -// // should not happen -// return true; -// } - } // namespace logtail diff --git a/core/plugin/flusher/sls/SLSClientManager.h b/core/plugin/flusher/sls/SLSClientManager.h index aa92dc3d65..2e69294fe3 100644 --- a/core/plugin/flusher/sls/SLSClientManager.h +++ b/core/plugin/flusher/sls/SLSClientManager.h @@ -16,109 +16,19 @@ #pragma once -// #include - -// #include -#include -// #include #include -// #include -// #include -#include #include -#include -#include #include #include -// #include -// #include -#include -// #include "common/http/HttpRequest.h" -// #include "common/http/HttpResponse.h" #include "pipeline/queue/SenderQueueItem.h" #include "plugin/flusher/sls/SLSResponse.h" namespace logtail { -// enum class EndpointMode { DEFAULT, ACCELERATE, CUSTOM }; -// const std::string& EndpointModeToString(EndpointMode mode); - -// struct CandidateEndpoints { -// // currently only remote endpoints can be updated -// EndpointMode mMode = EndpointMode::DEFAULT; -// // when mMode = ACCELERATE, mLocalEndpoints should be empty -// std::vector mLocalEndpoints; -// std::vector mRemoteEndpoints; -// }; - -// class HostInfo { -// public: -// HostInfo(const std::string& hostname) : mHostname(hostname) {} -// HostInfo(const HostInfo& rhs) : mHostname(rhs.mHostname) { mLatency = rhs.GetLatency(); } - -// const std::string& GetHostname() const { return mHostname; } -// std::chrono::milliseconds GetLatency() const; -// void SetLatency(const std::chrono::milliseconds& latency); -// void SetForbidden(); -// bool IsForbidden() const; - -// private: -// // normally in the form of ., except for the real ip scene, which equals to endpoint -// const std::string mHostname; - -// mutable std::mutex mLatencyMux; -// std::chrono::milliseconds mLatency = std::chrono::milliseconds::max(); -// }; - -// class CandidateHostsInfo { -// public: -// CandidateHostsInfo(const std::string& project, const std::string& region, EndpointMode mode) -// : mProject(project), mRegion(region), mMode(mode) {} - -// void UpdateHosts(const CandidateEndpoints& regionEndpoints); -// bool UpdateHostInfo(const std::string& hostname, const std::chrono::milliseconds& latency); -// void GetProbeHosts(std::vector& hosts) const; -// void GetAllHosts(std::vector& hosts) const; -// void SelectBestHost(); -// std::string GetCurrentHost() const; -// std::string GetFirstHost() const; -// const std::string& GetProject() const { return mProject; } -// const std::string& GetRegion() const { return mRegion; } -// EndpointMode GetMode() const { return mMode; } -// bool IsInitialized() const { return mInitialized.load(); } -// void SetInitialized() { mInitialized = true; } - -// private: -// bool HasValidHost() const; -// void SetCurrentHost(const std::string& host); - -// // for real ip scene, mProject is empty -// const std::string mProject; -// const std::string mRegion; -// const EndpointMode mMode; -// std::atomic_bool mInitialized = false; - -// mutable std::mutex mCurrentHostMux; -// std::string mCurrentHost; - -// // mMode = DEFAULT: each mCandidateHosts element has exactly one element -// // mMode = ACCELERATE: mCandidateHosts.size() == 1 && mCandidateHosts[0].size() == 2 -// // mMode = CUSTOM: mCandidateHosts.size() == 1 && mCandidateHosts[0].size() == 1 -// mutable std::mutex mCandidateHostsMux; -// std::vector> mCandidateHosts; - -// #ifdef APSARA_UNIT_TEST_MAIN -// friend class CandidateHostsInfoUnittest; -// friend class SLSClientManagerUnittest; -// friend class EnterpriseSLSClientManagerUnittest; -// #endif -// }; - class SLSClientManager { public: enum class AuthType { ANONYMOUS, AK }; - // enum class RemoteEndpointUpdateAction { CREATE, OVERWRITE, APPEND }; virtual ~SLSClientManager() = default; SLSClientManager(const SLSClientManager&) = delete; @@ -135,33 +45,7 @@ class SLSClientManager { GetAccessKey(const std::string& aliuid, AuthType& type, std::string& accessKeyId, std::string& accessKeySecret); virtual void UpdateAccessKeyStatus(const std::string& aliuid, bool success) {} - // // currently not support hot relead - // void UpdateLocalRegionEndpointsAndHttpsInfo(const std::string& region, const std::vector& - // endpoints); void UpdateRemoteRegionEndpoints(const std::string& region, - // const std::vector& endpoints, - // RemoteEndpointUpdateAction action = RemoteEndpointUpdateAction::OVERWRITE); - - // std::shared_ptr - // GetCandidateHostsInfo(const std::string& region, const std::string& project, EndpointMode mode); - // bool UpdateHostInfo(const std::string& project, - // EndpointMode mode, - // const std::string& host, - // const std::chrono::milliseconds& latency); - // // only for real ip - // bool UpdateHostInfo(const std::string& region, const std::string& host, const std::chrono::milliseconds& - // latency); - - // only for open source - // std::shared_ptr GetCandidateHostsInfo(const std::string& project, const std::string& endpoint); - - virtual bool UsingHttps(const std::string& region) const; - - // void UpdateOutdatedRealIpRegions(const std::string& region); - // std::string GetRealIp(const std::string& region) const; - -// #ifdef APSARA_UNIT_TEST_MAIN -// virtual void Clear(); -// #endif + virtual bool UsingHttps(const std::string& region) const { return true; } protected: SLSClientManager() = default; @@ -172,85 +56,7 @@ class SLSClientManager { std::string mUserAgent; private: - // struct ProbeNetworkHttpRequest : public AsynHttpRequest { - // std::string mRegion; - // std::string mProject; - // EndpointMode mMode; - // std::string mHost; - - // ProbeNetworkHttpRequest(const std::string& region, - // const std::string& project, - // EndpointMode mode, - // const std::string& host, - // bool httpsFlag); - - // bool IsContextValid() const override { return true; }; - // void OnSendDone(HttpResponse& response) override; - // }; - virtual void GenerateUserAgent(); - - // void UnInitializedHostProbeThread(); - // void DoProbeUnInitializedHost(); - // void HostProbeThread(); - // void DoProbeHost(); - // bool HasPartiallyInitializedCandidateHostsInfos() const; - // void ClearExpiredCandidateHostsInfos(); - - // void UpdateRealIpThread(); - // void DoUpdateRealIp(); - // void SetRealIp(const std::string& region, const std::string& ip); - // bool HasOutdatedRealIpRegions() const; - - // mutable std::mutex mRegionCandidateEndpointsMapMux; - // std::map mRegionCandidateEndpointsMap; - - // mutable std::mutex mHttpsRegionsMux; - // std::unordered_set mHttpsRegions; - - // mutable std::mutex mCandidateHostsInfosMapMux; - // only custom mode is supported, and one project supports multiple custom endpoints - // std::map>> mProjectCandidateHostsInfosMap; - - // CURLM* mUnInitializedHostProbeClient = nullptr; - // mutable std::mutex mUnInitializedCandidateHostsInfosMux; - // std::vector> mUnInitializedCandidateHostsInfos; - - // std::future mUnInitializedHostProbeThreadRes; - // mutable std::mutex mUnInitializedHostProbeThreadRunningMux; - // bool mIsUnInitializedHostProbeThreadRunning = true; - - // CURLM* mHostProbeClient = nullptr; - // mutable std::mutex mPartiallyInitializedCandidateHostsInfosMux; - // std::vector> mPartiallyInitializedCandidateHostsInfos; - - // std::future mHostProbeThreadRes; - // mutable std::mutex mHostProbeThreadRunningMux; - // bool mIsHostProbeThreadRunning = true; - // #ifdef APSARA_UNIT_TEST_MAIN - // HttpResponse (*mDoProbeNetwork)(const std::unique_ptr& req) = nullptr; - // #endif - - // mutable std::mutex mRegionRealIpMux; - // std::map mRegionRealIpMap; - // mutable std::mutex mOutdatedRealIpRegionsMux; - // std::vector mOutdatedRealIpRegions; - // mutable std::mutex mRegionRealIpCandidateHostsInfosMapMux; - // std::map mRegionRealIpCandidateHostsInfosMap; - // bool (*mGetEndpointRealIp)(const std::string& endpoint, std::string& ip) = nullptr; - - // std::future mUpdateRealIpThreadRes; - // mutable std::mutex mUpdateRealIpThreadRunningMux; - // bool mIsUpdateRealIpThreadRunning = true; - - // mutable std::condition_variable mStopCV; - -#ifdef APSARA_UNIT_TEST_MAIN - friend class FlusherSLSUnittest; - friend class SLSClientManagerUnittest; - // friend class ProbeNetworkMock; - // friend class GetRealIpMock; -#endif }; void PreparePostLogStoreLogsRequest(const std::string& accessKeyId, @@ -309,10 +115,5 @@ SLSResponse PutWebTracking(const std::string& host, const std::string& compressType, const std::string& body, size_t rawSize); -// bool GetEndpointRealIp(const std::string& endpoint, std::string& ip); - -// #ifdef APSARA_UNIT_TEST_MAIN -// extern HttpResponse (*DoGetRealIp)(const std::unique_ptr& req); -// #endif } // namespace logtail diff --git a/core/plugin/flusher/sls/SLSResponse.h b/core/plugin/flusher/sls/SLSResponse.h index 6e52a9c2cc..887c6f07f1 100644 --- a/core/plugin/flusher/sls/SLSResponse.h +++ b/core/plugin/flusher/sls/SLSResponse.h @@ -17,6 +17,7 @@ #pragma once #include +#include #include #include "common/http/HttpResponse.h" From d91e98a83068821c40fab6f6226e1ae964cfa1ec Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Thu, 12 Dec 2024 03:05:21 +0000 Subject: [PATCH 12/41] polish --- core/plugin/flusher/sls/SLSClientManager.h | 4 + .../flusher/SLSClientManagerUnittest.cpp | 1338 ----------------- 2 files changed, 4 insertions(+), 1338 deletions(-) diff --git a/core/plugin/flusher/sls/SLSClientManager.h b/core/plugin/flusher/sls/SLSClientManager.h index 2e69294fe3..98a1f81e88 100644 --- a/core/plugin/flusher/sls/SLSClientManager.h +++ b/core/plugin/flusher/sls/SLSClientManager.h @@ -57,6 +57,10 @@ class SLSClientManager { private: virtual void GenerateUserAgent(); + +#ifdef APSARA_UNIT_TEST_MAIN + friend class SLSClientManagerUnittest; +#endif }; void PreparePostLogStoreLogsRequest(const std::string& accessKeyId, diff --git a/core/unittest/flusher/SLSClientManagerUnittest.cpp b/core/unittest/flusher/SLSClientManagerUnittest.cpp index 01d23abe57..71dec9fec5 100644 --- a/core/unittest/flusher/SLSClientManagerUnittest.cpp +++ b/core/unittest/flusher/SLSClientManagerUnittest.cpp @@ -12,1321 +12,24 @@ // See the License for the specific language governing permissions and // limitations under the License. -// #include "app_config/AppConfig.h" #include "plugin/flusher/sls/SLSClientManager.h" #include "unittest/Unittest.h" DECLARE_FLAG_STRING(default_access_key_id); DECLARE_FLAG_STRING(default_access_key); -// DECLARE_FLAG_INT32(sls_hosts_probe_interval_sec); -// DECLARE_FLAG_INT32(sls_all_hosts_probe_interval_sec); -// DECLARE_FLAG_BOOL(send_prefer_real_ip); -// DECLARE_FLAG_INT32(send_switch_real_ip_interval); using namespace std; namespace logtail { -// class HostInfoUnittest : public ::testing::Test { -// public: -// void TestHostname(); -// void TestLatency(); - -// private: -// const std::string mHostname = "project.endpoint"; -// }; - -// void HostInfoUnittest::TestHostname() { -// HostInfo hostInfo(mHostname); -// APSARA_TEST_EQUAL(mHostname, hostInfo.GetHostname()); -// } - -// void HostInfoUnittest::TestLatency() { -// HostInfo hostInfo(mHostname); -// APSARA_TEST_TRUE(hostInfo.IsForbidden()); - -// auto latency = chrono::milliseconds(100); -// hostInfo.SetLatency(latency); -// APSARA_TEST_EQUAL(latency, hostInfo.GetLatency()); -// APSARA_TEST_FALSE(hostInfo.IsForbidden()); - -// hostInfo.SetForbidden(); -// APSARA_TEST_EQUAL(chrono::milliseconds::max(), hostInfo.GetLatency()); -// } - -// UNIT_TEST_CASE(HostInfoUnittest, TestHostname) -// UNIT_TEST_CASE(HostInfoUnittest, TestLatency) - -// class CandidateHostsInfoUnittest : public ::testing::Test { -// public: -// void TestBasicInfo(); -// void TestHostsInfo(); -// void TestUpdateHosts(); -// void TestFirstHost(); - -// private: -// const string mProject = "project"; -// const string mRegion = "region"; -// const EndpointMode mMode = EndpointMode::DEFAULT; -// }; - -// void CandidateHostsInfoUnittest::TestBasicInfo() { -// CandidateHostsInfo info(mProject, mRegion, mMode); -// APSARA_TEST_EQUAL(mProject, info.GetProject()); -// APSARA_TEST_EQUAL(mRegion, info.GetRegion()); -// APSARA_TEST_EQUAL(mMode, info.GetMode()); -// } - -// void CandidateHostsInfoUnittest::TestHostsInfo() { -// const string host1 = mProject + ".endpoint_1"; -// const string host21 = mProject + ".endpoint_2_1"; -// const string host22 = mProject + ".endpoint_2_2"; -// const string host3 = mProject + ".endpoint_3"; - -// CandidateHostsInfo info(mProject, mRegion, mMode); -// info.mCandidateHosts.push_back({host1}); -// info.mCandidateHosts.push_back({host21, host22}); -// info.mCandidateHosts.push_back({host3}); - -// // initialized -// APSARA_TEST_TRUE(info.GetCurrentHost().empty()); -// { -// vector res; -// info.GetAllHosts(res); -// APSARA_TEST_EQUAL(4U, res.size()); -// APSARA_TEST_EQUAL(host1, res[0]); -// APSARA_TEST_EQUAL(host21, res[1]); -// APSARA_TEST_EQUAL(host22, res[2]); -// APSARA_TEST_EQUAL(host3, res[3]); -// } -// { -// vector res; -// info.GetProbeHosts(res); -// APSARA_TEST_EQUAL(4U, res.size()); -// APSARA_TEST_EQUAL(host1, res[0]); -// APSARA_TEST_EQUAL(host21, res[1]); -// APSARA_TEST_EQUAL(host22, res[2]); -// APSARA_TEST_EQUAL(host3, res[3]); -// } - -// // no valid host -// info.SelectBestHost(); -// APSARA_TEST_TRUE(info.GetCurrentHost().empty()); - -// // some hosts become valid -// info.UpdateHostInfo(host21, chrono::milliseconds(100)); -// info.SelectBestHost(); - -// APSARA_TEST_EQUAL(host21, info.GetCurrentHost()); -// { -// vector res; -// info.GetProbeHosts(res); -// APSARA_TEST_EQUAL(1U, res.size()); -// APSARA_TEST_EQUAL(host21, res[0]); -// } -// { -// vector res; -// info.GetAllHosts(res); -// APSARA_TEST_EQUAL(4U, res.size()); -// APSARA_TEST_EQUAL(host1, res[0]); -// APSARA_TEST_EQUAL(host21, res[1]); -// APSARA_TEST_EQUAL(host22, res[2]); -// APSARA_TEST_EQUAL(host3, res[3]); -// } - -// // host with the same priority as the current one has lower latency -// info.UpdateHostInfo(host22, chrono::milliseconds(50)); -// info.SelectBestHost(); - -// APSARA_TEST_EQUAL(host22, info.GetCurrentHost()); -// { -// vector res; -// info.GetProbeHosts(res); -// APSARA_TEST_EQUAL(2U, res.size()); -// APSARA_TEST_EQUAL(host21, res[0]); -// APSARA_TEST_EQUAL(host22, res[1]); -// } -// { -// vector res; -// info.GetAllHosts(res); -// APSARA_TEST_EQUAL(4U, res.size()); -// APSARA_TEST_EQUAL(host1, res[0]); -// APSARA_TEST_EQUAL(host21, res[1]); -// APSARA_TEST_EQUAL(host22, res[2]); -// APSARA_TEST_EQUAL(host3, res[3]); -// } - -// // host with higher priority becomes valid -// info.UpdateHostInfo(host1, chrono::milliseconds(200)); -// info.SelectBestHost(); - -// APSARA_TEST_EQUAL(host1, info.GetCurrentHost()); -// { -// vector res; -// info.GetProbeHosts(res); -// APSARA_TEST_EQUAL(3U, res.size()); -// APSARA_TEST_EQUAL(host1, res[0]); -// APSARA_TEST_EQUAL(host21, res[1]); -// APSARA_TEST_EQUAL(host22, res[2]); -// } -// { -// vector res; -// info.GetAllHosts(res); -// APSARA_TEST_EQUAL(4U, res.size()); -// APSARA_TEST_EQUAL(host1, res[0]); -// APSARA_TEST_EQUAL(host21, res[1]); -// APSARA_TEST_EQUAL(host22, res[2]); -// APSARA_TEST_EQUAL(host3, res[3]); -// } - -// // no change -// info.SelectBestHost(); - -// APSARA_TEST_EQUAL(host1, info.GetCurrentHost()); -// { -// vector res; -// info.GetProbeHosts(res); -// APSARA_TEST_EQUAL(3U, res.size()); -// APSARA_TEST_EQUAL(host1, res[0]); -// APSARA_TEST_EQUAL(host21, res[1]); -// APSARA_TEST_EQUAL(host22, res[2]); -// } -// { -// vector res; -// info.GetAllHosts(res); -// APSARA_TEST_EQUAL(4U, res.size()); -// APSARA_TEST_EQUAL(host1, res[0]); -// APSARA_TEST_EQUAL(host21, res[1]); -// APSARA_TEST_EQUAL(host22, res[2]); -// APSARA_TEST_EQUAL(host3, res[3]); -// } - -// // all hosts becomes invalid -// info.UpdateHostInfo(host1, chrono::milliseconds::max()); -// info.UpdateHostInfo(host21, chrono::milliseconds::max()); -// info.UpdateHostInfo(host22, chrono::milliseconds::max()); -// info.UpdateHostInfo(host3, chrono::milliseconds::max()); -// info.SelectBestHost(); - -// APSARA_TEST_TRUE(info.GetCurrentHost().empty()); -// { -// vector res; -// info.GetProbeHosts(res); -// APSARA_TEST_EQUAL(4U, res.size()); -// APSARA_TEST_EQUAL(host1, res[0]); -// APSARA_TEST_EQUAL(host21, res[1]); -// APSARA_TEST_EQUAL(host22, res[2]); -// APSARA_TEST_EQUAL(host3, res[3]); -// } -// { -// vector res; -// info.GetAllHosts(res); -// APSARA_TEST_EQUAL(4U, res.size()); -// APSARA_TEST_EQUAL(host1, res[0]); -// APSARA_TEST_EQUAL(host21, res[1]); -// APSARA_TEST_EQUAL(host22, res[2]); -// APSARA_TEST_EQUAL(host3, res[3]); -// } -// } - -// void CandidateHostsInfoUnittest::TestUpdateHosts() { -// const string region = "region"; -// const string publicEndpoint = region + ".log.aliyuncs.com"; -// const string privateEndpoint = region + "-intranet.log.aliyuncs.com"; -// const string internalEndpoint = region + "-internal.log.aliyuncs.com"; -// const string globalEndpoint = "log-global.aliyuncs.com"; -// const string customEndpoint = "custom.endpoint"; -// const string publicHost = mProject + "." + publicEndpoint; -// const string privateHost = mProject + "." + privateEndpoint; -// const string internalHost = mProject + "." + internalEndpoint; -// const string globalHost = mProject + "." + globalEndpoint; -// const string customHost = mProject + "." + customEndpoint; -// #ifdef __ENTERPRISE__ -// { -// // default mode -// CandidateHostsInfo info("project", region, EndpointMode::DEFAULT); - -// // from no remote endpoints -// CandidateEndpoints regionEndpoints{EndpointMode::DEFAULT, {publicEndpoint}, {}}; -// info.UpdateHosts(regionEndpoints); -// APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); -// APSARA_TEST_EQUAL(1U, info.mCandidateHosts[0].size()); -// APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][0].GetHostname()); -// APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[0][0].GetLatency()); - -// info.UpdateHostInfo(publicHost, chrono::milliseconds(100)); - -// // to with remote endpoints -// regionEndpoints.mRemoteEndpoints = {privateEndpoint, publicEndpoint}; -// info.UpdateHosts(regionEndpoints); -// APSARA_TEST_EQUAL(2U, info.mCandidateHosts.size()); -// APSARA_TEST_EQUAL(1U, info.mCandidateHosts[0].size()); -// APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][0].GetHostname()); -// APSARA_TEST_EQUAL(chrono::milliseconds(100), info.mCandidateHosts[0][0].GetLatency()); -// APSARA_TEST_EQUAL(1U, info.mCandidateHosts[1].size()); -// APSARA_TEST_EQUAL(privateHost, info.mCandidateHosts[1][0].GetHostname()); -// APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[1][0].GetLatency()); - -// info.UpdateHostInfo(publicHost, chrono::milliseconds(50)); -// info.UpdateHostInfo(privateHost, chrono::milliseconds(150)); - -// // to updated remote endpoints -// regionEndpoints.mRemoteEndpoints = {internalEndpoint, privateEndpoint, publicEndpoint}; -// info.UpdateHosts(regionEndpoints); -// APSARA_TEST_EQUAL(3U, info.mCandidateHosts.size()); -// APSARA_TEST_EQUAL(1U, info.mCandidateHosts[0].size()); -// APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][0].GetHostname()); -// APSARA_TEST_EQUAL(chrono::milliseconds(50), info.mCandidateHosts[0][0].GetLatency()); -// APSARA_TEST_EQUAL(1U, info.mCandidateHosts[1].size()); -// APSARA_TEST_EQUAL(internalHost, info.mCandidateHosts[1][0].GetHostname()); -// APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[1][0].GetLatency()); -// APSARA_TEST_EQUAL(1U, info.mCandidateHosts[2].size()); -// APSARA_TEST_EQUAL(privateHost, info.mCandidateHosts[2][0].GetHostname()); -// APSARA_TEST_EQUAL(chrono::milliseconds(150), info.mCandidateHosts[2][0].GetLatency()); -// } -// { -// // accelerate mode -// { -// CandidateHostsInfo info("project", region, EndpointMode::ACCELERATE); -// // from no remote endpoints -// CandidateEndpoints regionEndpoints{EndpointMode::DEFAULT, {publicEndpoint}, {}}; -// info.UpdateHosts(regionEndpoints); -// APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); -// APSARA_TEST_EQUAL(1U, info.mCandidateHosts[0].size()); -// APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); -// APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[0][0].GetLatency()); - -// info.UpdateHostInfo(globalHost, chrono::milliseconds(100)); - -// // to with remote endpoints -// regionEndpoints.mRemoteEndpoints = {privateEndpoint, publicEndpoint}; -// info.UpdateHosts(regionEndpoints); -// APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); -// APSARA_TEST_EQUAL(2U, info.mCandidateHosts[0].size()); -// APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); -// APSARA_TEST_EQUAL(chrono::milliseconds(100), info.mCandidateHosts[0][0].GetLatency()); -// APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][1].GetHostname()); -// APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[0][1].GetLatency()); - -// info.UpdateHostInfo(globalHost, chrono::milliseconds(50)); -// info.UpdateHostInfo(publicHost, chrono::milliseconds(150)); - -// // to updated remote endpoints -// regionEndpoints.mRemoteEndpoints = {internalEndpoint, privateEndpoint, publicEndpoint}; -// info.UpdateHosts(regionEndpoints); -// APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); -// APSARA_TEST_EQUAL(2U, info.mCandidateHosts[0].size()); -// APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); -// APSARA_TEST_EQUAL(chrono::milliseconds(50), info.mCandidateHosts[0][0].GetLatency()); -// APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][1].GetHostname()); -// APSARA_TEST_EQUAL(chrono::milliseconds(150), info.mCandidateHosts[0][1].GetLatency()); -// } -// { -// CandidateHostsInfo info("project", region, EndpointMode::ACCELERATE); -// // from no remote endpoints -// CandidateEndpoints regionEndpoints{EndpointMode::ACCELERATE, {globalEndpoint}, {}}; -// info.UpdateHosts(regionEndpoints); -// APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); -// APSARA_TEST_EQUAL(1U, info.mCandidateHosts[0].size()); -// APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); -// APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[0][0].GetLatency()); - -// info.UpdateHostInfo(globalHost, chrono::milliseconds(100)); - -// // to with remote endpoints -// regionEndpoints.mRemoteEndpoints = {privateEndpoint, publicEndpoint}; -// info.UpdateHosts(regionEndpoints); -// APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); -// APSARA_TEST_EQUAL(2U, info.mCandidateHosts[0].size()); -// APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); -// APSARA_TEST_EQUAL(chrono::milliseconds(100), info.mCandidateHosts[0][0].GetLatency()); -// APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][1].GetHostname()); -// APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[0][1].GetLatency()); - -// info.UpdateHostInfo(globalHost, chrono::milliseconds(50)); -// info.UpdateHostInfo(publicHost, chrono::milliseconds(150)); - -// // to updated remote endpoints -// regionEndpoints.mRemoteEndpoints = {internalEndpoint, privateEndpoint, publicEndpoint}; -// info.UpdateHosts(regionEndpoints); -// APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); -// APSARA_TEST_EQUAL(2U, info.mCandidateHosts[0].size()); -// APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); -// APSARA_TEST_EQUAL(chrono::milliseconds(50), info.mCandidateHosts[0][0].GetLatency()); -// APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][1].GetHostname()); -// APSARA_TEST_EQUAL(chrono::milliseconds(150), info.mCandidateHosts[0][1].GetLatency()); -// } -// { -// CandidateHostsInfo info("project", region, EndpointMode::ACCELERATE); -// // from no remote endpoints -// CandidateEndpoints regionEndpoints{EndpointMode::CUSTOM, {customEndpoint}, {}}; -// info.UpdateHosts(regionEndpoints); -// APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); -// APSARA_TEST_EQUAL(1U, info.mCandidateHosts[0].size()); -// APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); -// APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[0][0].GetLatency()); - -// info.UpdateHostInfo(globalHost, chrono::milliseconds(100)); - -// // to with remote endpoints -// regionEndpoints.mRemoteEndpoints = {privateEndpoint, publicEndpoint}; -// info.UpdateHosts(regionEndpoints); -// APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); -// APSARA_TEST_EQUAL(2U, info.mCandidateHosts[0].size()); -// APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); -// APSARA_TEST_EQUAL(chrono::milliseconds(100), info.mCandidateHosts[0][0].GetLatency()); -// APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][1].GetHostname()); -// APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[0][1].GetLatency()); - -// info.UpdateHostInfo(globalHost, chrono::milliseconds(50)); -// info.UpdateHostInfo(publicHost, chrono::milliseconds(150)); - -// // to updated remote endpoints -// regionEndpoints.mRemoteEndpoints = {internalEndpoint, privateEndpoint, publicEndpoint}; -// info.UpdateHosts(regionEndpoints); -// APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); -// APSARA_TEST_EQUAL(2U, info.mCandidateHosts[0].size()); -// APSARA_TEST_EQUAL(globalHost, info.mCandidateHosts[0][0].GetHostname()); -// APSARA_TEST_EQUAL(chrono::milliseconds(50), info.mCandidateHosts[0][0].GetLatency()); -// APSARA_TEST_EQUAL(publicHost, info.mCandidateHosts[0][1].GetHostname()); -// APSARA_TEST_EQUAL(chrono::milliseconds(150), info.mCandidateHosts[0][1].GetLatency()); -// } -// } -// #endif -// { -// // custom mode -// CandidateHostsInfo info("project", region, EndpointMode::CUSTOM); - -// // from no remote endpoints -// CandidateEndpoints regionEndpoints{EndpointMode::CUSTOM, {customEndpoint}, {}}; -// info.UpdateHosts(regionEndpoints); -// APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); -// APSARA_TEST_EQUAL(1U, info.mCandidateHosts[0].size()); -// APSARA_TEST_EQUAL(customHost, info.mCandidateHosts[0][0].GetHostname()); -// APSARA_TEST_EQUAL(chrono::milliseconds::max(), info.mCandidateHosts[0][0].GetLatency()); - -// info.UpdateHostInfo(customHost, chrono::milliseconds(100)); - -// // to with remote endpoints -// regionEndpoints.mRemoteEndpoints = {privateEndpoint, publicEndpoint}; -// info.UpdateHosts(regionEndpoints); -// APSARA_TEST_EQUAL(1U, info.mCandidateHosts.size()); -// APSARA_TEST_EQUAL(1U, info.mCandidateHosts[0].size()); -// APSARA_TEST_EQUAL(customHost, info.mCandidateHosts[0][0].GetHostname()); -// APSARA_TEST_EQUAL(chrono::milliseconds(100), info.mCandidateHosts[0][0].GetLatency()); -// } -// } - -// void CandidateHostsInfoUnittest::TestFirstHost() { -// const string host1 = mProject + ".endpoint_1"; -// const string host2 = mProject + ".endpoint_2"; - -// CandidateHostsInfo info(mProject, mRegion, mMode); -// APSARA_TEST_EQUAL("", info.GetFirstHost()); - -// info.mCandidateHosts.push_back({host1, host2}); -// APSARA_TEST_EQUAL(host1, info.GetFirstHost()); - -// info.mCandidateHosts[0].clear(); -// APSARA_TEST_EQUAL("", info.GetFirstHost()); -// } - -// UNIT_TEST_CASE(CandidateHostsInfoUnittest, TestBasicInfo) -// UNIT_TEST_CASE(CandidateHostsInfoUnittest, TestHostsInfo) -// UNIT_TEST_CASE(CandidateHostsInfoUnittest, TestUpdateHosts) -// UNIT_TEST_CASE(CandidateHostsInfoUnittest, TestFirstHost) - -// class ProbeNetworkMock { -// public: -// static HttpResponse DoProbeNetwork(const std::unique_ptr& req) { -// HttpResponse response; -// chrono::milliseconds latency = chrono::milliseconds::max(); -// if (!enableUnavailableHosts) { -// if (req->mProject == project1 && req->mHost == project1 + "." + publicEndpoint1) { -// static uint32_t sec = 1; -// latency = chrono::milliseconds(sec++); -// } else if (req->mProject == project2) { -// if (req->mHost == project2 + "." + publicEndpoint2) { -// static uint32_t sec = 10; -// latency = chrono::milliseconds(sec++); -// } else if (req->mHost == project2 + "." + globalEndpoint) { -// static uint32_t sec = 5; -// latency = chrono::milliseconds(sec++); -// } -// } else if (req->mProject == project3 && req->mHost == project3 + "." + customEndpoint) { -// static uint32_t sec = 20; -// latency = chrono::milliseconds(sec++); -// } -// } else { -// static uint32_t sec = 200; -// latency = chrono::milliseconds(sec++); -// } - -// response.SetResponseTime(latency); -// return response; -// } - -// static void Prepare() { -// publicEndpoint1 = region1 + ".log.aliyuncs.com"; -// privateEndpoint1 = region1 + "-intranet.log.aliyuncs.com"; -// internalEndpoint1 = region1 + "-internal.log.aliyuncs.com"; -// publicEndpoint2 = region2 + ".log.aliyuncs.com"; -// privateEndpoint2 = region2 + "-intranet.log.aliyuncs.com"; -// internalEndpoint2 = region2 + "-internal.log.aliyuncs.com"; - -// // region_1: only public endpoint is available -// // region_2: both accelerate and public endpoints are available -// // region_3: only custom endpoint is available -// // project_x belongs to region_x -// SLSClientManager::GetInstance()->UpdateLocalRegionEndpointsAndHttpsInfo(region1, {publicEndpoint1}); -// SLSClientManager::GetInstance()->UpdateLocalRegionEndpointsAndHttpsInfo(region2, {globalEndpoint}); -// SLSClientManager::GetInstance()->UpdateLocalRegionEndpointsAndHttpsInfo(region3, {customEndpoint}); -// SLSClientManager::GetInstance()->UpdateRemoteRegionEndpoints( -// region1, {internalEndpoint1, privateEndpoint1, publicEndpoint1}); -// SLSClientManager::GetInstance()->UpdateRemoteRegionEndpoints( -// region2, {internalEndpoint2, privateEndpoint2, publicEndpoint2}); -// } - -// static bool enableUnavailableHosts; - -// static string project1; -// static string project2; -// static string project3; -// static string region1; -// static string region2; -// static string region3; -// static string publicEndpoint1; -// static string privateEndpoint1; -// static string internalEndpoint1; -// static string publicEndpoint2; -// static string privateEndpoint2; -// static string internalEndpoint2; -// static string globalEndpoint; -// static string customEndpoint; -// }; - -// bool ProbeNetworkMock::enableUnavailableHosts = false; -// string ProbeNetworkMock::region1 = "region_1"; -// string ProbeNetworkMock::region2 = "region_2"; -// string ProbeNetworkMock::region3 = "region_3"; -// string ProbeNetworkMock::project1 = "project_1"; -// string ProbeNetworkMock::project2 = "project_2"; -// string ProbeNetworkMock::project3 = "project_3"; -// string ProbeNetworkMock::publicEndpoint1; -// string ProbeNetworkMock::privateEndpoint1; -// string ProbeNetworkMock::internalEndpoint1; -// string ProbeNetworkMock::publicEndpoint2; -// string ProbeNetworkMock::privateEndpoint2; -// string ProbeNetworkMock::internalEndpoint2; -// string ProbeNetworkMock::globalEndpoint = "log-global.aliyuncs.com"; -// string ProbeNetworkMock::customEndpoint = "custom.endpoint"; - -// class GetRealIpMock { -// public: -// static bool GetEndpointRealIpMock(const string& endpoint, string& ip) { -// if (endpoint.find(normalRegion) != string::npos) { -// static size_t i = 0; -// ip = normalRegionIps[i]; -// i = (i + 1) % normalRegionIps.size(); -// return true; -// } else if (endpoint.find(unavailableRegion) != string::npos) { -// static size_t i = 0; -// ip = unavailableRegionIps[i]; -// i = (i + 1) % unavailableRegionIps.size(); -// return true; -// } else { -// return false; -// } -// } - -// static HttpResponse DoProbeNetwork(const std::unique_ptr& req) { -// HttpResponse response; -// response.SetResponseTime(chrono::milliseconds(100)); -// return response; -// } - -// static string normalRegion; -// static string unavailableRegion; -// static vector normalRegionIps; -// static vector unavailableRegionIps; -// }; - -// string GetRealIpMock::normalRegion = "region_1"; -// string GetRealIpMock::unavailableRegion = "region_2"; -// vector GetRealIpMock::normalRegionIps = {"192.168.0.1", "192.168.0.2"}; -// vector GetRealIpMock::unavailableRegionIps = {"10.0.0.1", "10.0.0.2"}; - class SLSClientManagerUnittest : public ::testing::Test { public: - // void TestLocalRegionEndpoints(); - // void TestRemoteRegionEndpoints(); - // void TestGetCandidateHostsInfo(); - // void TestUpdateHostInfo(); - // void TestUsingHttps(); - // void TestProbeNetwork(); - // void TestRealIp(); void TestAccessKeyManagement(); - // void TestGetCandidateHostsInfo(); - -protected: - // static void SetUpTestCase() { - // SLSClientManager::GetInstance()->mDoProbeNetwork = ProbeNetworkMock::DoProbeNetwork; - // SLSClientManager::GetInstance()->mGetEndpointRealIp = GetRealIpMock::GetEndpointRealIpMock; - // } - - // void TearDown() override { mManager.Clear(); } private: SLSClientManager mManager; }; -// void SLSClientManagerUnittest::TestLocalRegionEndpoints() { -// const string region = "region"; -// { -// // default -// const string endpoint = region + ".log.aliyuncs.com"; -// mManager.UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint}); - -// auto& item = mManager.mRegionCandidateEndpointsMap[region]; -// APSARA_TEST_EQUAL(EndpointMode::DEFAULT, item.mMode); -// APSARA_TEST_EQUAL(1U, item.mLocalEndpoints.size()); -// APSARA_TEST_EQUAL(endpoint, item.mLocalEndpoints[0]); -// APSARA_TEST_FALSE(mManager.UsingHttps(region)); -// } -// { -// // accelerate (+ default) -// const string endpoint1 = region + "-intranet.log.aliyuncs.com"; -// const string endpoint2 = kAccelerationDataEndpoint; -// mManager.UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); - -// auto& item = mManager.mRegionCandidateEndpointsMap[region]; -// APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, item.mMode); -// APSARA_TEST_EQUAL(0U, item.mLocalEndpoints.size()); -// APSARA_TEST_FALSE(mManager.UsingHttps(region)); -// } -// { -// // custom (+ default) -// const string endpoint1 = region + "-intranet.log.aliyuncs.com"; -// const string endpoint2 = "custom.endpoint"; -// mManager.UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); - -// auto& item = mManager.mRegionCandidateEndpointsMap[region]; -// APSARA_TEST_EQUAL(EndpointMode::CUSTOM, item.mMode); -// APSARA_TEST_EQUAL(1U, item.mLocalEndpoints.size()); -// APSARA_TEST_EQUAL(endpoint2, item.mLocalEndpoints[0]); -// APSARA_TEST_FALSE(mManager.UsingHttps(region)); -// } -// { -// // accelerate + custom (+ default) -// const string endpoint1 = "custom.endpoint"; -// const string endpoint2 = region + "-intranet.log.aliyuncs.com"; -// const string endpoint3 = kAccelerationDataEndpoint; -// mManager.UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2, endpoint3}); - -// auto& item = mManager.mRegionCandidateEndpointsMap[region]; -// APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, item.mMode); -// APSARA_TEST_EQUAL(0U, item.mLocalEndpoints.size()); -// APSARA_TEST_FALSE(mManager.UsingHttps(region)); -// } -// { -// // http -> https -> https -// const string endpoint1 = region + "-intranet.log.aliyuncs.com"; -// const string endpoint2 = "https://" + kAccelerationDataEndpoint; -// mManager.UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); -// APSARA_TEST_TRUE(mManager.UsingHttps(region)); - -// mManager.UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); -// APSARA_TEST_TRUE(mManager.UsingHttps(region)); -// } -// { -// // https -> http -> http -// const string endpoint1 = region + "-intranet.log.aliyuncs.com"; -// const string endpoint2 = kAccelerationDataEndpoint; -// mManager.UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); -// APSARA_TEST_FALSE(mManager.UsingHttps(region)); - -// mManager.UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); -// APSARA_TEST_FALSE(mManager.UsingHttps(region)); -// } -// } - -// void SLSClientManagerUnittest::TestRemoteRegionEndpoints() { -// { -// auto info1 = mManager.GetCandidateHostsInfo("region_1", "project_1", EndpointMode::DEFAULT); -// auto info2 = mManager.GetCandidateHostsInfo("region_1", "project_2", EndpointMode::DEFAULT); -// auto info3 = mManager.GetCandidateHostsInfo("region_2", "project_3", EndpointMode::DEFAULT); -// APSARA_TEST_EQUAL(0U, info3->mCandidateHosts.size()); - -// // create remote info -// mManager.UpdateRemoteRegionEndpoints("region_1", {"endpoint_1", "endpoint_2"}); -// APSARA_TEST_EQUAL(2U, info1->mCandidateHosts.size()); -// APSARA_TEST_EQUAL(2U, info2->mCandidateHosts.size()); -// APSARA_TEST_EQUAL(0U, info3->mCandidateHosts.size()); -// APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[0].size()); -// APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[1].size()); -// APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[0].size()); -// APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[1].size()); -// APSARA_TEST_EQUAL("project_1.endpoint_1", info1->mCandidateHosts[0][0].GetHostname()); -// APSARA_TEST_EQUAL("project_1.endpoint_2", info1->mCandidateHosts[1][0].GetHostname()); -// APSARA_TEST_EQUAL("project_2.endpoint_1", info2->mCandidateHosts[0][0].GetHostname()); -// APSARA_TEST_EQUAL("project_2.endpoint_2", info2->mCandidateHosts[1][0].GetHostname()); - -// // update remote info with overwrite -// mManager.UpdateRemoteRegionEndpoints("region_1", {"endpoint_2", "endpoint_3"}); -// APSARA_TEST_EQUAL(2U, info1->mCandidateHosts.size()); -// APSARA_TEST_EQUAL(2U, info2->mCandidateHosts.size()); -// APSARA_TEST_EQUAL(0U, info3->mCandidateHosts.size()); -// APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[0].size()); -// APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[1].size()); -// APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[0].size()); -// APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[1].size()); -// APSARA_TEST_EQUAL("project_1.endpoint_2", info1->mCandidateHosts[0][0].GetHostname()); -// APSARA_TEST_EQUAL("project_1.endpoint_3", info1->mCandidateHosts[1][0].GetHostname()); -// APSARA_TEST_EQUAL("project_2.endpoint_2", info2->mCandidateHosts[0][0].GetHostname()); -// APSARA_TEST_EQUAL("project_2.endpoint_3", info2->mCandidateHosts[1][0].GetHostname()); - -// // update remote info with create -// mManager.UpdateRemoteRegionEndpoints( -// "region_1", {"endpoint_4"}, SLSClientManager::RemoteEndpointUpdateAction::CREATE); -// APSARA_TEST_EQUAL(2U, info1->mCandidateHosts.size()); -// APSARA_TEST_EQUAL(2U, info2->mCandidateHosts.size()); -// APSARA_TEST_EQUAL(0U, info3->mCandidateHosts.size()); -// APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[0].size()); -// APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[1].size()); -// APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[0].size()); -// APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[1].size()); -// APSARA_TEST_EQUAL("project_1.endpoint_2", info1->mCandidateHosts[0][0].GetHostname()); -// APSARA_TEST_EQUAL("project_1.endpoint_3", info1->mCandidateHosts[1][0].GetHostname()); -// APSARA_TEST_EQUAL("project_2.endpoint_2", info2->mCandidateHosts[0][0].GetHostname()); -// APSARA_TEST_EQUAL("project_2.endpoint_3", info2->mCandidateHosts[1][0].GetHostname()); - -// // update remote info with append -// mManager.UpdateRemoteRegionEndpoints( -// "region_1", {"endpoint_3", "endpoint_1"}, SLSClientManager::RemoteEndpointUpdateAction::APPEND); -// APSARA_TEST_EQUAL(3U, info1->mCandidateHosts.size()); -// APSARA_TEST_EQUAL(3U, info2->mCandidateHosts.size()); -// APSARA_TEST_EQUAL(0U, info3->mCandidateHosts.size()); -// APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[0].size()); -// APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[1].size()); -// APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[2].size()); -// APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[0].size()); -// APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[1].size()); -// APSARA_TEST_EQUAL(1U, info2->mCandidateHosts[2].size()); -// APSARA_TEST_EQUAL("project_1.endpoint_2", info1->mCandidateHosts[0][0].GetHostname()); -// APSARA_TEST_EQUAL("project_1.endpoint_3", info1->mCandidateHosts[1][0].GetHostname()); -// APSARA_TEST_EQUAL("project_1.endpoint_1", info1->mCandidateHosts[2][0].GetHostname()); -// APSARA_TEST_EQUAL("project_2.endpoint_2", info2->mCandidateHosts[0][0].GetHostname()); -// APSARA_TEST_EQUAL("project_2.endpoint_3", info2->mCandidateHosts[1][0].GetHostname()); -// APSARA_TEST_EQUAL("project_2.endpoint_1", info2->mCandidateHosts[2][0].GetHostname()); -// } -// { -// // get candidate host info after remote info is updated -// auto info = mManager.GetCandidateHostsInfo("region_1", "project_1", EndpointMode::DEFAULT); -// APSARA_TEST_EQUAL(3U, info->mCandidateHosts.size()); -// APSARA_TEST_EQUAL(1U, info->mCandidateHosts[0].size()); -// APSARA_TEST_EQUAL(1U, info->mCandidateHosts[1].size()); -// APSARA_TEST_EQUAL(1U, info->mCandidateHosts[2].size()); -// APSARA_TEST_EQUAL("project_1.endpoint_2", info->mCandidateHosts[0][0].GetHostname()); -// APSARA_TEST_EQUAL("project_1.endpoint_3", info->mCandidateHosts[1][0].GetHostname()); -// APSARA_TEST_EQUAL("project_1.endpoint_1", info->mCandidateHosts[2][0].GetHostname()); -// } -// } - -// void SLSClientManagerUnittest::TestGetCandidateHostsInfo() { -// const string region = "region"; -// const string project = "project"; -// const string publicEndpoint = region + ".log.aliyuncs.com"; -// const string privateEndpoint = region + "-intranet.log.aliyuncs.com"; -// const string internalEndpoint = region + "-internal.log.aliyuncs.com"; -// const string globalEndpoint = kAccelerationDataEndpoint; -// const string customEndpoint = "custom.endpoint"; -// const string publicHost = project + "." + publicEndpoint; -// const string privateHost = project + "." + privateEndpoint; -// const string internalHost = project + "." + internalEndpoint; -// const string globalHost = project + "." + globalEndpoint; -// const string customHost = project + "." + customEndpoint; -// { -// // no candidate host info && no region endpoints -// { -// // default mode -// auto info = mManager.GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); -// APSARA_TEST_EQUAL(project, info->GetProject()); -// APSARA_TEST_EQUAL(region, info->GetRegion()); -// APSARA_TEST_EQUAL(EndpointMode::DEFAULT, info->GetMode()); -// APSARA_TEST_EQUAL(0U, info->mCandidateHosts.size()); -// { -// APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap.size()); -// auto& infos = mManager.mProjectCandidateHostsInfosMap[project]; -// APSARA_TEST_EQUAL(1U, infos.size()); -// auto& weakInfo = *infos.begin(); -// APSARA_TEST_FALSE(weakInfo.expired()); -// APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); -// } -// { -// APSARA_TEST_EQUAL(1U, mManager.mRegionCandidateHostsInfosMap.size()); -// auto& infos = mManager.mRegionCandidateHostsInfosMap[region]; -// APSARA_TEST_EQUAL(1U, infos.size()); -// auto& weakInfo = *infos.begin(); -// APSARA_TEST_FALSE(weakInfo.expired()); -// APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); -// } -// APSARA_TEST_EQUAL(1U, mManager.mUnInitializedCandidateHostsInfos.size()); -// APSARA_TEST_EQUAL(info.get(), mManager.mUnInitializedCandidateHostsInfos[0].lock().get()); -// } -// { -// // accelerate mode -// auto info = mManager.GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); -// APSARA_TEST_EQUAL(project, info->GetProject()); -// APSARA_TEST_EQUAL(region, info->GetRegion()); -// APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, info->GetMode()); -// APSARA_TEST_EQUAL(1U, info->mCandidateHosts.size()); -// APSARA_TEST_EQUAL(1U, info->mCandidateHosts[0].size()); -// APSARA_TEST_EQUAL(globalHost, info->mCandidateHosts[0][0].GetHostname()); -// APSARA_TEST_EQUAL(chrono::milliseconds::max(), info->mCandidateHosts[0][0].GetLatency()); -// { -// APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap.size()); -// auto& infos = mManager.mProjectCandidateHostsInfosMap[project]; -// APSARA_TEST_EQUAL(2U, infos.size()); -// auto it = infos.begin(); -// APSARA_TEST_TRUE(it->expired()); -// ++it; -// APSARA_TEST_FALSE(it->expired()); -// APSARA_TEST_EQUAL(info.get(), it->lock().get()); -// } -// { -// APSARA_TEST_EQUAL(1U, mManager.mRegionCandidateHostsInfosMap.size()); -// auto& infos = mManager.mRegionCandidateHostsInfosMap[region]; -// APSARA_TEST_EQUAL(2U, infos.size()); -// auto it = infos.begin(); -// APSARA_TEST_TRUE(it->expired()); -// ++it; -// APSARA_TEST_FALSE(it->expired()); -// APSARA_TEST_EQUAL(info.get(), it->lock().get()); -// } -// APSARA_TEST_EQUAL(2U, mManager.mUnInitializedCandidateHostsInfos.size()); -// APSARA_TEST_EQUAL(info.get(), mManager.mUnInitializedCandidateHostsInfos[1].lock().get()); -// } -// mManager.Clear(); -// } -// { -// // no candidate host info && with region endpoints && region endpoint mode is default -// mManager.UpdateLocalRegionEndpointsAndHttpsInfo(region, {privateEndpoint}); -// { -// // default mode -// auto info = mManager.GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); -// APSARA_TEST_EQUAL(project, info->GetProject()); -// APSARA_TEST_EQUAL(region, info->GetRegion()); -// APSARA_TEST_EQUAL(EndpointMode::DEFAULT, info->GetMode()); -// APSARA_TEST_EQUAL(1U, info->mCandidateHosts.size()); -// APSARA_TEST_EQUAL(1U, info->mCandidateHosts[0].size()); -// APSARA_TEST_EQUAL(privateHost, info->mCandidateHosts[0][0].GetHostname()); -// { -// APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap.size()); -// auto& infos = mManager.mProjectCandidateHostsInfosMap[project]; -// APSARA_TEST_EQUAL(1U, infos.size()); -// auto& weakInfo = *infos.begin(); -// APSARA_TEST_FALSE(weakInfo.expired()); -// APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); -// } -// { -// APSARA_TEST_EQUAL(1U, mManager.mRegionCandidateHostsInfosMap.size()); -// auto& infos = mManager.mRegionCandidateHostsInfosMap[region]; -// APSARA_TEST_EQUAL(1U, infos.size()); -// auto& weakInfo = *infos.begin(); -// APSARA_TEST_FALSE(weakInfo.expired()); -// APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); -// } -// APSARA_TEST_EQUAL(1U, mManager.mUnInitializedCandidateHostsInfos.size()); -// APSARA_TEST_EQUAL(info.get(), mManager.mUnInitializedCandidateHostsInfos[0].lock().get()); -// } -// { -// // accelerate mode -// auto info = mManager.GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); -// APSARA_TEST_EQUAL(project, info->GetProject()); -// APSARA_TEST_EQUAL(region, info->GetRegion()); -// APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, info->GetMode()); -// APSARA_TEST_EQUAL(1U, info->mCandidateHosts.size()); -// APSARA_TEST_EQUAL(1U, info->mCandidateHosts[0].size()); -// APSARA_TEST_EQUAL(globalHost, info->mCandidateHosts[0][0].GetHostname()); -// APSARA_TEST_EQUAL(chrono::milliseconds::max(), info->mCandidateHosts[0][0].GetLatency()); -// { -// APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap.size()); -// auto& infos = mManager.mProjectCandidateHostsInfosMap[project]; -// APSARA_TEST_EQUAL(2U, infos.size()); -// auto it = infos.begin(); -// APSARA_TEST_TRUE(it->expired()); -// ++it; -// APSARA_TEST_FALSE(it->expired()); -// APSARA_TEST_EQUAL(info.get(), it->lock().get()); -// } -// { -// APSARA_TEST_EQUAL(1U, mManager.mRegionCandidateHostsInfosMap.size()); -// auto& infos = mManager.mRegionCandidateHostsInfosMap[region]; -// APSARA_TEST_EQUAL(2U, infos.size()); -// auto it = infos.begin(); -// APSARA_TEST_TRUE(it->expired()); -// ++it; -// APSARA_TEST_FALSE(it->expired()); -// APSARA_TEST_EQUAL(info.get(), it->lock().get()); -// } -// APSARA_TEST_EQUAL(2U, mManager.mUnInitializedCandidateHostsInfos.size()); -// APSARA_TEST_EQUAL(info.get(), mManager.mUnInitializedCandidateHostsInfos[1].lock().get()); -// } -// mManager.Clear(); -// } -// { -// // no candidate host info && with region endpoints && region endpoint mode is accelerate -// mManager.UpdateLocalRegionEndpointsAndHttpsInfo(region, {globalEndpoint}); -// { -// // default mode -// auto info = mManager.GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); -// APSARA_TEST_EQUAL(project, info->GetProject()); -// APSARA_TEST_EQUAL(region, info->GetRegion()); -// APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, info->GetMode()); -// APSARA_TEST_EQUAL(1U, info->mCandidateHosts.size()); -// APSARA_TEST_EQUAL(1U, info->mCandidateHosts[0].size()); -// APSARA_TEST_EQUAL(globalHost, info->mCandidateHosts[0][0].GetHostname()); -// APSARA_TEST_EQUAL(chrono::milliseconds::max(), info->mCandidateHosts[0][0].GetLatency()); -// { -// APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap.size()); -// auto& infos = mManager.mProjectCandidateHostsInfosMap[project]; -// APSARA_TEST_EQUAL(1U, infos.size()); -// auto& weakInfo = *infos.begin(); -// APSARA_TEST_FALSE(weakInfo.expired()); -// APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); -// } -// { -// APSARA_TEST_EQUAL(1U, mManager.mRegionCandidateHostsInfosMap.size()); -// auto& infos = mManager.mRegionCandidateHostsInfosMap[region]; -// APSARA_TEST_EQUAL(1U, infos.size()); -// auto& weakInfo = *infos.begin(); -// APSARA_TEST_FALSE(weakInfo.expired()); -// APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); -// } -// APSARA_TEST_EQUAL(1U, mManager.mUnInitializedCandidateHostsInfos.size()); -// APSARA_TEST_EQUAL(info.get(), mManager.mUnInitializedCandidateHostsInfos[0].lock().get()); -// } -// { -// // accelerate mode -// auto info = mManager.GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); -// APSARA_TEST_EQUAL(project, info->GetProject()); -// APSARA_TEST_EQUAL(region, info->GetRegion()); -// APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, info->GetMode()); -// APSARA_TEST_EQUAL(1U, info->mCandidateHosts.size()); -// APSARA_TEST_EQUAL(1U, info->mCandidateHosts[0].size()); -// APSARA_TEST_EQUAL(globalHost, info->mCandidateHosts[0][0].GetHostname()); -// APSARA_TEST_EQUAL(chrono::milliseconds::max(), info->mCandidateHosts[0][0].GetLatency()); -// { -// APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap.size()); -// auto& infos = mManager.mProjectCandidateHostsInfosMap[project]; -// APSARA_TEST_EQUAL(2U, infos.size()); -// auto it = infos.begin(); -// APSARA_TEST_TRUE(it->expired()); -// ++it; -// APSARA_TEST_FALSE(it->expired()); -// APSARA_TEST_EQUAL(info.get(), it->lock().get()); -// } -// { -// APSARA_TEST_EQUAL(1U, mManager.mRegionCandidateHostsInfosMap.size()); -// auto& infos = mManager.mRegionCandidateHostsInfosMap[region]; -// APSARA_TEST_EQUAL(2U, infos.size()); -// auto it = infos.begin(); -// APSARA_TEST_TRUE(it->expired()); -// ++it; -// APSARA_TEST_FALSE(it->expired()); -// APSARA_TEST_EQUAL(info.get(), it->lock().get()); -// } -// APSARA_TEST_EQUAL(2U, mManager.mUnInitializedCandidateHostsInfos.size()); -// APSARA_TEST_EQUAL(info.get(), mManager.mUnInitializedCandidateHostsInfos[1].lock().get()); -// } -// mManager.Clear(); -// } -// { -// // no candidate host info && with region endpoints && region endpoint mode is custom -// mManager.UpdateLocalRegionEndpointsAndHttpsInfo(region, {customEndpoint}); -// { -// // default mode -// auto info = mManager.GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); -// APSARA_TEST_EQUAL(project, info->GetProject()); -// APSARA_TEST_EQUAL(region, info->GetRegion()); -// APSARA_TEST_EQUAL(EndpointMode::CUSTOM, info->GetMode()); -// APSARA_TEST_EQUAL(1U, info->mCandidateHosts.size()); -// APSARA_TEST_EQUAL(1U, info->mCandidateHosts[0].size()); -// APSARA_TEST_EQUAL(customHost, info->mCandidateHosts[0][0].GetHostname()); -// APSARA_TEST_EQUAL(chrono::milliseconds::max(), info->mCandidateHosts[0][0].GetLatency()); -// { -// APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap.size()); -// auto& infos = mManager.mProjectCandidateHostsInfosMap[project]; -// APSARA_TEST_EQUAL(1U, infos.size()); -// auto& weakInfo = *infos.begin(); -// APSARA_TEST_FALSE(weakInfo.expired()); -// APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); -// } -// { -// APSARA_TEST_EQUAL(1U, mManager.mRegionCandidateHostsInfosMap.size()); -// auto& infos = mManager.mRegionCandidateHostsInfosMap[region]; -// APSARA_TEST_EQUAL(1U, infos.size()); -// auto& weakInfo = *infos.begin(); -// APSARA_TEST_FALSE(weakInfo.expired()); -// APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); -// } -// APSARA_TEST_EQUAL(1U, mManager.mUnInitializedCandidateHostsInfos.size()); -// APSARA_TEST_EQUAL(info.get(), mManager.mUnInitializedCandidateHostsInfos[0].lock().get()); -// } -// { -// // accelerate mode -// auto info = mManager.GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); -// APSARA_TEST_EQUAL(project, info->GetProject()); -// APSARA_TEST_EQUAL(region, info->GetRegion()); -// APSARA_TEST_EQUAL(EndpointMode::ACCELERATE, info->GetMode()); -// APSARA_TEST_EQUAL(1U, info->mCandidateHosts.size()); -// APSARA_TEST_EQUAL(1U, info->mCandidateHosts[0].size()); -// APSARA_TEST_EQUAL(globalHost, info->mCandidateHosts[0][0].GetHostname()); -// APSARA_TEST_EQUAL(chrono::milliseconds::max(), info->mCandidateHosts[0][0].GetLatency()); -// { -// APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap.size()); -// auto& infos = mManager.mProjectCandidateHostsInfosMap[project]; -// APSARA_TEST_EQUAL(2U, infos.size()); -// auto& weakInfo = *(++infos.begin()); -// APSARA_TEST_FALSE(weakInfo.expired()); -// APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); -// } -// { -// APSARA_TEST_EQUAL(1U, mManager.mRegionCandidateHostsInfosMap.size()); -// auto& infos = mManager.mRegionCandidateHostsInfosMap[region]; -// APSARA_TEST_EQUAL(2U, infos.size()); -// auto& weakInfo = *(++infos.begin()); -// APSARA_TEST_FALSE(weakInfo.expired()); -// APSARA_TEST_EQUAL(info.get(), weakInfo.lock().get()); -// } -// APSARA_TEST_EQUAL(2U, mManager.mUnInitializedCandidateHostsInfos.size()); -// APSARA_TEST_EQUAL(info.get(), mManager.mUnInitializedCandidateHostsInfos[1].lock().get()); -// } -// mManager.Clear(); -// } -// { -// // candidate host info exists && no region endpoints -// auto info1 = mManager.GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); -// auto info2 = mManager.GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); -// APSARA_TEST_EQUAL(info1.get(), info2.get()); -// APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap.size()); -// APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap[project].size()); -// APSARA_TEST_EQUAL(1U, mManager.mUnInitializedCandidateHostsInfos.size()); -// APSARA_TEST_EQUAL(info1.get(), mManager.mUnInitializedCandidateHostsInfos[0].lock().get()); - -// mManager.Clear(); -// } -// { -// // candidate host info exists && with region endpoints && region endpoint mode is default -// mManager.UpdateLocalRegionEndpointsAndHttpsInfo(region, {privateEndpoint}); -// auto infoDefault = mManager.GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); -// auto infoAcc = mManager.GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); -// APSARA_TEST_EQUAL(2U, mManager.mUnInitializedCandidateHostsInfos.size()); -// APSARA_TEST_EQUAL(infoDefault.get(), mManager.mUnInitializedCandidateHostsInfos[0].lock().get()); -// APSARA_TEST_EQUAL(infoAcc.get(), mManager.mUnInitializedCandidateHostsInfos[1].lock().get()); -// { -// // default mode -// auto info = mManager.GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); -// APSARA_TEST_EQUAL(infoDefault.get(), info.get()); -// APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap.size()); -// APSARA_TEST_EQUAL(2U, mManager.mProjectCandidateHostsInfosMap[project].size()); -// } -// { -// // accelerate mode -// auto info = mManager.GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); -// APSARA_TEST_EQUAL(infoAcc.get(), info.get()); -// APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap.size()); -// APSARA_TEST_EQUAL(2U, mManager.mProjectCandidateHostsInfosMap[project].size()); -// } -// mManager.Clear(); -// } -// { -// // candidate host info exists && with region endpoints && region endpoint mode is accelerate -// mManager.UpdateLocalRegionEndpointsAndHttpsInfo(region, {globalEndpoint}); -// auto infoAcc = mManager.GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); -// APSARA_TEST_EQUAL(1U, mManager.mUnInitializedCandidateHostsInfos.size()); -// APSARA_TEST_EQUAL(infoAcc.get(), mManager.mUnInitializedCandidateHostsInfos[0].lock().get()); -// { -// // default mode -// auto info = mManager.GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); -// APSARA_TEST_EQUAL(infoAcc.get(), info.get()); -// APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap.size()); -// APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap[project].size()); -// } -// { -// // accelerate mode -// auto info = mManager.GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); -// APSARA_TEST_EQUAL(infoAcc.get(), info.get()); -// APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap.size()); -// APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap[project].size()); -// } -// mManager.Clear(); -// } -// { -// // candidate host info exists && with region endpoints && region endpoint mode is custom -// mManager.UpdateLocalRegionEndpointsAndHttpsInfo(region, {customEndpoint}); -// auto infoCustom = mManager.GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); -// auto infoAcc = mManager.GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); -// APSARA_TEST_EQUAL(2U, mManager.mUnInitializedCandidateHostsInfos.size()); -// APSARA_TEST_EQUAL(infoCustom.get(), mManager.mUnInitializedCandidateHostsInfos[0].lock().get()); -// APSARA_TEST_EQUAL(infoAcc.get(), mManager.mUnInitializedCandidateHostsInfos[1].lock().get()); -// { -// // default mode -// auto info = mManager.GetCandidateHostsInfo(region, project, EndpointMode::DEFAULT); -// APSARA_TEST_EQUAL(infoCustom.get(), info.get()); -// APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap.size()); -// APSARA_TEST_EQUAL(2U, mManager.mProjectCandidateHostsInfosMap[project].size()); -// } -// { -// // accelerate mode -// auto info = mManager.GetCandidateHostsInfo(region, project, EndpointMode::ACCELERATE); -// APSARA_TEST_EQUAL(infoAcc.get(), info.get()); -// APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap.size()); -// APSARA_TEST_EQUAL(2U, mManager.mProjectCandidateHostsInfosMap[project].size()); -// } -// mManager.Clear(); -// } -// } - -// void SLSClientManagerUnittest::TestUpdateHostInfo() { -// const string region = "region"; -// const string project = "project"; -// const EndpointMode mode = EndpointMode::DEFAULT; -// const string endpoint1 = region + ".log.aliyuncs.com"; -// const string endpoint2 = region + "-intranet.log.aliyuncs.com"; -// const string host1 = project + "." + endpoint1; -// const string host2 = project + "." + endpoint2; -// mManager.UpdateLocalRegionEndpointsAndHttpsInfo(region, {endpoint1, endpoint2}); -// { -// auto info = mManager.GetCandidateHostsInfo(region, project, mode); - -// APSARA_TEST_TRUE(mManager.UpdateHostInfo(project, mode, host1, chrono::milliseconds(100))); -// APSARA_TEST_EQUAL(chrono::milliseconds(100), info->mCandidateHosts[0][0].GetLatency()); -// } - -// // expired info -// APSARA_TEST_FALSE(mManager.UpdateHostInfo(project, mode, host1, chrono::milliseconds(50))); - -// // unknown project -// APSARA_TEST_FALSE(mManager.UpdateHostInfo("unknown_project", mode, host1, chrono::milliseconds(50))); -// } - -// void SLSClientManagerUnittest::TestUsingHttps() { -// const string region = "region"; -// const string host = region + ".log.aliyuncs.com"; -// mManager.UpdateLocalRegionEndpointsAndHttpsInfo(region, {host}); -// APSARA_TEST_FALSE(mManager.UsingHttps(region)); - -// AppConfig::GetInstance()->mSendDataPort = 443; -// APSARA_TEST_TRUE(mManager.UsingHttps(region)); -// } - -// void SLSClientManagerUnittest::TestProbeNetwork() { -// ProbeNetworkMock::Prepare(); -// const string publicHost1 = ProbeNetworkMock::project1 + "." + ProbeNetworkMock::publicEndpoint1; -// const string privateHost1 = ProbeNetworkMock::project1 + "." + ProbeNetworkMock::privateEndpoint1; -// const string internalHost1 = ProbeNetworkMock::project1 + "." + ProbeNetworkMock::internalEndpoint1; -// const string globalHost1 = ProbeNetworkMock::project1 + "." + ProbeNetworkMock::globalEndpoint; -// const string publicHost2 = ProbeNetworkMock::project2 + "." + ProbeNetworkMock::publicEndpoint2; -// const string globalHost2 = ProbeNetworkMock::project2 + "." + ProbeNetworkMock::globalEndpoint; -// const string globalHost3 = ProbeNetworkMock::project3 + "." + ProbeNetworkMock::globalEndpoint; -// const string customHost3 = ProbeNetworkMock::project3 + "." + ProbeNetworkMock::customEndpoint; - -// // ! means not available -// vector> infos; -// infos.push_back(mManager.GetCandidateHostsInfo( -// ProbeNetworkMock::region1, ProbeNetworkMock::project1, EndpointMode::DEFAULT)); // public !internal !private -// infos.push_back(mManager.GetCandidateHostsInfo( -// ProbeNetworkMock::region1, ProbeNetworkMock::project1, EndpointMode::ACCELERATE)); // !accelerate public -// infos.push_back(mManager.GetCandidateHostsInfo( -// ProbeNetworkMock::region2, ProbeNetworkMock::project2, EndpointMode::DEFAULT)); // accelerate public -// infos.push_back(mManager.GetCandidateHostsInfo( -// ProbeNetworkMock::region3, ProbeNetworkMock::project3, EndpointMode::DEFAULT)); // custom -// infos.push_back(mManager.GetCandidateHostsInfo( -// ProbeNetworkMock::region3, ProbeNetworkMock::project3, EndpointMode::ACCELERATE)); // !accelerate - -// // probe uninitialized host -// mManager.DoProbeUnInitializedHost(); -// APSARA_TEST_TRUE(mManager.mUnInitializedCandidateHostsInfos.empty()); -// APSARA_TEST_EQUAL(infos.size(), mManager.mPartiallyInitializedCandidateHostsInfos.size()); -// for (size_t i = 0; i < infos.size(); ++i) { -// APSARA_TEST_EQUAL(infos[i].get(), mManager.mPartiallyInitializedCandidateHostsInfos[i].lock().get()); -// } - -// APSARA_TEST_EQUAL(publicHost1, infos[0]->GetCurrentHost()); -// APSARA_TEST_EQUAL("", infos[1]->GetCurrentHost()); -// APSARA_TEST_EQUAL(globalHost2, infos[2]->GetCurrentHost()); -// APSARA_TEST_EQUAL(customHost3, infos[3]->GetCurrentHost()); -// APSARA_TEST_EQUAL("", infos[4]->GetCurrentHost()); -// vector len = {1, 2, 1, 1, 1}; -// for (size_t i = 0; i < infos.size(); ++i) { -// vector res; -// infos[i]->GetProbeHosts(res); -// APSARA_TEST_EQUAL(len[i], res.size()); -// } - -// // probe partially initialized host -// mManager.DoProbeHost(); -// APSARA_TEST_TRUE(mManager.mPartiallyInitializedCandidateHostsInfos.empty()); - -// APSARA_TEST_EQUAL(publicHost1, infos[0]->GetCurrentHost()); -// APSARA_TEST_EQUAL(publicHost1, infos[1]->GetCurrentHost()); -// APSARA_TEST_EQUAL(globalHost2, infos[2]->GetCurrentHost()); -// APSARA_TEST_EQUAL(customHost3, infos[3]->GetCurrentHost()); -// APSARA_TEST_EQUAL("", infos[4]->GetCurrentHost()); -// len = {1, 1, 2, 1, 1}; -// for (size_t i = 0; i < infos.size(); ++i) { -// vector res; -// infos[i]->GetProbeHosts(res); -// APSARA_TEST_EQUAL(len[i], res.size()); -// } - -// vector> oldLatencies; -// for (const auto& info : infos) { -// auto& latency = oldLatencies.emplace_back(); -// for (const auto& item : info->mCandidateHosts) { -// for (const auto& entry : item) { -// latency.push_back(entry.GetLatency()); -// } -// } -// } - -// // probe all avaialble hosts -// INT32_FLAG(sls_hosts_probe_interval_sec) = 0; -// mManager.DoProbeHost(); - -// APSARA_TEST_EQUAL(publicHost1, infos[0]->GetCurrentHost()); -// APSARA_TEST_EQUAL(publicHost1, infos[1]->GetCurrentHost()); -// APSARA_TEST_EQUAL(globalHost2, infos[2]->GetCurrentHost()); -// APSARA_TEST_EQUAL(customHost3, infos[3]->GetCurrentHost()); -// APSARA_TEST_EQUAL("", infos[4]->GetCurrentHost()); -// len = {1, 1, 2, 1, 1}; -// for (size_t i = 0; i < infos.size(); ++i) { -// vector res; -// infos[i]->GetProbeHosts(res); -// APSARA_TEST_EQUAL(len[i], res.size()); -// } - -// vector> newLatencies; -// for (const auto& info : infos) { -// auto& latency = newLatencies.emplace_back(); -// for (const auto& item : info->mCandidateHosts) { -// for (const auto& entry : item) { -// latency.push_back(entry.GetLatency()); -// } -// } -// } - -// for (size_t i = 0; i < oldLatencies.size(); ++i) { -// for (size_t j = 0; j < oldLatencies[i].size(); ++j) { -// if (oldLatencies[i][j] != chrono::milliseconds::max()) { -// APSARA_TEST_NOT_EQUAL(newLatencies[i][j], oldLatencies[i][j]); -// } else { -// APSARA_TEST_EQUAL(newLatencies[i][j], oldLatencies[i][j]); -// } -// } -// } -// INT32_FLAG(sls_hosts_probe_interval_sec) = 60; - -// // probe all hosts -// INT32_FLAG(sls_all_hosts_probe_interval_sec) = 0; -// ProbeNetworkMock::enableUnavailableHosts = true; -// mManager.DoProbeHost(); - -// APSARA_TEST_EQUAL(publicHost1, infos[0]->GetCurrentHost()); -// APSARA_TEST_EQUAL(globalHost1, infos[1]->GetCurrentHost()); -// APSARA_TEST_EQUAL(globalHost2, infos[2]->GetCurrentHost()); -// APSARA_TEST_EQUAL(customHost3, infos[3]->GetCurrentHost()); -// APSARA_TEST_EQUAL(globalHost3, infos[4]->GetCurrentHost()); - -// len = {3, 2, 2, 1, 1}; -// for (size_t i = 0; i < infos.size(); ++i) { -// vector res; -// infos[i]->GetProbeHosts(res); -// APSARA_TEST_EQUAL(len[i], res.size()); -// } -// ProbeNetworkMock::enableUnavailableHosts = false; -// INT32_FLAG(sls_all_hosts_probe_interval_sec) = 300; - -// // expired info -// infos[2].reset(); -// infos[4].reset(); -// mManager.DoProbeHost(); -// APSARA_TEST_EQUAL(2U, mManager.mProjectCandidateHostsInfosMap.size()); -// APSARA_TEST_EQUAL(2U, mManager.mProjectCandidateHostsInfosMap[ProbeNetworkMock::project1].size()); -// APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap[ProbeNetworkMock::project3].size()); -// APSARA_TEST_EQUAL(2U, mManager.mRegionCandidateHostsInfosMap.size()); -// APSARA_TEST_EQUAL(2U, mManager.mRegionCandidateHostsInfosMap[ProbeNetworkMock::region1].size()); -// APSARA_TEST_EQUAL(1U, mManager.mRegionCandidateHostsInfosMap[ProbeNetworkMock::region3].size()); -// } - -// void SLSClientManagerUnittest::TestRealIp() { -// BOOL_FLAG(send_prefer_real_ip) = true; -// mManager.mDoProbeNetwork = GetRealIpMock::DoProbeNetwork; - -// const string unknownRegion = "unknown_region"; -// const string normalRegionEndpoint1 = GetRealIpMock::normalRegion + "-intranet.log.aliyuncs.com"; -// const string normalRegionEndpoint2 = GetRealIpMock::normalRegion + ".log.aliyuncs.com"; -// const string unavailableRegionEndpoint = GetRealIpMock::unavailableRegion + ".log.aliyuncs.com"; -// const string unknownRegionEndpoint = unknownRegion + ".log.aliyuncs.com"; -// mManager.UpdateRemoteRegionEndpoints(GetRealIpMock::normalRegion, {normalRegionEndpoint1, -// normalRegionEndpoint2}); mManager.UpdateRemoteRegionEndpoints(GetRealIpMock::unavailableRegion, -// {unavailableRegionEndpoint}); mManager.UpdateRemoteRegionEndpoints(unknownRegion, {unknownRegionEndpoint}); -// APSARA_TEST_EQUAL(3U, mManager.mRegionRealIpCandidateHostsInfosMap.size()); -// { -// vector hosts; -// mManager.mRegionRealIpCandidateHostsInfosMap.at(GetRealIpMock::normalRegion).GetAllHosts(hosts); -// APSARA_TEST_EQUAL(2U, hosts.size()); -// APSARA_TEST_EQUAL(normalRegionEndpoint1, hosts[0]); -// APSARA_TEST_EQUAL(normalRegionEndpoint2, hosts[1]); -// } -// { -// vector hosts; -// mManager.mRegionRealIpCandidateHostsInfosMap.at(GetRealIpMock::unavailableRegion).GetAllHosts(hosts); -// APSARA_TEST_EQUAL(1U, hosts.size()); -// APSARA_TEST_EQUAL(unavailableRegionEndpoint, hosts[0]); -// } -// { -// vector hosts; -// mManager.mRegionRealIpCandidateHostsInfosMap.at(unknownRegion).GetAllHosts(hosts); -// APSARA_TEST_EQUAL(1U, hosts.size()); -// APSARA_TEST_EQUAL(unknownRegionEndpoint, hosts[0]); -// } - -// // no endpoint available -// mManager.DoUpdateRealIp(); -// APSARA_TEST_EQUAL(0U, mManager.mRegionRealIpMap.size()); - -// // probe host -// INT32_FLAG(sls_all_hosts_probe_interval_sec) = 0; -// mManager.DoProbeHost(); -// INT32_FLAG(sls_all_hosts_probe_interval_sec) = 300; -// for (const auto& item : mManager.mRegionRealIpCandidateHostsInfosMap) { -// APSARA_TEST_NOT_EQUAL("", item.second.GetCurrentHost()); -// } - -// // update all regions -// INT32_FLAG(send_switch_real_ip_interval) = 0; -// mManager.UpdateOutdatedRealIpRegions(GetRealIpMock::unavailableRegion); -// mManager.DoUpdateRealIp(); -// APSARA_TEST_TRUE(mManager.mOutdatedRealIpRegions.empty()); -// APSARA_TEST_EQUAL(2U, mManager.mRegionRealIpMap.size()); -// APSARA_TEST_EQUAL(GetRealIpMock::normalRegionIps[0], mManager.GetRealIp(GetRealIpMock::normalRegion)); -// APSARA_TEST_EQUAL(GetRealIpMock::unavailableRegionIps[0], mManager.GetRealIp(GetRealIpMock::unavailableRegion)); -// INT32_FLAG(send_switch_real_ip_interval) = 60; - -// // update only outdated regions -// mManager.UpdateOutdatedRealIpRegions(GetRealIpMock::normalRegion); -// mManager.DoUpdateRealIp(); -// APSARA_TEST_TRUE(mManager.mOutdatedRealIpRegions.empty()); -// APSARA_TEST_EQUAL(2U, mManager.mRegionRealIpMap.size()); -// APSARA_TEST_EQUAL(GetRealIpMock::normalRegionIps[1], mManager.GetRealIp(GetRealIpMock::normalRegion)); -// APSARA_TEST_EQUAL(GetRealIpMock::unavailableRegionIps[0], mManager.GetRealIp(GetRealIpMock::unavailableRegion)); - -// mManager.mDoProbeNetwork = ProbeNetworkMock::DoProbeNetwork; -// BOOL_FLAG(send_prefer_real_ip) = false; -// } - void SLSClientManagerUnittest::TestAccessKeyManagement() { string accessKeyId, accessKeySecret; SLSClientManager::AuthType type; @@ -1336,48 +39,7 @@ void SLSClientManagerUnittest::TestAccessKeyManagement() { APSARA_TEST_EQUAL(STRING_FLAG(default_access_key), accessKeySecret); } -// void SLSClientManagerUnittest::TestGetCandidateHostsInfo() { -// const string project = "project"; -// const string endpoint = "endpoint"; -// CandidateHostsInfo* infoPtr = nullptr; -// { -// auto info1 = mManager.GetCandidateHostsInfo(project, endpoint); -// auto info2 = mManager.GetCandidateHostsInfo(project, endpoint); -// infoPtr = info1.get(); -// APSARA_TEST_EQUAL(info1.get(), info2.get()); -// APSARA_TEST_EQUAL(project, info1->GetProject()); -// APSARA_TEST_EQUAL("", info1->GetRegion()); -// APSARA_TEST_EQUAL(EndpointMode::CUSTOM, info1->GetMode()); -// APSARA_TEST_TRUE(info1->IsInitialized()); -// APSARA_TEST_EQUAL(project + "." + endpoint, info1->GetCurrentHost()); -// APSARA_TEST_EQUAL(1U, info1->mCandidateHosts.size()); -// APSARA_TEST_EQUAL(1U, info1->mCandidateHosts[0].size()); -// APSARA_TEST_EQUAL(project + "." + endpoint, info1->mCandidateHosts[0][0].GetHostname()); -// APSARA_TEST_EQUAL(chrono::milliseconds(10), info1->mCandidateHosts[0][0].GetLatency()); -// APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap.size()); -// auto& infos = mManager.mProjectCandidateHostsInfosMap[project]; -// APSARA_TEST_EQUAL(1U, infos.size()); -// auto& weakInfo = *infos.begin(); -// APSARA_TEST_FALSE(weakInfo.expired()); -// APSARA_TEST_EQUAL(info1.get(), weakInfo.lock().get()); -// } -// { -// auto info = mManager.GetCandidateHostsInfo(project, endpoint); -// APSARA_TEST_NOT_EQUAL(infoPtr, info.get()); -// APSARA_TEST_EQUAL(1U, mManager.mProjectCandidateHostsInfosMap.size()); -// APSARA_TEST_EQUAL(2U, mManager.mProjectCandidateHostsInfosMap[project].size()); -// } -// } - -// UNIT_TEST_CASE(SLSClientManagerUnittest, TestLocalRegionEndpoints) -// UNIT_TEST_CASE(SLSClientManagerUnittest, TestRemoteRegionEndpoints) -// UNIT_TEST_CASE(SLSClientManagerUnittest, TestGetCandidateHostsInfo) -// UNIT_TEST_CASE(SLSClientManagerUnittest, TestUpdateHostInfo) -// UNIT_TEST_CASE(SLSClientManagerUnittest, TestUsingHttps) -// UNIT_TEST_CASE(SLSClientManagerUnittest, TestProbeNetwork) -// UNIT_TEST_CASE(SLSClientManagerUnittest, TestRealIp) UNIT_TEST_CASE(SLSClientManagerUnittest, TestAccessKeyManagement) -// UNIT_TEST_CASE(SLSClientManagerUnittest, TestGetCandidateHostsInfo) } // namespace logtail From dfdad390c87090a7d687f23887c508e243f93a29 Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Thu, 12 Dec 2024 08:56:45 +0000 Subject: [PATCH 13/41] polish --- core/plugin/flusher/sls/FlusherSLS.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/core/plugin/flusher/sls/FlusherSLS.cpp b/core/plugin/flusher/sls/FlusherSLS.cpp index 6e5d5b029b..8246df81dd 100644 --- a/core/plugin/flusher/sls/FlusherSLS.cpp +++ b/core/plugin/flusher/sls/FlusherSLS.cpp @@ -374,6 +374,9 @@ bool FlusherSLS::Init(const Json::Value& config, Json::Value& optionalGoPipeline } mCandidateHostsInfo = EnterpriseSLSClientManager::GetInstance()->GetCandidateHostsInfo(mRegion, mProject, mEndpointMode); + LOG_INFO(mContext->GetLogger(), + ("get candidate hosts info, region", mRegion)("project", mProject)("logstore", mLogstore)( + "endpoint mode", EndpointModeToString(mCandidateHostsInfo->GetMode()))); #else // Endpoint if (!GetMandatoryStringParam(config, "Endpoint", mEndpoint, errorMsg)) { @@ -600,6 +603,10 @@ bool FlusherSLS::BuildRequest(SenderQueueItem* item, unique_ptr auto info = EnterpriseSLSClientManager::GetInstance()->GetCandidateHostsInfo(mRegion, mProject, mEndpointMode); if (mCandidateHostsInfo.get() != info.get()) { + LOG_INFO(sLogger, + ("update candidate hosts info, region", mRegion)("project", mProject)("logstore", mLogstore)( + "from", EndpointModeToString(mCandidateHostsInfo->GetMode()))( + "to", EndpointModeToString(info->GetMode()))); mCandidateHostsInfo = info; } data->mCurrentHost = mCandidateHostsInfo->GetCurrentHost(); @@ -611,6 +618,10 @@ bool FlusherSLS::BuildRequest(SenderQueueItem* item, unique_ptr // in case local region endpoint mode is changed, we should always check before sending auto info = EnterpriseSLSClientManager::GetInstance()->GetCandidateHostsInfo(mRegion, mProject, mEndpointMode); if (mCandidateHostsInfo.get() != info.get()) { + LOG_INFO(sLogger, + ("update candidate hosts info, region", mRegion)("project", mProject)("logstore", mLogstore)( + "from", EndpointModeToString(mCandidateHostsInfo->GetMode()))( + "to", EndpointModeToString(info->GetMode()))); mCandidateHostsInfo = info; } data->mCurrentHost = mCandidateHostsInfo->GetCurrentHost(); From ecb339820c033cd43ade73deb4437da4bea2cd55 Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Fri, 13 Dec 2024 09:18:25 +0000 Subject: [PATCH 14/41] polish --- core/plugin/flusher/sls/DiskBufferWriter.cpp | 11 +++++++++-- core/plugin/flusher/sls/FlusherSLS.cpp | 12 +++++++----- core/plugin/flusher/sls/SLSClientManager.h | 1 - 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/core/plugin/flusher/sls/DiskBufferWriter.cpp b/core/plugin/flusher/sls/DiskBufferWriter.cpp index 6b1b4d8a9d..7197d6f59a 100644 --- a/core/plugin/flusher/sls/DiskBufferWriter.cpp +++ b/core/plugin/flusher/sls/DiskBufferWriter.cpp @@ -576,7 +576,14 @@ void DiskBufferWriter::SendEncryptionBuffer(const std::string& filename, int32_t discardCount++; break; } - SLSClientManager::GetInstance()->UpdateAccessKeyStatus(bufferMeta.aliuid(), !hasAuthError); +#ifdef __ENTERPRISE__ + if (sendRes != SEND_NETWORK_ERROR && sendRes != SEND_SERVER_ERROR) { + EnterpriseSLSClientManager::GetInstance()->UpdateAccessKeyStatus(bufferMeta.aliuid(), + !hasAuthError); + EnterpriseSLSClientManager::GetInstance()->UpdateProjectAnonymousWriteStatus( + bufferMeta.project(), !hasAuthError); + } +#endif if (time(nullptr) - beginTime >= INT32_FLAG(discard_send_fail_interval)) { sendResult = true; discardCount++; @@ -813,7 +820,7 @@ SLSResponse DiskBufferWriter::SendBufferFileData(const sls_logs::LogtailBufferMe if (!SLSClientManager::GetInstance()->GetAccessKey(bufferMeta.aliuid(), type, accessKeyId, accessKeySecret)) { #ifdef __ENTERPRISE__ if (!EnterpriseSLSClientManager::GetInstance()->GetAccessKeyIfProjectSupportsAnonymousWrite( - project, type, accessKeyId, accessKeySecret)) { + bufferMeta.project(), type, accessKeyId, accessKeySecret)) { SLSResponse response; response.mErrorCode = LOGE_UNAUTHORIZED; response.mErrorMsg = "can not get valid access key"; diff --git a/core/plugin/flusher/sls/FlusherSLS.cpp b/core/plugin/flusher/sls/FlusherSLS.cpp index 84ecf36e40..d08b258826 100644 --- a/core/plugin/flusher/sls/FlusherSLS.cpp +++ b/core/plugin/flusher/sls/FlusherSLS.cpp @@ -596,7 +596,7 @@ bool FlusherSLS::BuildRequest(SenderQueueItem* item, unique_ptr if (!SLSClientManager::GetInstance()->GetAccessKey(mAliuid, type, accessKeyId, accessKeySecret)) { #ifdef __ENTERPRISE__ if (!EnterpriseSLSClientManager::GetInstance()->GetAccessKeyIfProjectSupportsAnonymousWrite( - project, type, accessKeyId, accessKeySecret)) { + mProject, type, accessKeyId, accessKeySecret)) { *keepItem = true; return false; } @@ -671,6 +671,7 @@ void FlusherSLS::OnSendDone(const HttpResponse& response, SenderQueueItem* item) int32_t curTime = time(NULL); auto curSystemTime = chrono::system_clock::now(); bool hasAuthError = false; + SendResult sendResult = SEND_OK; if (slsResponse.mStatusCode == 200) { auto& cpt = data->mExactlyOnceCheckpoint; if (cpt) { @@ -698,7 +699,7 @@ void FlusherSLS::OnSendDone(const HttpResponse& response, SenderQueueItem* item) } } else { OperationOnFail operation; - SendResult sendResult = ConvertErrorCode(slsResponse.mErrorCode); + sendResult = ConvertErrorCode(slsResponse.mErrorCode); ostringstream failDetail, suggestion; if (sendResult == SEND_NETWORK_ERROR || sendResult == SEND_SERVER_ERROR) { if (sendResult == SEND_NETWORK_ERROR) { @@ -880,10 +881,11 @@ void FlusherSLS::OnSendDone(const HttpResponse& response, SenderQueueItem* item) break; } } - SLSClientManager::GetInstance()->UpdateAccessKeyStatus(mAliuid, !hasAuthError); #ifdef __ENTERPRISE__ - static auto manager = static_cast(SLSClientManager::GetInstance()); - manager->UpdateProjectAnonymousWriteStatus(mProject, !hasAuthError); + if (sendResult != SEND_NETWORK_ERROR && sendResult != SEND_SERVER_ERROR) { + EnterpriseSLSClientManager::GetInstance()->UpdateAccessKeyStatus(mAliuid, !hasAuthError); + EnterpriseSLSClientManager::GetInstance()->UpdateProjectAnonymousWriteStatus(mProject, !hasAuthError); + } #endif } diff --git a/core/plugin/flusher/sls/SLSClientManager.h b/core/plugin/flusher/sls/SLSClientManager.h index 98a1f81e88..a49f3123a9 100644 --- a/core/plugin/flusher/sls/SLSClientManager.h +++ b/core/plugin/flusher/sls/SLSClientManager.h @@ -43,7 +43,6 @@ class SLSClientManager { virtual bool GetAccessKey(const std::string& aliuid, AuthType& type, std::string& accessKeyId, std::string& accessKeySecret); - virtual void UpdateAccessKeyStatus(const std::string& aliuid, bool success) {} virtual bool UsingHttps(const std::string& region) const { return true; } From 3ca2ae73bedbb7efcb3e7c5bb06114fc28aa91ce Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Fri, 13 Dec 2024 09:27:35 +0000 Subject: [PATCH 15/41] polish --- core/unittest/flusher/FlusherSLSUnittest.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/unittest/flusher/FlusherSLSUnittest.cpp b/core/unittest/flusher/FlusherSLSUnittest.cpp index f402a093c0..6b6594dcbb 100644 --- a/core/unittest/flusher/FlusherSLSUnittest.cpp +++ b/core/unittest/flusher/FlusherSLSUnittest.cpp @@ -690,7 +690,9 @@ void FlusherSLSUnittest::TestBuildRequest() { bool keepItem = false; #ifdef __ENTERPRISE__ { - // empty ak + // empty ak, first try + APSARA_TEST_FALSE(flusher.BuildRequest(&item, req, &keepItem)); + // empty ak, second try APSARA_TEST_FALSE(flusher.BuildRequest(&item, req, &keepItem)); APSARA_TEST_EQUAL(nullptr, req); APSARA_TEST_TRUE(keepItem); From 598b5065a71e401c555fedf949c35d4f1fcfa8cf Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Fri, 13 Dec 2024 09:43:10 +0000 Subject: [PATCH 16/41] polish --- core/plugin/flusher/sls/DiskBufferWriter.cpp | 3 +-- core/plugin/flusher/sls/FlusherSLS.cpp | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/core/plugin/flusher/sls/DiskBufferWriter.cpp b/core/plugin/flusher/sls/DiskBufferWriter.cpp index 7197d6f59a..070d2b3b58 100644 --- a/core/plugin/flusher/sls/DiskBufferWriter.cpp +++ b/core/plugin/flusher/sls/DiskBufferWriter.cpp @@ -536,7 +536,6 @@ void DiskBufferWriter::SendEncryptionBuffer(const std::string& filename, int32_t string host; auto response = SendBufferFileData(bufferMeta, logData, host); auto sendRes = ConvertErrorCode(response.mErrorCode); - bool hasAuthError = false; switch (sendRes) { case SEND_OK: sendResult = true; @@ -568,7 +567,6 @@ void DiskBufferWriter::SendEncryptionBuffer(const std::string& filename, int32_t usleep(INT32_FLAG(quota_exceed_wait_interval)); break; case SEND_UNAUTHORIZED: - hasAuthError = true; usleep(INT32_FLAG(unauthorized_wait_interval)); break; default: @@ -578,6 +576,7 @@ void DiskBufferWriter::SendEncryptionBuffer(const std::string& filename, int32_t } #ifdef __ENTERPRISE__ if (sendRes != SEND_NETWORK_ERROR && sendRes != SEND_SERVER_ERROR) { + bool hasAuthError = sendRes == SEND_UNAUTHORIZED; EnterpriseSLSClientManager::GetInstance()->UpdateAccessKeyStatus(bufferMeta.aliuid(), !hasAuthError); EnterpriseSLSClientManager::GetInstance()->UpdateProjectAnonymousWriteStatus( diff --git a/core/plugin/flusher/sls/FlusherSLS.cpp b/core/plugin/flusher/sls/FlusherSLS.cpp index d08b258826..5eec77a7d2 100644 --- a/core/plugin/flusher/sls/FlusherSLS.cpp +++ b/core/plugin/flusher/sls/FlusherSLS.cpp @@ -670,7 +670,6 @@ void FlusherSLS::OnSendDone(const HttpResponse& response, SenderQueueItem* item) bool isProfileData = GetProfileSender()->IsProfileData(mRegion, mProject, data->mLogstore); int32_t curTime = time(NULL); auto curSystemTime = chrono::system_clock::now(); - bool hasAuthError = false; SendResult sendResult = SEND_OK; if (slsResponse.mStatusCode == 200) { auto& cpt = data->mExactlyOnceCheckpoint; @@ -758,7 +757,6 @@ void FlusherSLS::OnSendDone(const HttpResponse& response, SenderQueueItem* item) failDetail << "write unauthorized"; suggestion << "check access keys provided"; operation = OperationOnFail::RETRY_LATER; - hasAuthError = true; if (mUnauthErrorCnt) { mUnauthErrorCnt->Add(1); } @@ -883,6 +881,7 @@ void FlusherSLS::OnSendDone(const HttpResponse& response, SenderQueueItem* item) } #ifdef __ENTERPRISE__ if (sendResult != SEND_NETWORK_ERROR && sendResult != SEND_SERVER_ERROR) { + bool hasAuthError = sendResult == SEND_UNAUTHORIZED; EnterpriseSLSClientManager::GetInstance()->UpdateAccessKeyStatus(mAliuid, !hasAuthError); EnterpriseSLSClientManager::GetInstance()->UpdateProjectAnonymousWriteStatus(mProject, !hasAuthError); } From 3bf8f084172adbc5fdf17168936bea224d8505fa Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Tue, 17 Dec 2024 06:12:29 +0000 Subject: [PATCH 17/41] polish --- core/plugin/flusher/sls/FlusherSLS.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/core/plugin/flusher/sls/FlusherSLS.cpp b/core/plugin/flusher/sls/FlusherSLS.cpp index 5eec77a7d2..38e73c8a01 100644 --- a/core/plugin/flusher/sls/FlusherSLS.cpp +++ b/core/plugin/flusher/sls/FlusherSLS.cpp @@ -832,6 +832,13 @@ void FlusherSLS::OnSendDone(const HttpResponse& response, SenderQueueItem* item) if (isProfileData && data->mTryCnt >= static_cast(INT32_FLAG(profile_data_send_retrytimes))) { operation = OperationOnFail::DISCARD; } +#ifdef __ENTERPRISE__ + if (sendResult != SEND_NETWORK_ERROR && sendResult != SEND_SERVER_ERROR) { + bool hasAuthError = sendResult == SEND_UNAUTHORIZED; + EnterpriseSLSClientManager::GetInstance()->UpdateAccessKeyStatus(mAliuid, !hasAuthError); + EnterpriseSLSClientManager::GetInstance()->UpdateProjectAnonymousWriteStatus(mProject, !hasAuthError); + } +#endif #define LOG_PATTERN \ ("failed to send request", failDetail.str())("operation", GetOperationString(operation))("suggestion", \ @@ -879,13 +886,6 @@ void FlusherSLS::OnSendDone(const HttpResponse& response, SenderQueueItem* item) break; } } -#ifdef __ENTERPRISE__ - if (sendResult != SEND_NETWORK_ERROR && sendResult != SEND_SERVER_ERROR) { - bool hasAuthError = sendResult == SEND_UNAUTHORIZED; - EnterpriseSLSClientManager::GetInstance()->UpdateAccessKeyStatus(mAliuid, !hasAuthError); - EnterpriseSLSClientManager::GetInstance()->UpdateProjectAnonymousWriteStatus(mProject, !hasAuthError); - } -#endif } bool FlusherSLS::Send(string&& data, const string& shardHashKey, const string& logstore) { From 6c0c5dea3795805fa5c19c29e96afa1d884522e8 Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Thu, 19 Dec 2024 11:16:23 +0000 Subject: [PATCH 18/41] polish --- core/common/http/Curl.cpp | 4 ++++ core/common/http/HttpResponse.h | 26 +++++++++++--------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/core/common/http/Curl.cpp b/core/common/http/Curl.cpp index 437f5ba9ae..ffc30d5147 100644 --- a/core/common/http/Curl.cpp +++ b/core/common/http/Curl.cpp @@ -211,6 +211,7 @@ bool AddRequestToMultiCurlHandler(CURLM* multiCurl, unique_ptr& request->mTls); if (curl == NULL) { LOG_ERROR(sLogger, ("failed to send request", "failed to init curl handler")("request address", request.get())); + request->mResponse.SetNetworkStatus(CURLE_FAILED_INIT); request->OnSendDone(request->mResponse); return false; } @@ -223,6 +224,7 @@ bool AddRequestToMultiCurlHandler(CURLM* multiCurl, unique_ptr& LOG_ERROR(sLogger, ("failed to send request", "failed to add the easy curl handle to multi_handle")( "errMsg", curl_multi_strerror(res))("request address", request.get())); + request->mResponse.SetNetworkStatus(CURLE_FAILED_INIT); request->OnSendDone(request->mResponse); curl_easy_cleanup(curl); return false; @@ -247,6 +249,7 @@ void HandleCompletedAsynRequests(CURLM* multiCurl, int& runningHandlers) { case CURLE_OK: { long statusCode = 0; curl_easy_getinfo(handler, CURLINFO_RESPONSE_CODE, &statusCode); + request->mResponse.SetNetworkStatus(CURLE_OK); request->mResponse.SetStatusCode(statusCode); request->mResponse.SetResponseTime(responseTimeMs); request->OnSendDone(request->mResponse); @@ -272,6 +275,7 @@ void HandleCompletedAsynRequests(CURLM* multiCurl, int& runningHandlers) { ++runningHandlers; requestReused = true; } else { + request->mResponse.SetNetworkStatus(msg->data.result); request->OnSendDone(request->mResponse); LOG_DEBUG(sLogger, ("failed to send http request", "abort")("request address", request)( diff --git a/core/common/http/HttpResponse.h b/core/common/http/HttpResponse.h index 589e6dc6af..1518e5b0d4 100644 --- a/core/common/http/HttpResponse.h +++ b/core/common/http/HttpResponse.h @@ -18,8 +18,6 @@ #include -#include - #include #include #include @@ -85,6 +83,8 @@ class HttpResponse { : mHeader(compareHeader), mBody(body, bodyDeleter), mWriteCallback(callback) {} int32_t GetStatusCode() const { return mStatusCode; } + void SetStatusCode(int32_t code) { mStatusCode = code; } + const std::map& GetHeader() const { return mHeader; } template @@ -97,19 +97,10 @@ class HttpResponse { return static_cast(mBody.get()); } - void SetStatusCode(int32_t code) { mStatusCode = code; } void SetResponseTime(const std::chrono::milliseconds& time) { mResponseTime = time; } std::chrono::milliseconds GetResponseTime() const { return mResponseTime; } -#ifdef APSARA_UNIT_TEST_MAIN - template - void SetBody(const T& body) { - *mBody = body; - } - - void AddHeader(const std::string& key, const std::string& value) { mHeader[key] = value; } -#endif - + NetworkStatus GetNetworkStatus() { return mNetworkStatus; } void SetNetworkStatus(CURLcode code) { mNetworkStatus.mMessage = curl_easy_strerror(code); // please refer to https://curl.se/libcurl/c/libcurl-errors.html @@ -161,13 +152,18 @@ class HttpResponse { } } - const NetworkStatus& GetNetworkStatus() { return mNetworkStatus; } +#ifdef APSARA_UNIT_TEST_MAIN + template + void SetBody(const T& body) { + *mBody = body; + } - + void AddHeader(const std::string& key, const std::string& value) { mHeader[key] = value; } +#endif private: int32_t mStatusCode = 0; // 0 means no response from server - NetworkStatus mNetworkStatus; // 0 means no error + NetworkStatus mNetworkStatus; std::map mHeader; std::unique_ptr> mBody; size_t (*mWriteCallback)(char*, size_t, size_t, void*) = nullptr; From 0f9d6b0f30771ef33c3ff215bbee165516f5a0cc Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Thu, 19 Dec 2024 11:33:31 +0000 Subject: [PATCH 19/41] polish --- core/common/http/Curl.cpp | 53 +++++++++++++++++++++++++++---- core/common/http/Curl.h | 2 ++ core/common/http/HttpResponse.h | 56 +++------------------------------ 3 files changed, 53 insertions(+), 58 deletions(-) diff --git a/core/common/http/Curl.cpp b/core/common/http/Curl.cpp index ffc30d5147..f8d9c2e1c7 100644 --- a/core/common/http/Curl.cpp +++ b/core/common/http/Curl.cpp @@ -28,6 +28,46 @@ using namespace std; namespace logtail { +NetworkCode GetNetworkStatus(CURLcode code) { + // please refer to https://curl.se/libcurl/c/libcurl-errors.html + switch (code) { + case CURLE_OK: + return NetworkCode::Ok; + case CURLE_COULDNT_CONNECT: + return NetworkCode::ConnectionFailed; + case CURLE_LOGIN_DENIED: + case CURLE_REMOTE_ACCESS_DENIED: + return NetworkCode::RemoteAccessDenied; + case CURLE_OPERATION_TIMEDOUT: + return NetworkCode::Timeout; + case CURLE_SSL_CONNECT_ERROR: + return NetworkCode::SSLConnectError; + case CURLE_SSL_CERTPROBLEM: + case CURLE_SSL_CACERT: + return NetworkCode::SSLCertError; + case CURLE_SEND_ERROR: + case CURLE_SEND_FAIL_REWIND: + return NetworkCode::SendDataFailed; + case CURLE_RECV_ERROR: + return NetworkCode::RecvDataFailed; + case CURLE_SSL_PINNEDPUBKEYNOTMATCH: + case CURLE_SSL_INVALIDCERTSTATUS: + case CURLE_SSL_CACERT_BADFILE: + case CURLE_SSL_CIPHER: + case CURLE_SSL_ENGINE_NOTFOUND: + case CURLE_SSL_ENGINE_SETFAILED: + case CURLE_USE_SSL_FAILED: + case CURLE_SSL_ENGINE_INITFAILED: + case CURLE_SSL_CRL_BADFILE: + case CURLE_SSL_ISSUER_ERROR: + case CURLE_SSL_SHUTDOWN_FAILED: + return NetworkCode::SSLOtherProblem; + case CURLE_FAILED_INIT: + default: + return NetworkCode::Other; + } +} + static size_t header_write_callback(char* buffer, size_t size, size_t nmemb, @@ -211,7 +251,7 @@ bool AddRequestToMultiCurlHandler(CURLM* multiCurl, unique_ptr& request->mTls); if (curl == NULL) { LOG_ERROR(sLogger, ("failed to send request", "failed to init curl handler")("request address", request.get())); - request->mResponse.SetNetworkStatus(CURLE_FAILED_INIT); + request->mResponse.SetNetworkStatus(NetworkCode::Other, "failed to init curl handler"); request->OnSendDone(request->mResponse); return false; } @@ -224,7 +264,7 @@ bool AddRequestToMultiCurlHandler(CURLM* multiCurl, unique_ptr& LOG_ERROR(sLogger, ("failed to send request", "failed to add the easy curl handle to multi_handle")( "errMsg", curl_multi_strerror(res))("request address", request.get())); - request->mResponse.SetNetworkStatus(CURLE_FAILED_INIT); + request->mResponse.SetNetworkStatus(NetworkCode::Other, "failed to add the easy curl handle to multi_handle"); request->OnSendDone(request->mResponse); curl_easy_cleanup(curl); return false; @@ -249,7 +289,7 @@ void HandleCompletedAsynRequests(CURLM* multiCurl, int& runningHandlers) { case CURLE_OK: { long statusCode = 0; curl_easy_getinfo(handler, CURLINFO_RESPONSE_CODE, &statusCode); - request->mResponse.SetNetworkStatus(CURLE_OK); + request->mResponse.SetNetworkStatus(NetworkCode::Ok, ""); request->mResponse.SetStatusCode(statusCode); request->mResponse.SetResponseTime(responseTimeMs); request->OnSendDone(request->mResponse); @@ -275,12 +315,13 @@ void HandleCompletedAsynRequests(CURLM* multiCurl, int& runningHandlers) { ++runningHandlers; requestReused = true; } else { - request->mResponse.SetNetworkStatus(msg->data.result); + auto errMsg = curl_easy_strerror(msg->data.result); + request->mResponse.SetNetworkStatus(GetNetworkStatus(msg->data.result), errMsg); request->OnSendDone(request->mResponse); LOG_DEBUG(sLogger, ("failed to send http request", "abort")("request address", request)( - "response time", - ToString(responseTimeMs.count()) + "ms")("try cnt", ToString(request->mTryCnt))); + "response time", ToString(responseTimeMs.count()) + "ms")( + "try cnt", ToString(request->mTryCnt))("errMsg", errMsg)); } break; } diff --git a/core/common/http/Curl.h b/core/common/http/Curl.h index 4d4fdd54c9..ad21f876e8 100644 --- a/core/common/http/Curl.h +++ b/core/common/http/Curl.h @@ -28,6 +28,8 @@ namespace logtail { +NetworkCode GetNetworkStatus(CURLcode code); + CURL* CreateCurlHandler(const std::string& method, bool httpsFlag, const std::string& host, diff --git a/core/common/http/HttpResponse.h b/core/common/http/HttpResponse.h index 1518e5b0d4..8cc4212673 100644 --- a/core/common/http/HttpResponse.h +++ b/core/common/http/HttpResponse.h @@ -16,8 +16,6 @@ #pragma once -#include - #include #include #include @@ -100,56 +98,10 @@ class HttpResponse { void SetResponseTime(const std::chrono::milliseconds& time) { mResponseTime = time; } std::chrono::milliseconds GetResponseTime() const { return mResponseTime; } - NetworkStatus GetNetworkStatus() { return mNetworkStatus; } - void SetNetworkStatus(CURLcode code) { - mNetworkStatus.mMessage = curl_easy_strerror(code); - // please refer to https://curl.se/libcurl/c/libcurl-errors.html - switch (code) { - case CURLE_OK: - mNetworkStatus.mCode = NetworkCode::Ok; - break; - case CURLE_COULDNT_CONNECT: - mNetworkStatus.mCode = NetworkCode::ConnectionFailed; - break; - case CURLE_LOGIN_DENIED: - case CURLE_REMOTE_ACCESS_DENIED: - mNetworkStatus.mCode = NetworkCode::RemoteAccessDenied; - break; - case CURLE_OPERATION_TIMEDOUT: - mNetworkStatus.mCode = NetworkCode::Timeout; - break; - case CURLE_SSL_CONNECT_ERROR: - mNetworkStatus.mCode = NetworkCode::SSLConnectError; - break; - case CURLE_SSL_CERTPROBLEM: - case CURLE_SSL_CACERT: - mNetworkStatus.mCode = NetworkCode::SSLCertError; - break; - case CURLE_SEND_ERROR: - case CURLE_SEND_FAIL_REWIND: - mNetworkStatus.mCode = NetworkCode::SendDataFailed; - break; - case CURLE_RECV_ERROR: - mNetworkStatus.mCode = NetworkCode::RecvDataFailed; - break; - case CURLE_SSL_PINNEDPUBKEYNOTMATCH: - case CURLE_SSL_INVALIDCERTSTATUS: - case CURLE_SSL_CACERT_BADFILE: - case CURLE_SSL_CIPHER: - case CURLE_SSL_ENGINE_NOTFOUND: - case CURLE_SSL_ENGINE_SETFAILED: - case CURLE_USE_SSL_FAILED: - case CURLE_SSL_ENGINE_INITFAILED: - case CURLE_SSL_CRL_BADFILE: - case CURLE_SSL_ISSUER_ERROR: - case CURLE_SSL_SHUTDOWN_FAILED: - mNetworkStatus.mCode = NetworkCode::SSLOtherProblem; - break; - case CURLE_FAILED_INIT: - default: - mNetworkStatus.mCode = NetworkCode::Other; - break; - } + const NetworkStatus& GetNetworkStatus() { return mNetworkStatus; } + void SetNetworkStatus(NetworkCode code, const std::string& msg) { + mNetworkStatus.mCode = code; + mNetworkStatus.mMessage = msg; } #ifdef APSARA_UNIT_TEST_MAIN From 5d37ced401c81caad9faf9c5928392e609a371c8 Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Thu, 19 Dec 2024 11:39:10 +0000 Subject: [PATCH 20/41] polish --- core/application/Application.cpp | 28 ++++++---- core/common/http/Curl.cpp | 15 +++--- core/config/watcher/PipelineConfigWatcher.cpp | 2 +- core/plugin/flusher/sls/DiskBufferWriter.cpp | 51 +++++++++++-------- core/plugin/flusher/sls/FlusherSLS.cpp | 15 +++--- core/plugin/flusher/sls/FlusherSLS.h | 3 +- 6 files changed, 66 insertions(+), 48 deletions(-) diff --git a/core/application/Application.cpp b/core/application/Application.cpp index a000020970..a32acb08a8 100644 --- a/core/application/Application.cpp +++ b/core/application/Application.cpp @@ -46,6 +46,7 @@ #include "pipeline/queue/ExactlyOnceQueueManager.h" #include "pipeline/queue/SenderQueueManager.h" #include "plugin/flusher/sls/DiskBufferWriter.h" +#include "plugin/flusher/sls/FlusherSLS.h" #include "plugin/input/InputFeedbackInterfaceRegistry.h" #include "prometheus/PrometheusInputRunner.h" #include "runner/FlusherRunner.h" @@ -196,11 +197,13 @@ void Application::Start() { // GCOVR_EXCL_START #if defined(__ENTERPRISE__) && defined(_MSC_VER) InitWindowsSignalObject(); #endif - BoundedSenderQueueInterface::SetFeedback(ProcessQueueManager::GetInstance()); - HttpSink::GetInstance()->Init(); - FlusherRunner::GetInstance()->Init(); + // resource monitor + // TODO: move metric related initialization to input Init + LoongCollectorMonitor::GetInstance()->Init(); + LogtailMonitor::GetInstance()->Init(); + // config provider { // add local config dir filesystem::path localConfigPath = filesystem::path(AppConfig::GetInstance()->GetLoongcollectorConfDir()) @@ -214,7 +217,6 @@ void Application::Start() { // GCOVR_EXCL_START } PipelineConfigWatcher::GetInstance()->AddSource(localConfigPath.string()); } - #ifdef __ENTERPRISE__ EnterpriseConfigProvider::GetInstance()->Start(); LegacyConfigProvider::GetInstance()->Init("legacy"); @@ -222,10 +224,16 @@ void Application::Start() { // GCOVR_EXCL_START InitRemoteConfigProviders(); #endif - AlarmManager::GetInstance()->Init(); - LoongCollectorMonitor::GetInstance()->Init(); - LogtailMonitor::GetInstance()->Init(); + // runner + BoundedSenderQueueInterface::SetFeedback(ProcessQueueManager::GetInstance()); + HttpSink::GetInstance()->Init(); + FlusherRunner::GetInstance()->Init(); + ProcessorRunner::GetInstance()->Init(); + + // flusher_sls resource should be explicitly initialized to allow internal metrics and alarms to be sent + FlusherSLS::InitResource(); + // plugin registration PluginRegistry::GetInstance()->LoadPlugins(); InputFeedbackInterfaceRegistry::GetInstance()->LoadFeedbackInterfaces(); @@ -255,10 +263,10 @@ void Application::Start() { // GCOVR_EXCL_START LogtailPlugin::GetInstance()->LoadPluginBase(); } - ProcessorRunner::GetInstance()->Init(); + // TODO: this should be refactored to internal pipeline + AlarmManager::GetInstance()->Init(); - time_t curTime = 0, lastConfigCheckTime = 0, lastUpdateMetricTime = 0, - lastCheckTagsTime = 0, lastQueueGCTime = 0; + time_t curTime = 0, lastConfigCheckTime = 0, lastUpdateMetricTime = 0, lastCheckTagsTime = 0, lastQueueGCTime = 0; #ifndef LOGTAIL_NO_TC_MALLOC time_t lastTcmallocReleaseMemTime = 0; #endif diff --git a/core/common/http/Curl.cpp b/core/common/http/Curl.cpp index f8d9c2e1c7..4a841b046e 100644 --- a/core/common/http/Curl.cpp +++ b/core/common/http/Curl.cpp @@ -219,7 +219,7 @@ bool SendHttpRequest(unique_ptr&& request, HttpResponse& response) } else if (request->mTryCnt < request->mMaxTryCnt) { LOG_WARNING(sLogger, ("failed to send http request", "retry immediately")("request address", request.get())( - "try cnt", request->mTryCnt)("errMsg", curl_easy_strerror(res))); + "host", request->mHost)("try cnt", request->mTryCnt)("errMsg", curl_easy_strerror(res))); ++request->mTryCnt; } else { break; @@ -294,16 +294,17 @@ void HandleCompletedAsynRequests(CURLM* multiCurl, int& runningHandlers) { request->mResponse.SetResponseTime(responseTimeMs); request->OnSendDone(request->mResponse); LOG_DEBUG(sLogger, - ("send http request succeeded, request address", - request)("response time", ToString(responseTimeMs.count()) + "ms")( - "try cnt", ToString(request->mTryCnt))); + ("send http request succeeded, request address", request)("host", request->mHost)( + "response time", + ToString(responseTimeMs.count()) + "ms")("try cnt", ToString(request->mTryCnt))); break; } default: // considered as network error if (request->mTryCnt < request->mMaxTryCnt) { - LOG_WARNING(sLogger, - ("failed to send http request", "retry immediately")("request address", request)( + LOG_DEBUG(sLogger, + ("failed to send http request", + "retry immediately")("request address", request)("host", request->mHost)( "try cnt", request->mTryCnt)("errMsg", curl_easy_strerror(msg->data.result))); // free first，becase mPrivateData will be reset in AddRequestToMultiCurlHandler if (request->mPrivateData) { @@ -320,7 +321,7 @@ void HandleCompletedAsynRequests(CURLM* multiCurl, int& runningHandlers) { request->OnSendDone(request->mResponse); LOG_DEBUG(sLogger, ("failed to send http request", "abort")("request address", request)( - "response time", ToString(responseTimeMs.count()) + "ms")( + "host", request->mHost)("response time", ToString(responseTimeMs.count()) + "ms")( "try cnt", ToString(request->mTryCnt))("errMsg", errMsg)); } break; diff --git a/core/config/watcher/PipelineConfigWatcher.cpp b/core/config/watcher/PipelineConfigWatcher.cpp index fb914c4d55..0cd8c00570 100644 --- a/core/config/watcher/PipelineConfigWatcher.cpp +++ b/core/config/watcher/PipelineConfigWatcher.cpp @@ -440,7 +440,7 @@ bool PipelineConfigWatcher::CheckUnchangedConfig(const std::string& configName, return false; } if (!IsConfigEnabled(configName, *detail)) { - LOG_INFO(sLogger, ("unchanged config found and disabled", "skip current object")("config", configName)); + LOG_DEBUG(sLogger, ("unchanged config found and disabled", "skip current object")("config", configName)); return false; } PipelineConfig config(configName, std::move(detail)); diff --git a/core/plugin/flusher/sls/DiskBufferWriter.cpp b/core/plugin/flusher/sls/DiskBufferWriter.cpp index 070d2b3b58..1d9d202802 100644 --- a/core/plugin/flusher/sls/DiskBufferWriter.cpp +++ b/core/plugin/flusher/sls/DiskBufferWriter.cpp @@ -535,18 +535,24 @@ void DiskBufferWriter::SendEncryptionBuffer(const std::string& filename, int32_t while (true) { string host; auto response = SendBufferFileData(bufferMeta, logData, host); - auto sendRes = ConvertErrorCode(response.mErrorCode); + SendResult sendRes = SEND_OK; + if (response.mStatusCode != 200) { + sendRes = ConvertErrorCode(response.mErrorCode); + } switch (sendRes) { case SEND_OK: sendResult = true; break; case SEND_NETWORK_ERROR: case SEND_SERVER_ERROR: - LOG_WARNING(sLogger, - ("send data to SLS fail", "retry later")("error_code", response.mErrorCode)( - "error_message", response.mErrorMsg)("endpoint", host)( - "projectName", bufferMeta.project())("logstore", bufferMeta.logstore())( - "rawsize", bufferMeta.rawsize())); + if (response.mErrorMsg != "can not get available host") { + LOG_WARNING( + sLogger, + ("send data to SLS fail", "retry later")("request id", response.mRequestId)( + "error_code", response.mErrorCode)("error_message", response.mErrorMsg)( + "endpoint", host)("projectName", bufferMeta.project())( + "logstore", bufferMeta.logstore())("rawsize", bufferMeta.rawsize())); + } usleep(INT32_FLAG(send_retry_sleep_interval)); break; case SEND_QUOTA_EXCEED: @@ -560,10 +566,10 @@ void DiskBufferWriter::SendEncryptionBuffer(const std::string& filename, int32_t if (!GetProfileSender()->IsProfileData("", bufferMeta.project(), bufferMeta.logstore())) LOG_WARNING( sLogger, - ("send data to SLS fail, error_code", - response.mErrorCode)("error_message", response.mErrorMsg)("endpoint", host)( - "projectName", bufferMeta.project())("logstore", bufferMeta.logstore())( - "rawsize", bufferMeta.rawsize())); + ("send data to SLS fail", "retry later")("request id", response.mRequestId)( + "error_code", response.mErrorCode)("error_message", response.mErrorMsg)( + "endpoint", host)("projectName", bufferMeta.project())( + "logstore", bufferMeta.logstore())("rawsize", bufferMeta.rawsize())); usleep(INT32_FLAG(quota_exceed_wait_interval)); break; case SEND_UNAUTHORIZED: @@ -576,7 +582,8 @@ void DiskBufferWriter::SendEncryptionBuffer(const std::string& filename, int32_t } #ifdef __ENTERPRISE__ if (sendRes != SEND_NETWORK_ERROR && sendRes != SEND_SERVER_ERROR) { - bool hasAuthError = sendRes == SEND_UNAUTHORIZED; + bool hasAuthError + = sendRes == SEND_UNAUTHORIZED && response.mErrorMsg != "can not get valid access key"; EnterpriseSLSClientManager::GetInstance()->UpdateAccessKeyStatus(bufferMeta.aliuid(), !hasAuthError); EnterpriseSLSClientManager::GetInstance()->UpdateProjectAnonymousWriteStatus( @@ -857,6 +864,17 @@ SLSResponse DiskBufferWriter::SendBufferFileData(const sls_logs::LogtailBufferMe dataType = RawDataType::EVENT_GROUP; } if (bufferMeta.has_telemetrytype() && bufferMeta.telemetrytype() == sls_logs::SLS_TELEMETRY_TYPE_METRICS) { + return PostMetricStoreLogs(accessKeyId, + accessKeySecret, + type, + host, + httpsFlag, + bufferMeta.project(), + bufferMeta.logstore(), + GetSLSCompressTypeString(bufferMeta.compresstype()), + logData, + bufferMeta.rawsize()); + } else { return PostLogStoreLogs(accessKeyId, accessKeySecret, type, @@ -869,17 +887,6 @@ SLSResponse DiskBufferWriter::SendBufferFileData(const sls_logs::LogtailBufferMe logData, bufferMeta.rawsize(), bufferMeta.has_shardhashkey() ? bufferMeta.shardhashkey() : ""); - } else { - return PostMetricStoreLogs(accessKeyId, - accessKeySecret, - type, - host, - httpsFlag, - bufferMeta.project(), - bufferMeta.logstore(), - GetSLSCompressTypeString(bufferMeta.compresstype()), - logData, - bufferMeta.rawsize()); } } diff --git a/core/plugin/flusher/sls/FlusherSLS.cpp b/core/plugin/flusher/sls/FlusherSLS.cpp index 5d963e65dd..e0980a6137 100644 --- a/core/plugin/flusher/sls/FlusherSLS.cpp +++ b/core/plugin/flusher/sls/FlusherSLS.cpp @@ -550,7 +550,6 @@ bool FlusherSLS::Init(const Json::Value& config, Json::Value& optionalGoPipeline bool FlusherSLS::Start() { Flusher::Start(); - InitResource(); IncreaseProjectRegionReferenceCnt(mProject, mRegion); return true; @@ -625,6 +624,10 @@ bool FlusherSLS::BuildRequest(SenderQueueItem* item, unique_ptr } else { // in case local region endpoint mode is changed, we should always check before sending auto info = EnterpriseSLSClientManager::GetInstance()->GetCandidateHostsInfo(mRegion, mProject, mEndpointMode); + if (mCandidateHostsInfo == nullptr) { + // TODO: temporarily used here, for send logtail alarm only, should be removed after alarm is refactored + mCandidateHostsInfo = info; + } if (mCandidateHostsInfo.get() != info.get()) { LOG_INFO(sLogger, ("update candidate hosts info, region", mRegion)("project", mProject)("logstore", mLogstore)( @@ -833,11 +836,11 @@ void FlusherSLS::OnSendDone(const HttpResponse& response, SenderQueueItem* item) operation = OperationOnFail::DISCARD; } #ifdef __ENTERPRISE__ - if (sendResult != SEND_NETWORK_ERROR && sendResult != SEND_SERVER_ERROR) { - bool hasAuthError = sendResult == SEND_UNAUTHORIZED; - EnterpriseSLSClientManager::GetInstance()->UpdateAccessKeyStatus(mAliuid, !hasAuthError); - EnterpriseSLSClientManager::GetInstance()->UpdateProjectAnonymousWriteStatus(mProject, !hasAuthError); - } + if (sendResult != SEND_NETWORK_ERROR && sendResult != SEND_SERVER_ERROR) { + bool hasAuthError = sendResult == SEND_UNAUTHORIZED; + EnterpriseSLSClientManager::GetInstance()->UpdateAccessKeyStatus(mAliuid, !hasAuthError); + EnterpriseSLSClientManager::GetInstance()->UpdateProjectAnonymousWriteStatus(mProject, !hasAuthError); + } #endif #define LOG_PATTERN \ diff --git a/core/plugin/flusher/sls/FlusherSLS.h b/core/plugin/flusher/sls/FlusherSLS.h index b3d8292576..7fa4610056 100644 --- a/core/plugin/flusher/sls/FlusherSLS.h +++ b/core/plugin/flusher/sls/FlusherSLS.h @@ -48,6 +48,7 @@ class FlusherSLS : public HttpFlusher { static std::shared_ptr GetRegionConcurrencyLimiter(const std::string& region); static void ClearInvalidConcurrencyLimiters(); + static void InitResource(); static void RecycleResourceIfNotUsed(); static std::string GetDefaultRegion(); @@ -90,8 +91,6 @@ class FlusherSLS : public HttpFlusher { std::unique_ptr mCompressor; private: - static void InitResource(); - static void IncreaseProjectRegionReferenceCnt(const std::string& project, const std::string& region); static void DecreaseProjectRegionReferenceCnt(const std::string& project, const std::string& region); From f7e4f83bca605cc8e3371cf32bb718e086c59c3d Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Fri, 20 Dec 2024 02:07:31 +0000 Subject: [PATCH 21/41] polish --- core/unittest/common/CMakeLists.txt | 5 ---- .../common/http/HttpResponseUnittest.cpp | 27 ------------------- .../prometheus/ScrapeSchedulerUnittest.cpp | 10 +++---- 3 files changed, 5 insertions(+), 37 deletions(-) delete mode 100644 core/unittest/common/http/HttpResponseUnittest.cpp diff --git a/core/unittest/common/CMakeLists.txt b/core/unittest/common/CMakeLists.txt index 6ddd095c73..80de5c490b 100644 --- a/core/unittest/common/CMakeLists.txt +++ b/core/unittest/common/CMakeLists.txt @@ -54,9 +54,6 @@ target_link_libraries(timer_unittest ${UT_BASE_TARGET}) add_executable(curl_unittest http/CurlUnittest.cpp) target_link_libraries(curl_unittest ${UT_BASE_TARGET}) -add_executable(http_response_unittest http/HttpResponseUnittest.cpp) -target_link_libraries(http_response_unittest ${UT_BASE_TARGET}) - include(GoogleTest) gtest_discover_tests(common_simple_utils_unittest) gtest_discover_tests(common_logfileoperator_unittest) @@ -69,5 +66,3 @@ gtest_discover_tests(safe_queue_unittest) gtest_discover_tests(http_request_timer_event_unittest) gtest_discover_tests(timer_unittest) gtest_discover_tests(curl_unittest) -gtest_discover_tests(http_response_unittest) - diff --git a/core/unittest/common/http/HttpResponseUnittest.cpp b/core/unittest/common/http/HttpResponseUnittest.cpp deleted file mode 100644 index 2df46b7d86..0000000000 --- a/core/unittest/common/http/HttpResponseUnittest.cpp +++ /dev/null @@ -1,27 +0,0 @@ - -#include "common/http/HttpResponse.h" -#include "unittest/Unittest.h" - -using namespace std; - -namespace logtail { -class HttpResponseUnittest : public ::testing::Test { -public: - void TestNetworkStatus(); -}; - -void HttpResponseUnittest::TestNetworkStatus() { - HttpResponse resp; - resp.SetNetworkStatus(CURLE_OK); - APSARA_TEST_EQUAL(resp.GetNetworkStatus().mCode, NetworkCode::Ok); - - resp.SetNetworkStatus(CURLE_RECV_ERROR); - APSARA_TEST_EQUAL(resp.GetNetworkStatus().mCode, NetworkCode::RecvDataFailed); - - resp.SetNetworkStatus(CURLE_FAILED_INIT); - APSARA_TEST_EQUAL(resp.GetNetworkStatus().mCode, NetworkCode::Other); -} - -UNIT_TEST_CASE(HttpResponseUnittest, TestNetworkStatus); -} // namespace logtail -UNIT_TEST_MAIN diff --git a/core/unittest/prometheus/ScrapeSchedulerUnittest.cpp b/core/unittest/prometheus/ScrapeSchedulerUnittest.cpp index a8d696db4d..282a8eba0f 100644 --- a/core/unittest/prometheus/ScrapeSchedulerUnittest.cpp +++ b/core/unittest/prometheus/ScrapeSchedulerUnittest.cpp @@ -14,11 +14,11 @@ * limitations under the License. */ - #include #include #include "common/StringTools.h" +#include "common/http/Curl.h" #include "common/http/HttpResponse.h" #include "common/timer/Timer.h" #include "models/RawEvent.h" @@ -81,14 +81,14 @@ void ScrapeSchedulerUnittest::TestProcess() { // if status code is not 200, no data will be processed // but will continue running, sending self-monitoring metrics httpResponse.SetStatusCode(503); - httpResponse.SetNetworkStatus(CURLE_OK); + httpResponse.SetNetworkStatus(NetworkCode::Ok, ""); event.OnMetricResult(httpResponse, 0); APSARA_TEST_EQUAL(1UL, event.mItem.size()); event.mItem.clear(); httpResponse.GetBody()->mEventGroup = PipelineEventGroup(std::make_shared()); httpResponse.SetStatusCode(503); - httpResponse.SetNetworkStatus(CURLE_COULDNT_CONNECT); + httpResponse.SetNetworkStatus(GetNetworkStatus(CURLE_COULDNT_CONNECT), ""); event.OnMetricResult(httpResponse, 0); APSARA_TEST_EQUAL(event.mScrapeState, "ERR_CONN_FAILED"); APSARA_TEST_EQUAL(1UL, event.mItem.size()); @@ -96,7 +96,7 @@ void ScrapeSchedulerUnittest::TestProcess() { httpResponse.GetBody()->mEventGroup = PipelineEventGroup(std::make_shared()); httpResponse.SetStatusCode(200); - httpResponse.SetNetworkStatus(CURLE_OK); + httpResponse.SetNetworkStatus(NetworkCode::Ok, ""); string body1 = "# HELP go_gc_duration_seconds A summary of the pause duration of garbage collection cycles.\n" "# TYPE go_gc_duration_seconds summary\n" "go_gc_duration_seconds{quantile=\"0\"} 1.5531e-05\n" @@ -285,4 +285,4 @@ UNIT_TEST_CASE(ScrapeSchedulerUnittest, TestExactlyScrape) } // namespace logtail -UNIT_TEST_MAIN \ No newline at end of file +UNIT_TEST_MAIN From 019045ca92ac593ece91ad9cdf0e3cd39a7ede66 Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Mon, 23 Dec 2024 02:03:24 +0000 Subject: [PATCH 22/41] polish --- core/common/DNSCache.cpp | 144 +++++++++++++++-------------- core/common/DNSCache.h | 7 +- core/common/http/Curl.cpp | 57 +++++++----- core/runner/sink/http/HttpSink.cpp | 23 +++-- 4 files changed, 131 insertions(+), 100 deletions(-) diff --git a/core/common/DNSCache.cpp b/core/common/DNSCache.cpp index 8c1ce33f76..a5fc00480f 100644 --- a/core/common/DNSCache.cpp +++ b/core/common/DNSCache.cpp @@ -13,6 +13,7 @@ // limitations under the License. #include "DNSCache.h" + #include #if defined(__linux__) #include @@ -22,86 +23,91 @@ #include #endif +DEFINE_FLAG_INT32(dns_cache_ttl_sec, "", 600); + namespace logtail { - // ParseHost only supports IPv4 now. - bool DnsCache::ParseHost(const char* host, std::string& ip) { +DnsCache::DnsCache(const int32_t ttlSeconds) : mUpdateTime(time(NULL)), mDnsTTL(ttlSeconds) { +} + +// ParseHost only supports IPv4 now. +bool DnsCache::ParseHost(const char* host, std::string& ip) { #if defined(__linux__) - struct sockaddr_in addr; - memset(&addr, 0, sizeof(addr)); - addr.sin_family = AF_INET; + struct sockaddr_in addr; + memset(&addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; - char* buffer = NULL; - if (host && host[0]) { - if (IsRawIp(host)) { - if ((addr.sin_addr.s_addr = inet_addr(host)) == INADDR_NONE) - return false; - } else { - int bufferLen = 2048; - int rc, res; - struct hostent* hp = NULL; - struct hostent h; - while (true) { - buffer = new char[bufferLen]; - res = gethostbyname_r(host, &h, buffer, bufferLen, &hp, &rc); - if (res == ERANGE) { - if (buffer != NULL) - delete[] buffer; - bufferLen *= 4; - if (bufferLen > 32768) // 32KB - return false; - continue; - } - if (res != 0 || hp == NULL || hp->h_addr == NULL) { - if (buffer != NULL) - delete[] buffer; + char* buffer = NULL; + if (host && host[0]) { + if (IsRawIp(host)) { + if ((addr.sin_addr.s_addr = inet_addr(host)) == INADDR_NONE) + return false; + } else { + int bufferLen = 2048; + int rc, res; + struct hostent* hp = NULL; + struct hostent h; + while (true) { + buffer = new char[bufferLen]; + res = gethostbyname_r(host, &h, buffer, bufferLen, &hp, &rc); + if (res == ERANGE) { + if (buffer != NULL) + delete[] buffer; + bufferLen *= 4; + if (bufferLen > 32768) // 32KB return false; - } else - break; + continue; } - addr.sin_addr.s_addr = *((in_addr_t*)(hp->h_addr)); + if (res != 0 || hp == NULL || hp->h_addr == NULL) { + if (buffer != NULL) + delete[] buffer; + return false; + } else + break; } - } else { - addr.sin_addr.s_addr = htonl(INADDR_ANY); + addr.sin_addr.s_addr = *((in_addr_t*)(hp->h_addr)); } - ip = inet_ntoa(addr.sin_addr); - if (buffer != NULL) - delete[] buffer; - return true; + } else { + addr.sin_addr.s_addr = htonl(INADDR_ANY); + } + ip = inet_ntoa(addr.sin_addr); + if (buffer != NULL) + delete[] buffer; + return true; #elif defined(_MSC_VER) - struct sockaddr_in addr; - memset(&addr, 0, sizeof(addr)); - addr.sin_family = AF_INET; - if (host && host[0]) { - if (IsRawIp(host)) { - if ((addr.sin_addr.s_addr = inet_addr(host)) == INADDR_NONE) - return false; - } else { - addrinfo hints; - struct addrinfo* result = NULL; - std::memset(&hints, 0, sizeof(hints)); - auto ret = ::getaddrinfo(host, NULL, &hints, &result); - if (ret != 0) { - return false; - } + struct sockaddr_in addr; + memset(&addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + if (host && host[0]) { + if (IsRawIp(host)) { + if ((addr.sin_addr.s_addr = inet_addr(host)) == INADDR_NONE) + return false; + } else { + addrinfo hints; + struct addrinfo* result = NULL; + std::memset(&hints, 0, sizeof(hints)); + auto ret = ::getaddrinfo(host, NULL, &hints, &result); + if (ret != 0) { + return false; + } - bool found = false; - for (auto ptr = result; ptr != NULL; ptr = ptr->ai_next) { - if (AF_INET == ptr->ai_family) { - addr.sin_addr = ((struct sockaddr_in*)ptr->ai_addr)->sin_addr; - found = true; - break; - } + bool found = false; + for (auto ptr = result; ptr != NULL; ptr = ptr->ai_next) { + if (AF_INET == ptr->ai_family) { + addr.sin_addr = ((struct sockaddr_in*)ptr->ai_addr)->sin_addr; + found = true; + break; } - freeaddrinfo(result); - if (!found) - return false; } - } else { - addr.sin_addr.s_addr = htonl(INADDR_ANY); + freeaddrinfo(result); + if (!found) + return false; } - ip = inet_ntoa(addr.sin_addr); - return true; -#endif + } else { + addr.sin_addr.s_addr = htonl(INADDR_ANY); } + ip = inet_ntoa(addr.sin_addr); + return true; +#endif +} -} // namespace logtail \ No newline at end of file +} // namespace logtail diff --git a/core/common/DNSCache.h b/core/common/DNSCache.h index aec08786b0..90395ed187 100644 --- a/core/common/DNSCache.h +++ b/core/common/DNSCache.h @@ -15,11 +15,16 @@ */ #pragma once + #include #include #include #include +#include "common/Flags.h" + +DECLARE_FLAG_INT32(dns_cache_ttl_sec); + namespace logtail { class DnsCache { @@ -75,7 +80,7 @@ class DnsCache { } private: - DnsCache(const int32_t ttlSeconds = 60 * 10) : mUpdateTime(time(NULL)), mDnsTTL(ttlSeconds) {} + DnsCache(const int32_t ttlSeconds = INT32_FLAG(dns_cache_ttl_sec)); ~DnsCache() = default; bool IsRawIp(const char* host) { diff --git a/core/common/http/Curl.cpp b/core/common/http/Curl.cpp index 4a841b046e..f6ab028d57 100644 --- a/core/common/http/Curl.cpp +++ b/core/common/http/Curl.cpp @@ -211,17 +211,30 @@ bool SendHttpRequest(unique_ptr&& request, HttpResponse& response) while (true) { CURLcode res = curl_easy_perform(curl); if (res == CURLE_OK) { - long http_code = 0; - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code); - response.SetStatusCode(http_code); + long statusCode = 0; + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &statusCode); + curl_off_t responseTime; + curl_easy_getinfo(curl, CURLINFO_TOTAL_TIME_T, &responseTime); + auto responseTimeMs = responseTime / 1000; + response.SetNetworkStatus(NetworkCode::Ok, ""); + response.SetStatusCode(statusCode); + response.SetResponseTime(chrono::milliseconds(responseTimeMs)); success = true; + LOG_DEBUG(sLogger, + ("send http request succeeded, request address", request)("host", request->mHost)( + "response time", ToString(responseTimeMs) + "ms")("try cnt", ToString(request->mTryCnt))); break; } else if (request->mTryCnt < request->mMaxTryCnt) { - LOG_WARNING(sLogger, - ("failed to send http request", "retry immediately")("request address", request.get())( - "host", request->mHost)("try cnt", request->mTryCnt)("errMsg", curl_easy_strerror(res))); + LOG_DEBUG(sLogger, + ("failed to send http request", "retry immediately")("request address", request.get())( + "host", request->mHost)("try cnt", request->mTryCnt)("errMsg", curl_easy_strerror(res))); ++request->mTryCnt; } else { + auto errMsg = curl_easy_strerror(res); + response.SetNetworkStatus(GetNetworkStatus(res), errMsg); + LOG_DEBUG(sLogger, + ("failed to send http request", "abort")("request address", request)("host", request->mHost)( + "try cnt", ToString(request->mTryCnt))("errMsg", errMsg)); break; } } @@ -250,8 +263,8 @@ bool AddRequestToMultiCurlHandler(CURLM* multiCurl, unique_ptr& request->mFollowRedirects, request->mTls); if (curl == NULL) { - LOG_ERROR(sLogger, ("failed to send request", "failed to init curl handler")("request address", request.get())); request->mResponse.SetNetworkStatus(NetworkCode::Other, "failed to init curl handler"); + LOG_ERROR(sLogger, ("failed to send request", "failed to init curl handler")("request address", request.get())); request->OnSendDone(request->mResponse); return false; } @@ -259,12 +272,13 @@ bool AddRequestToMultiCurlHandler(CURLM* multiCurl, unique_ptr& request->mPrivateData = headers; curl_easy_setopt(curl, CURLOPT_PRIVATE, request.get()); request->mLastSendTime = chrono::system_clock::now(); + CURLMcode res = curl_multi_add_handle(multiCurl, curl); if (res != CURLM_OK) { + request->mResponse.SetNetworkStatus(NetworkCode::Other, "failed to add the easy curl handle to multi_handle"); LOG_ERROR(sLogger, ("failed to send request", "failed to add the easy curl handle to multi_handle")( "errMsg", curl_multi_strerror(res))("request address", request.get())); - request->mResponse.SetNetworkStatus(NetworkCode::Other, "failed to add the easy curl handle to multi_handle"); request->OnSendDone(request->mResponse); curl_easy_cleanup(curl); return false; @@ -283,29 +297,31 @@ void HandleCompletedAsynRequests(CURLM* multiCurl, int& runningHandlers) { CURL* handler = msg->easy_handle; AsynHttpRequest* request = nullptr; curl_easy_getinfo(handler, CURLINFO_PRIVATE, &request); - auto responseTimeMs - = chrono::duration_cast(chrono::system_clock::now() - request->mLastSendTime); switch (msg->data.result) { case CURLE_OK: { long statusCode = 0; curl_easy_getinfo(handler, CURLINFO_RESPONSE_CODE, &statusCode); + curl_off_t responseTime; + curl_easy_getinfo(handler, CURLINFO_TOTAL_TIME_T, &responseTime); + auto responseTimeMs = responseTime / 1000; request->mResponse.SetNetworkStatus(NetworkCode::Ok, ""); request->mResponse.SetStatusCode(statusCode); - request->mResponse.SetResponseTime(responseTimeMs); + request->mResponse.SetResponseTime(chrono::milliseconds(responseTimeMs)); + LOG_DEBUG( + sLogger, + ("send http request succeeded, request address", + request)("host", request->mHost)("response time", ToString(responseTimeMs) + "ms")( + "try cnt", ToString(request->mTryCnt))("errMsg", curl_easy_strerror(msg->data.result))); request->OnSendDone(request->mResponse); - LOG_DEBUG(sLogger, - ("send http request succeeded, request address", request)("host", request->mHost)( - "response time", - ToString(responseTimeMs.count()) + "ms")("try cnt", ToString(request->mTryCnt))); break; } default: // considered as network error if (request->mTryCnt < request->mMaxTryCnt) { LOG_DEBUG(sLogger, - ("failed to send http request", - "retry immediately")("request address", request)("host", request->mHost)( - "try cnt", request->mTryCnt)("errMsg", curl_easy_strerror(msg->data.result))); + ("failed to send http request", "retry immediately")("request address", + request)("host", request->mHost)( + "try cnt", request->mTryCnt)("errMsg", curl_easy_strerror(msg->data.result))); // free first，becase mPrivateData will be reset in AddRequestToMultiCurlHandler if (request->mPrivateData) { curl_slist_free_all((curl_slist*)request->mPrivateData); @@ -318,11 +334,10 @@ void HandleCompletedAsynRequests(CURLM* multiCurl, int& runningHandlers) { } else { auto errMsg = curl_easy_strerror(msg->data.result); request->mResponse.SetNetworkStatus(GetNetworkStatus(msg->data.result), errMsg); - request->OnSendDone(request->mResponse); LOG_DEBUG(sLogger, ("failed to send http request", "abort")("request address", request)( - "host", request->mHost)("response time", ToString(responseTimeMs.count()) + "ms")( - "try cnt", ToString(request->mTryCnt))("errMsg", errMsg)); + "host", request->mHost)("try cnt", ToString(request->mTryCnt))("errMsg", errMsg)); + request->OnSendDone(request->mResponse); } break; } diff --git a/core/runner/sink/http/HttpSink.cpp b/core/runner/sink/http/HttpSink.cpp index bf4fcd0163..bc46fb5ffb 100644 --- a/core/runner/sink/http/HttpSink.cpp +++ b/core/runner/sink/http/HttpSink.cpp @@ -122,6 +122,7 @@ bool HttpSink::AddRequestToClient(unique_ptr&& request) { AppConfig::GetInstance()->GetBindInterface()); if (curl == nullptr) { request->mItem->mStatus = SendingStatus::IDLE; + request->mResponse.SetNetworkStatus(NetworkCode::Other, "failed to init curl handler"); FlusherRunner::GetInstance()->DecreaseHttpSendingCnt(); mOutFailedItemsTotal->Add(1); LOG_ERROR(sLogger, @@ -134,11 +135,11 @@ bool HttpSink::AddRequestToClient(unique_ptr&& request) { request->mPrivateData = headers; curl_easy_setopt(curl, CURLOPT_PRIVATE, request.get()); - request->mLastSendTime = chrono::system_clock::now(); auto res = curl_multi_add_handle(mClient, curl); if (res != CURLM_OK) { request->mItem->mStatus = SendingStatus::IDLE; + request->mResponse.SetNetworkStatus(NetworkCode::Other, "failed to add the easy curl handle to multi_handle"); FlusherRunner::GetInstance()->DecreaseHttpSendingCnt(); curl_easy_cleanup(curl); mOutFailedItemsTotal->Add(1); @@ -245,14 +246,16 @@ void HttpSink::HandleCompletedRequests(int& runningHandlers) { case CURLE_OK: { long statusCode = 0; curl_easy_getinfo(handler, CURLINFO_RESPONSE_CODE, &statusCode); + request->mResponse.SetNetworkStatus(NetworkCode::Ok, ""); request->mResponse.SetStatusCode(statusCode); request->mResponse.SetResponseTime(responseTimeMs); - LOG_DEBUG( - sLogger, - ("send http request succeeded, item address", request->mItem)( - "config-flusher-dst", QueueKeyManager::GetInstance()->GetName(request->mItem->mQueueKey))( - "response time", ToString(responseTimeMs.count()) + "ms")("try cnt", ToString(request->mTryCnt))( - "sending cnt", ToString(FlusherRunner::GetInstance()->GetSendingBufferCount()))); + LOG_DEBUG(sLogger, + ("send http request succeeded, item address", + request->mItem)("config-flusher-dst", + QueueKeyManager::GetInstance()->GetName(request->mItem->mQueueKey))( + "response time", ToString(responseTimeMs.count()) + "ms")("try cnt", + ToString(request->mTryCnt))( + "sending cnt", ToString(FlusherRunner::GetInstance()->GetSendingBufferCount()))); static_cast(request->mItem->mFlusher)->OnSendDone(request->mResponse, request->mItem); FlusherRunner::GetInstance()->DecreaseHttpSendingCnt(); mOutSuccessfulItemsTotal->Add(1); @@ -279,12 +282,14 @@ void HttpSink::HandleCompletedRequests(int& runningHandlers) { ++runningHandlers; requestReused = true; } else { + auto errMsg = curl_easy_strerror(msg->data.result); + request->mResponse.SetNetworkStatus(GetNetworkStatus(msg->data.result), errMsg); LOG_DEBUG(sLogger, ("failed to send http request", "abort")("item address", request->mItem)( "config-flusher-dst", QueueKeyManager::GetInstance()->GetName(request->mItem->mQueueKey))( - "response time", ToString(responseTimeMs.count()) + "ms")("try cnt", - ToString(request->mTryCnt))( + "response time", ToString(responseTimeMs.count()) + "ms")( + "try cnt", ToString(request->mTryCnt))("errMsg", errMsg)( "sending cnt", ToString(FlusherRunner::GetInstance()->GetSendingBufferCount()))); static_cast(request->mItem->mFlusher) ->OnSendDone(request->mResponse, request->mItem); From d4183b136d9f2dbf758e8deb1a5265f09f0f8b50 Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Mon, 23 Dec 2024 03:06:46 +0000 Subject: [PATCH 23/41] polish --- core/common/http/Curl.cpp | 10 +++++----- core/plugin/flusher/sls/FlusherSLS.cpp | 5 ++++- core/unittest/flusher/FlusherSLSUnittest.cpp | 6 +++--- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/core/common/http/Curl.cpp b/core/common/http/Curl.cpp index f6ab028d57..075ae6629d 100644 --- a/core/common/http/Curl.cpp +++ b/core/common/http/Curl.cpp @@ -221,20 +221,20 @@ bool SendHttpRequest(unique_ptr&& request, HttpResponse& response) response.SetResponseTime(chrono::milliseconds(responseTimeMs)); success = true; LOG_DEBUG(sLogger, - ("send http request succeeded, request address", request)("host", request->mHost)( + ("send http request succeeded, host", request->mHost)( "response time", ToString(responseTimeMs) + "ms")("try cnt", ToString(request->mTryCnt))); break; } else if (request->mTryCnt < request->mMaxTryCnt) { LOG_DEBUG(sLogger, - ("failed to send http request", "retry immediately")("request address", request.get())( - "host", request->mHost)("try cnt", request->mTryCnt)("errMsg", curl_easy_strerror(res))); + ("failed to send http request", "retry immediately")("host", request->mHost)( + "try cnt", request->mTryCnt)("errMsg", curl_easy_strerror(res))); ++request->mTryCnt; } else { auto errMsg = curl_easy_strerror(res); response.SetNetworkStatus(GetNetworkStatus(res), errMsg); LOG_DEBUG(sLogger, - ("failed to send http request", "abort")("request address", request)("host", request->mHost)( - "try cnt", ToString(request->mTryCnt))("errMsg", errMsg)); + ("failed to send http request", + "abort")("host", request->mHost)("try cnt", ToString(request->mTryCnt))("errMsg", errMsg)); break; } } diff --git a/core/plugin/flusher/sls/FlusherSLS.cpp b/core/plugin/flusher/sls/FlusherSLS.cpp index e0980a6137..efe51d7f89 100644 --- a/core/plugin/flusher/sls/FlusherSLS.cpp +++ b/core/plugin/flusher/sls/FlusherSLS.cpp @@ -836,7 +836,10 @@ void FlusherSLS::OnSendDone(const HttpResponse& response, SenderQueueItem* item) operation = OperationOnFail::DISCARD; } #ifdef __ENTERPRISE__ - if (sendResult != SEND_NETWORK_ERROR && sendResult != SEND_SERVER_ERROR) { + bool hasNetworkError = (sendResult == SEND_NETWORK_ERROR || sendResult == SEND_SERVER_ERROR); + EnterpriseSLSClientManager::GetInstance()->UpdateHostStatus(mProject, mCandidateHostsInfo->GetMode(), data->mCurrentHost, !hasNetworkError); + mCandidateHostsInfo->SelectBestHost(); + if (!hasNetworkError) { bool hasAuthError = sendResult == SEND_UNAUTHORIZED; EnterpriseSLSClientManager::GetInstance()->UpdateAccessKeyStatus(mAliuid, !hasAuthError); EnterpriseSLSClientManager::GetInstance()->UpdateProjectAnonymousWriteStatus(mProject, !hasAuthError); diff --git a/core/unittest/flusher/FlusherSLSUnittest.cpp b/core/unittest/flusher/FlusherSLSUnittest.cpp index 6b6594dcbb..c8c344a1dd 100644 --- a/core/unittest/flusher/FlusherSLSUnittest.cpp +++ b/core/unittest/flusher/FlusherSLSUnittest.cpp @@ -716,7 +716,7 @@ void FlusherSLSUnittest::TestBuildRequest() { APSARA_TEST_EQUAL(static_cast(AppConfig::GetInstance()->GetSendRequestConcurrency()) / 2, FlusherSLS::GetRegionConcurrencyLimiter(flusher.mRegion)->GetCurrentLimit()); } - EnterpriseSLSClientManager::GetInstance()->UpdateHostInfo("test_project", + EnterpriseSLSClientManager::GetInstance()->UpdateHostLatency("test_project", EndpointMode::DEFAULT, "test_project.test_region-b.log.aliyuncs.com", chrono::milliseconds(100)); @@ -1043,7 +1043,7 @@ void FlusherSLSUnittest::TestBuildRequest() { APSARA_TEST_FALSE(flusher.BuildRequest(&item, req, &keepItem)); APSARA_TEST_NOT_EQUAL(old, flusher.mCandidateHostsInfo.get()); - EnterpriseSLSClientManager::GetInstance()->UpdateHostInfo("test_project", + EnterpriseSLSClientManager::GetInstance()->UpdateHostLatency("test_project", EndpointMode::ACCELERATE, "test_project." + kAccelerationDataEndpoint, chrono::milliseconds(10)); @@ -1122,7 +1122,7 @@ void FlusherSLSUnittest::TestBuildRequest() { APSARA_TEST_FALSE(flusher.BuildRequest(&item, req, &keepItem)); APSARA_TEST_NOT_EQUAL(old, flusher.mCandidateHostsInfo.get()); - EnterpriseSLSClientManager::GetInstance()->UpdateHostInfo( + EnterpriseSLSClientManager::GetInstance()->UpdateHostLatency( "test_project", EndpointMode::CUSTOM, "test_project.custom.endpoint", chrono::milliseconds(10)); flusher.mCandidateHostsInfo->SelectBestHost(); APSARA_TEST_TRUE(flusher.BuildRequest(&item, req, &keepItem)); From e4039dca8c4869397e8e4776438a05e322244560 Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Mon, 23 Dec 2024 03:53:04 +0000 Subject: [PATCH 24/41] polish --- core/runner/sink/http/HttpSink.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/core/runner/sink/http/HttpSink.cpp b/core/runner/sink/http/HttpSink.cpp index bc46fb5ffb..1cef22d257 100644 --- a/core/runner/sink/http/HttpSink.cpp +++ b/core/runner/sink/http/HttpSink.cpp @@ -266,12 +266,11 @@ void HttpSink::HandleCompletedRequests(int& runningHandlers) { default: // considered as network error if (request->mTryCnt <= request->mMaxTryCnt) { - LOG_WARNING( - sLogger, - ("failed to send http request", "retry immediately")("item address", request->mItem)( - "config-flusher-dst", - QueueKeyManager::GetInstance()->GetName(request->mItem->mFlusher->GetQueueKey()))( - "try cnt", request->mTryCnt)("errMsg", curl_easy_strerror(msg->data.result))); + LOG_DEBUG(sLogger, + ("failed to send http request", "retry immediately")("item address", request->mItem)( + "config-flusher-dst", + QueueKeyManager::GetInstance()->GetName(request->mItem->mFlusher->GetQueueKey()))( + "try cnt", request->mTryCnt)("errMsg", curl_easy_strerror(msg->data.result))); // free first，becase mPrivateData will be reset in AddRequestToClient if (request->mPrivateData) { curl_slist_free_all((curl_slist*)request->mPrivateData); From f637af86c6f248bb3b5ee49c9ed460ea3fd7e38d Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Mon, 23 Dec 2024 05:40:26 +0000 Subject: [PATCH 25/41] polish --- core/plugin/flusher/sls/FlusherSLS.cpp | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/core/plugin/flusher/sls/FlusherSLS.cpp b/core/plugin/flusher/sls/FlusherSLS.cpp index efe51d7f89..d8bdccad71 100644 --- a/core/plugin/flusher/sls/FlusherSLS.cpp +++ b/core/plugin/flusher/sls/FlusherSLS.cpp @@ -835,16 +835,6 @@ void FlusherSLS::OnSendDone(const HttpResponse& response, SenderQueueItem* item) if (isProfileData && data->mTryCnt >= static_cast(INT32_FLAG(profile_data_send_retrytimes))) { operation = OperationOnFail::DISCARD; } -#ifdef __ENTERPRISE__ - bool hasNetworkError = (sendResult == SEND_NETWORK_ERROR || sendResult == SEND_SERVER_ERROR); - EnterpriseSLSClientManager::GetInstance()->UpdateHostStatus(mProject, mCandidateHostsInfo->GetMode(), data->mCurrentHost, !hasNetworkError); - mCandidateHostsInfo->SelectBestHost(); - if (!hasNetworkError) { - bool hasAuthError = sendResult == SEND_UNAUTHORIZED; - EnterpriseSLSClientManager::GetInstance()->UpdateAccessKeyStatus(mAliuid, !hasAuthError); - EnterpriseSLSClientManager::GetInstance()->UpdateProjectAnonymousWriteStatus(mProject, !hasAuthError); - } -#endif #define LOG_PATTERN \ ("failed to send request", failDetail.str())("operation", GetOperationString(operation))("suggestion", \ @@ -892,6 +882,18 @@ void FlusherSLS::OnSendDone(const HttpResponse& response, SenderQueueItem* item) break; } } +#ifdef __ENTERPRISE__ + bool hasNetworkError = (sendResult == SEND_NETWORK_ERROR || sendResult == SEND_SERVER_ERROR); + EnterpriseSLSClientManager::GetInstance()->UpdateHostStatus( + mProject, mCandidateHostsInfo->GetMode(), data->mCurrentHost, !hasNetworkError); + mCandidateHostsInfo->SelectBestHost(); + + if (!hasNetworkError) { + bool hasAuthError = sendResult == SEND_UNAUTHORIZED; + EnterpriseSLSClientManager::GetInstance()->UpdateAccessKeyStatus(mAliuid, !hasAuthError); + EnterpriseSLSClientManager::GetInstance()->UpdateProjectAnonymousWriteStatus(mProject, !hasAuthError); + } +#endif } bool FlusherSLS::Send(string&& data, const string& shardHashKey, const string& logstore) { From f319b53b29494f383440c13fca040ec3f5824547 Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Mon, 23 Dec 2024 08:32:57 +0000 Subject: [PATCH 26/41] polish --- core/runner/FlusherRunner.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/runner/FlusherRunner.cpp b/core/runner/FlusherRunner.cpp index 6dbc2de68e..e8bd56b023 100644 --- a/core/runner/FlusherRunner.cpp +++ b/core/runner/FlusherRunner.cpp @@ -125,10 +125,12 @@ void FlusherRunner::PushToHttpSink(SenderQueueItem* item, bool withLimit) { LOG_DEBUG(sLogger, ("failed to build request", "retry later")("item address", item)( "config-flusher-dst", QueueKeyManager::GetInstance()->GetName(item->mQueueKey))); + SenderQueueManager::GetInstance()->DecreaseConcurrencyLimiterInSendingCnt(item->mQueueKey); } else { LOG_WARNING(sLogger, ("failed to build request", "discard item")("item address", item)( "config-flusher-dst", QueueKeyManager::GetInstance()->GetName(item->mQueueKey))); + SenderQueueManager::GetInstance()->DecreaseConcurrencyLimiterInSendingCnt(item->mQueueKey); SenderQueueManager::GetInstance()->RemoveItem(item->mQueueKey, item); } return; From ed59495f0e4de0e41d26b5bad4c6f0a107073b72 Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Mon, 23 Dec 2024 09:46:57 +0000 Subject: [PATCH 27/41] polish --- core/pipeline/plugin/interface/HttpFlusher.h | 4 ++- core/plugin/flusher/sls/FlusherSLS.cpp | 4 ++- core/plugin/flusher/sls/FlusherSLS.h | 2 +- core/runner/FlusherRunner.cpp | 3 +- core/unittest/flusher/FlusherSLSUnittest.cpp | 31 ++++++++++--------- core/unittest/plugin/PluginMock.h | 2 +- .../how-to-write-native-flusher-plugins.md | 2 +- 7 files changed, 27 insertions(+), 21 deletions(-) diff --git a/core/pipeline/plugin/interface/HttpFlusher.h b/core/pipeline/plugin/interface/HttpFlusher.h index 377a075b31..f68eab2bd2 100644 --- a/core/pipeline/plugin/interface/HttpFlusher.h +++ b/core/pipeline/plugin/interface/HttpFlusher.h @@ -16,6 +16,8 @@ #pragma once +#include + #include "common/http/HttpResponse.h" #include "pipeline/plugin/interface/Flusher.h" #include "pipeline/queue/SenderQueueItem.h" @@ -27,7 +29,7 @@ class HttpFlusher : public Flusher { public: virtual ~HttpFlusher() = default; - virtual bool BuildRequest(SenderQueueItem* item, std::unique_ptr& req, bool* keepItem) = 0; + virtual bool BuildRequest(SenderQueueItem* item, std::unique_ptr& req, bool* keepItem, std::string* errMsg) = 0; virtual void OnSendDone(const HttpResponse& response, SenderQueueItem* item) = 0; virtual SinkType GetSinkType() override { return SinkType::HTTP; } diff --git a/core/plugin/flusher/sls/FlusherSLS.cpp b/core/plugin/flusher/sls/FlusherSLS.cpp index d8bdccad71..bfa41b54b1 100644 --- a/core/plugin/flusher/sls/FlusherSLS.cpp +++ b/core/plugin/flusher/sls/FlusherSLS.cpp @@ -585,7 +585,7 @@ bool FlusherSLS::FlushAll() { return SerializeAndPush(std::move(res)); } -bool FlusherSLS::BuildRequest(SenderQueueItem* item, unique_ptr& req, bool* keepItem) { +bool FlusherSLS::BuildRequest(SenderQueueItem* item, unique_ptr& req, bool* keepItem, string* errMsg) { if (mSendCnt) { mSendCnt->Add(1); } @@ -597,6 +597,7 @@ bool FlusherSLS::BuildRequest(SenderQueueItem* item, unique_ptr if (!EnterpriseSLSClientManager::GetInstance()->GetAccessKeyIfProjectSupportsAnonymousWrite( mProject, type, accessKeyId, accessKeySecret)) { *keepItem = true; + *errMsg = "failed to get access key"; return false; } #endif @@ -641,6 +642,7 @@ bool FlusherSLS::BuildRequest(SenderQueueItem* item, unique_ptr if (mCandidateHostsInfo->IsInitialized()) { GetRegionConcurrencyLimiter(mRegion)->OnFail(); } + *errMsg = "failed to get available host"; *keepItem = true; return false; } diff --git a/core/plugin/flusher/sls/FlusherSLS.h b/core/plugin/flusher/sls/FlusherSLS.h index 7fa4610056..0d83f49ba0 100644 --- a/core/plugin/flusher/sls/FlusherSLS.h +++ b/core/plugin/flusher/sls/FlusherSLS.h @@ -67,7 +67,7 @@ class FlusherSLS : public HttpFlusher { bool Send(PipelineEventGroup&& g) override; bool Flush(size_t key) override; bool FlushAll() override; - bool BuildRequest(SenderQueueItem* item, std::unique_ptr& req, bool* keepItem) override; + bool BuildRequest(SenderQueueItem* item, std::unique_ptr& req, bool* keepItem, std::string* errMsg) override; void OnSendDone(const HttpResponse& response, SenderQueueItem* item) override; CompressType GetCompressType() const { return mCompressor ? mCompressor->GetCompressType() : CompressType::NONE; } diff --git a/core/runner/FlusherRunner.cpp b/core/runner/FlusherRunner.cpp index e8bd56b023..079fe978f7 100644 --- a/core/runner/FlusherRunner.cpp +++ b/core/runner/FlusherRunner.cpp @@ -117,7 +117,8 @@ void FlusherRunner::PushToHttpSink(SenderQueueItem* item, bool withLimit) { unique_ptr req; bool keepItem = false; - if (!static_cast(item->mFlusher)->BuildRequest(item, req, &keepItem)) { + string errMsg; + if (!static_cast(item->mFlusher)->BuildRequest(item, req, &keepItem, &errMsg)) { if (keepItem && chrono::duration_cast(chrono::system_clock::now() - item->mFirstEnqueTime).count() < INT32_FLAG(discard_send_fail_interval)) { diff --git a/core/unittest/flusher/FlusherSLSUnittest.cpp b/core/unittest/flusher/FlusherSLSUnittest.cpp index c8c344a1dd..ae974fae33 100644 --- a/core/unittest/flusher/FlusherSLSUnittest.cpp +++ b/core/unittest/flusher/FlusherSLSUnittest.cpp @@ -688,12 +688,13 @@ void FlusherSLSUnittest::TestBuildRequest() { SLSSenderQueueItem item("hello, world!", rawSize, &flusher, flusher.GetQueueKey(), flusher.mLogstore); unique_ptr req; bool keepItem = false; + string errMsg; #ifdef __ENTERPRISE__ { // empty ak, first try - APSARA_TEST_FALSE(flusher.BuildRequest(&item, req, &keepItem)); + APSARA_TEST_FALSE(flusher.BuildRequest(&item, req, &keepItem, &errMsg)); // empty ak, second try - APSARA_TEST_FALSE(flusher.BuildRequest(&item, req, &keepItem)); + APSARA_TEST_FALSE(flusher.BuildRequest(&item, req, &keepItem, &errMsg)); APSARA_TEST_EQUAL(nullptr, req); APSARA_TEST_TRUE(keepItem); } @@ -701,7 +702,7 @@ void FlusherSLSUnittest::TestBuildRequest() { "1234567890", SLSClientManager::AuthType::ANONYMOUS, "test_ak", "test_sk"); { // no available host, uninitialized - APSARA_TEST_FALSE(flusher.BuildRequest(&item, req, &keepItem)); + APSARA_TEST_FALSE(flusher.BuildRequest(&item, req, &keepItem, &errMsg)); APSARA_TEST_EQUAL(nullptr, req); APSARA_TEST_TRUE(keepItem); APSARA_TEST_EQUAL(static_cast(AppConfig::GetInstance()->GetSendRequestConcurrency()), @@ -710,7 +711,7 @@ void FlusherSLSUnittest::TestBuildRequest() { { // no available host, initialized flusher.mCandidateHostsInfo->SetInitialized(); - APSARA_TEST_FALSE(flusher.BuildRequest(&item, req, &keepItem)); + APSARA_TEST_FALSE(flusher.BuildRequest(&item, req, &keepItem, &errMsg)); APSARA_TEST_EQUAL(nullptr, req); APSARA_TEST_TRUE(keepItem); APSARA_TEST_EQUAL(static_cast(AppConfig::GetInstance()->GetSendRequestConcurrency()) / 2, @@ -726,7 +727,7 @@ void FlusherSLSUnittest::TestBuildRequest() { { // normal SLSSenderQueueItem item("hello, world!", rawSize, &flusher, flusher.GetQueueKey(), flusher.mLogstore); - APSARA_TEST_TRUE(flusher.BuildRequest(&item, req, &keepItem)); + APSARA_TEST_TRUE(flusher.BuildRequest(&item, req, &keepItem, &errMsg)); APSARA_TEST_EQUAL(HTTP_POST, req->mMethod); #ifdef __ENTERPRISE__ APSARA_TEST_FALSE(req->mHTTPSFlag); @@ -788,7 +789,7 @@ void FlusherSLSUnittest::TestBuildRequest() { flusher.GetQueueKey(), flusher.mLogstore, RawDataType::EVENT_GROUP_LIST); - APSARA_TEST_TRUE(flusher.BuildRequest(&item, req, &keepItem)); + APSARA_TEST_TRUE(flusher.BuildRequest(&item, req, &keepItem, &errMsg)); APSARA_TEST_EQUAL(HTTP_POST, req->mMethod); #ifdef __ENTERPRISE__ APSARA_TEST_FALSE(req->mHTTPSFlag); @@ -851,7 +852,7 @@ void FlusherSLSUnittest::TestBuildRequest() { flusher.mLogstore, RawDataType::EVENT_GROUP, "hash_key"); - APSARA_TEST_TRUE(flusher.BuildRequest(&item, req, &keepItem)); + APSARA_TEST_TRUE(flusher.BuildRequest(&item, req, &keepItem, &errMsg)); APSARA_TEST_EQUAL(HTTP_POST, req->mMethod); #ifdef __ENTERPRISE__ APSARA_TEST_FALSE(req->mHTTPSFlag); @@ -920,7 +921,7 @@ void FlusherSLSUnittest::TestBuildRequest() { RawDataType::EVENT_GROUP, "hash_key_0", std::move(cpt)); - APSARA_TEST_TRUE(flusher.BuildRequest(&item, req, &keepItem)); + APSARA_TEST_TRUE(flusher.BuildRequest(&item, req, &keepItem, &errMsg)); APSARA_TEST_EQUAL(HTTP_POST, req->mMethod); #ifdef __ENTERPRISE__ APSARA_TEST_FALSE(req->mHTTPSFlag); @@ -979,7 +980,7 @@ void FlusherSLSUnittest::TestBuildRequest() { flusher.mTelemetryType = sls_logs::SlsTelemetryType::SLS_TELEMETRY_TYPE_METRICS; { SLSSenderQueueItem item("hello, world!", rawSize, &flusher, flusher.GetQueueKey(), flusher.mLogstore); - APSARA_TEST_TRUE(flusher.BuildRequest(&item, req, &keepItem)); + APSARA_TEST_TRUE(flusher.BuildRequest(&item, req, &keepItem, &errMsg)); APSARA_TEST_EQUAL(HTTP_POST, req->mMethod); #ifdef __ENTERPRISE__ APSARA_TEST_FALSE(req->mHTTPSFlag); @@ -1040,7 +1041,7 @@ void FlusherSLSUnittest::TestBuildRequest() { EnterpriseSLSClientManager::GetInstance()->CopyLocalRegionEndpointsAndHttpsInfoIfNotExisted("test_region", "test_region-b"); auto old = flusher.mCandidateHostsInfo.get(); - APSARA_TEST_FALSE(flusher.BuildRequest(&item, req, &keepItem)); + APSARA_TEST_FALSE(flusher.BuildRequest(&item, req, &keepItem, &errMsg)); APSARA_TEST_NOT_EQUAL(old, flusher.mCandidateHostsInfo.get()); EnterpriseSLSClientManager::GetInstance()->UpdateHostLatency("test_project", @@ -1048,7 +1049,7 @@ void FlusherSLSUnittest::TestBuildRequest() { "test_project." + kAccelerationDataEndpoint, chrono::milliseconds(10)); flusher.mCandidateHostsInfo->SelectBestHost(); - APSARA_TEST_TRUE(flusher.BuildRequest(&item, req, &keepItem)); + APSARA_TEST_TRUE(flusher.BuildRequest(&item, req, &keepItem, &errMsg)); APSARA_TEST_EQUAL("test_project." + kAccelerationDataEndpoint, req->mHost); } // real ip @@ -1057,7 +1058,7 @@ void FlusherSLSUnittest::TestBuildRequest() { // ip not empty EnterpriseSLSClientManager::GetInstance()->SetRealIp("test_region-b", "192.168.0.1"); SLSSenderQueueItem item("hello, world!", rawSize, &flusher, flusher.GetQueueKey(), flusher.mLogstore); - APSARA_TEST_TRUE(flusher.BuildRequest(&item, req, &keepItem)); + APSARA_TEST_TRUE(flusher.BuildRequest(&item, req, &keepItem, &errMsg)); APSARA_TEST_EQUAL(HTTP_POST, req->mMethod); APSARA_TEST_FALSE(req->mHTTPSFlag); APSARA_TEST_EQUAL("/logstores/test_logstore/shards/lb", req->mUrl); @@ -1089,7 +1090,7 @@ void FlusherSLSUnittest::TestBuildRequest() { // ip empty EnterpriseSLSClientManager::GetInstance()->SetRealIp("test_region-b", ""); SLSSenderQueueItem item("hello, world!", rawSize, &flusher, flusher.GetQueueKey(), flusher.mLogstore); - APSARA_TEST_TRUE(flusher.BuildRequest(&item, req, &keepItem)); + APSARA_TEST_TRUE(flusher.BuildRequest(&item, req, &keepItem, &errMsg)); APSARA_TEST_EQUAL("test_project." + kAccelerationDataEndpoint, req->mHeader[HOST]); APSARA_TEST_EQUAL(SLSClientManager::GetInstance()->GetUserAgent(), req->mHeader[USER_AGENT]); APSARA_TEST_FALSE(req->mHeader[DATE].empty()); @@ -1119,13 +1120,13 @@ void FlusherSLSUnittest::TestBuildRequest() { endpoints.mLocalEndpoints = {"custom.endpoint"}; auto old = flusher.mCandidateHostsInfo.get(); - APSARA_TEST_FALSE(flusher.BuildRequest(&item, req, &keepItem)); + APSARA_TEST_FALSE(flusher.BuildRequest(&item, req, &keepItem, &errMsg)); APSARA_TEST_NOT_EQUAL(old, flusher.mCandidateHostsInfo.get()); EnterpriseSLSClientManager::GetInstance()->UpdateHostLatency( "test_project", EndpointMode::CUSTOM, "test_project.custom.endpoint", chrono::milliseconds(10)); flusher.mCandidateHostsInfo->SelectBestHost(); - APSARA_TEST_TRUE(flusher.BuildRequest(&item, req, &keepItem)); + APSARA_TEST_TRUE(flusher.BuildRequest(&item, req, &keepItem, &errMsg)); APSARA_TEST_EQUAL("test_project.custom.endpoint", req->mHost); } BOOL_FLAG(send_prefer_real_ip) = false; diff --git a/core/unittest/plugin/PluginMock.h b/core/unittest/plugin/PluginMock.h index a20e9fdce4..16a7b14e1f 100644 --- a/core/unittest/plugin/PluginMock.h +++ b/core/unittest/plugin/PluginMock.h @@ -128,7 +128,7 @@ class FlusherHttpMock : public HttpFlusher { return true; } bool FlushAll() override { return mIsValid; } - bool BuildRequest(SenderQueueItem* item, std::unique_ptr& req, bool* keepItem) override { + bool BuildRequest(SenderQueueItem* item, std::unique_ptr& req, bool* keepItem, std::string* errMsg) override { if (item->mData == "invalid_keep") { *keepItem = true; return false; diff --git a/docs/cn/developer-guide/plugin-development/native-plugins/how-to-write-native-flusher-plugins.md b/docs/cn/developer-guide/plugin-development/native-plugins/how-to-write-native-flusher-plugins.md index 048a1d1386..4b62fe91f6 100644 --- a/docs/cn/developer-guide/plugin-development/native-plugins/how-to-write-native-flusher-plugins.md +++ b/docs/cn/developer-guide/plugin-development/native-plugins/how-to-write-native-flusher-plugins.md @@ -24,7 +24,7 @@ public: class HttpFlusher : public Flusher { public: // 用于将待发送数据打包成http请求 - virtual bool BuildRequest(SenderQueueItem* item, std::unique_ptr& req, bool* keepItem) = 0; + virtual bool BuildRequest(SenderQueueItem* item, std::unique_ptr& req, bool* keepItem, std::string* errMsg) = 0; // 用于发送完成后进行记录和处理 virtual void OnSendDone(const HttpResponse& response, SenderQueueItem* item) = 0; }; From 5d2120709005b422a627e6d46444b2820871a8d0 Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Mon, 23 Dec 2024 09:54:16 +0000 Subject: [PATCH 28/41] polish --- core/common/http/HttpRequest.cpp | 2 +- core/common/http/HttpRequest.h | 8 ++++---- core/models/EventPool.cpp | 4 ++-- core/runner/FlusherRunner.cpp | 4 ++-- core/runner/ProcessorRunner.cpp | 4 ++-- core/runner/sink/http/HttpSink.cpp | 4 ++-- core/unittest/flusher/FlusherSLSUnittest.cpp | 14 +++++++------- 7 files changed, 20 insertions(+), 20 deletions(-) diff --git a/core/common/http/HttpRequest.cpp b/core/common/http/HttpRequest.cpp index 34a4f3428e..c108032283 100644 --- a/core/common/http/HttpRequest.cpp +++ b/core/common/http/HttpRequest.cpp @@ -14,7 +14,7 @@ #include "common/http/HttpRequest.h" -DEFINE_FLAG_INT32(default_http_request_timeout_secs, "", 15); +DEFINE_FLAG_INT32(default_http_request_timeout_sec, "", 15); DEFINE_FLAG_INT32(default_http_request_max_try_cnt, "", 3); using namespace std; diff --git a/core/common/http/HttpRequest.h b/core/common/http/HttpRequest.h index f0a2abeb85..2218568344 100644 --- a/core/common/http/HttpRequest.h +++ b/core/common/http/HttpRequest.h @@ -25,7 +25,7 @@ #include "common/Flags.h" #include "common/http/HttpResponse.h" -DECLARE_FLAG_INT32(default_http_request_timeout_secs); +DECLARE_FLAG_INT32(default_http_request_timeout_sec); DECLARE_FLAG_INT32(default_http_request_max_try_cnt); namespace logtail { @@ -49,7 +49,7 @@ struct HttpRequest { std::string mBody; std::string mHost; int32_t mPort; - uint32_t mTimeout = static_cast(INT32_FLAG(default_http_request_timeout_secs)); + uint32_t mTimeout = static_cast(INT32_FLAG(default_http_request_timeout_sec)); uint32_t mMaxTryCnt = static_cast(INT32_FLAG(default_http_request_max_try_cnt)); bool mFollowRedirects = false; std::optional mTls = std::nullopt; @@ -65,7 +65,7 @@ struct HttpRequest { const std::string& query, const std::map& header, const std::string& body, - uint32_t timeout = static_cast(INT32_FLAG(default_http_request_timeout_secs)), + uint32_t timeout = static_cast(INT32_FLAG(default_http_request_timeout_sec)), uint32_t maxTryCnt = static_cast(INT32_FLAG(default_http_request_max_try_cnt)), bool followRedirects = false, std::optional tls = std::nullopt) @@ -98,7 +98,7 @@ struct AsynHttpRequest : public HttpRequest { const std::map& header, const std::string& body, HttpResponse&& response = HttpResponse(), - uint32_t timeout = static_cast(INT32_FLAG(default_http_request_timeout_secs)), + uint32_t timeout = static_cast(INT32_FLAG(default_http_request_timeout_sec)), uint32_t maxTryCnt = static_cast(INT32_FLAG(default_http_request_max_try_cnt)), bool followRedirects = false, std::optional tls = std::nullopt) diff --git a/core/models/EventPool.cpp b/core/models/EventPool.cpp index 5e50091750..0c045438b5 100644 --- a/core/models/EventPool.cpp +++ b/core/models/EventPool.cpp @@ -19,7 +19,7 @@ #include "common/Flags.h" #include "logger/Logger.h" -DEFINE_FLAG_INT32(event_pool_gc_interval_secs, "", 60); +DEFINE_FLAG_INT32(event_pool_gc_interval_sec, "", 60); using namespace std; @@ -143,7 +143,7 @@ void DoGC(vector& pool, vector& poolBak, size_t& minUnusedCnt, mutex* mu } void EventPool::CheckGC() { - if (time(nullptr) - mLastGCTime > INT32_FLAG(event_pool_gc_interval_secs)) { + if (time(nullptr) - mLastGCTime > INT32_FLAG(event_pool_gc_interval_sec)) { if (mEnableLock) { lock_guard lock(mPoolMux); DoGC(mLogEventPool, mLogEventPoolBak, mMinUnusedLogEventsCnt, &mPoolBakMux, "log"); diff --git a/core/runner/FlusherRunner.cpp b/core/runner/FlusherRunner.cpp index 079fe978f7..2830994ee0 100644 --- a/core/runner/FlusherRunner.cpp +++ b/core/runner/FlusherRunner.cpp @@ -28,7 +28,7 @@ #include "plugin/flusher/sls/DiskBufferWriter.h" #include "runner/sink/http/HttpSink.h" -DEFINE_FLAG_INT32(flusher_runner_exit_timeout_secs, "", 60); +DEFINE_FLAG_INT32(flusher_runner_exit_timeout_sec, "", 60); DECLARE_FLAG_INT32(discard_send_fail_interval); @@ -95,7 +95,7 @@ void FlusherRunner::Stop() { if (!mThreadRes.valid()) { return; } - future_status s = mThreadRes.wait_for(chrono::seconds(INT32_FLAG(flusher_runner_exit_timeout_secs))); + future_status s = mThreadRes.wait_for(chrono::seconds(INT32_FLAG(flusher_runner_exit_timeout_sec))); if (s == future_status::ready) { LOG_INFO(sLogger, ("flusher runner", "stopped successfully")); } else { diff --git a/core/runner/ProcessorRunner.cpp b/core/runner/ProcessorRunner.cpp index 76c34afe28..a147a5bc9d 100644 --- a/core/runner/ProcessorRunner.cpp +++ b/core/runner/ProcessorRunner.cpp @@ -27,7 +27,7 @@ #include "queue/QueueKeyManager.h" DEFINE_FLAG_INT32(default_flush_merged_buffer_interval, "default flush merged buffer, seconds", 1); -DEFINE_FLAG_INT32(processor_runner_exit_timeout_secs, "", 60); +DEFINE_FLAG_INT32(processor_runner_exit_timeout_sec, "", 60); DECLARE_FLAG_INT32(max_send_log_group_size); @@ -59,7 +59,7 @@ void ProcessorRunner::Stop() { continue; } future_status s - = mThreadRes[threadNo].wait_for(chrono::seconds(INT32_FLAG(processor_runner_exit_timeout_secs))); + = mThreadRes[threadNo].wait_for(chrono::seconds(INT32_FLAG(processor_runner_exit_timeout_sec))); if (s == future_status::ready) { LOG_INFO(sLogger, ("processor runner", "stopped successfully")("threadNo", threadNo)); } else { diff --git a/core/runner/sink/http/HttpSink.cpp b/core/runner/sink/http/HttpSink.cpp index 1cef22d257..723edf542f 100644 --- a/core/runner/sink/http/HttpSink.cpp +++ b/core/runner/sink/http/HttpSink.cpp @@ -25,7 +25,7 @@ #include "pipeline/queue/SenderQueueItem.h" #include "runner/FlusherRunner.h" -DEFINE_FLAG_INT32(http_sink_exit_timeout_secs, "", 5); +DEFINE_FLAG_INT32(http_sink_exit_timeout_sec, "", 5); using namespace std; @@ -65,7 +65,7 @@ void HttpSink::Stop() { if (!mThreadRes.valid()) { return; } - future_status s = mThreadRes.wait_for(chrono::seconds(INT32_FLAG(http_sink_exit_timeout_secs))); + future_status s = mThreadRes.wait_for(chrono::seconds(INT32_FLAG(http_sink_exit_timeout_sec))); if (s == future_status::ready) { LOG_INFO(sLogger, ("http sink", "stopped successfully")); } else { diff --git a/core/unittest/flusher/FlusherSLSUnittest.cpp b/core/unittest/flusher/FlusherSLSUnittest.cpp index ae974fae33..a52e386ff8 100644 --- a/core/unittest/flusher/FlusherSLSUnittest.cpp +++ b/core/unittest/flusher/FlusherSLSUnittest.cpp @@ -770,7 +770,7 @@ void FlusherSLSUnittest::TestBuildRequest() { #else APSARA_TEST_EQUAL(443, req->mPort); #endif - APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_timeout_secs)), req->mTimeout); + APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_timeout_sec)), req->mTimeout); APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_max_try_cnt)), req->mMaxTryCnt); APSARA_TEST_FALSE(req->mFollowRedirects); APSARA_TEST_EQUAL(&item, req->mItem); @@ -833,7 +833,7 @@ void FlusherSLSUnittest::TestBuildRequest() { #else APSARA_TEST_EQUAL(443, req->mPort); #endif - APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_timeout_secs)), req->mTimeout); + APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_timeout_sec)), req->mTimeout); APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_max_try_cnt)), req->mMaxTryCnt); APSARA_TEST_FALSE(req->mFollowRedirects); APSARA_TEST_EQUAL(&item, req->mItem); @@ -896,7 +896,7 @@ void FlusherSLSUnittest::TestBuildRequest() { #else APSARA_TEST_EQUAL(443, req->mPort); #endif - APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_timeout_secs)), req->mTimeout); + APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_timeout_sec)), req->mTimeout); APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_max_try_cnt)), req->mMaxTryCnt); APSARA_TEST_FALSE(req->mFollowRedirects); APSARA_TEST_EQUAL(&item, req->mItem); @@ -965,7 +965,7 @@ void FlusherSLSUnittest::TestBuildRequest() { #else APSARA_TEST_EQUAL(443, req->mPort); #endif - APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_timeout_secs)), req->mTimeout); + APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_timeout_sec)), req->mTimeout); APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_max_try_cnt)), req->mMaxTryCnt); APSARA_TEST_FALSE(req->mFollowRedirects); APSARA_TEST_EQUAL(&item, req->mItem); @@ -1023,7 +1023,7 @@ void FlusherSLSUnittest::TestBuildRequest() { #else APSARA_TEST_EQUAL(443, req->mPort); #endif - APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_timeout_secs)), req->mTimeout); + APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_timeout_sec)), req->mTimeout); APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_max_try_cnt)), req->mMaxTryCnt); APSARA_TEST_FALSE(req->mFollowRedirects); APSARA_TEST_EQUAL(&item, req->mItem); @@ -1079,7 +1079,7 @@ void FlusherSLSUnittest::TestBuildRequest() { APSARA_TEST_EQUAL(body, req->mBody); APSARA_TEST_EQUAL("192.168.0.1", req->mHost); APSARA_TEST_EQUAL(80, req->mPort); - APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_timeout_secs)), req->mTimeout); + APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_timeout_sec)), req->mTimeout); APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_max_try_cnt)), req->mMaxTryCnt); APSARA_TEST_FALSE(req->mFollowRedirects); APSARA_TEST_EQUAL(&item, req->mItem); @@ -1106,7 +1106,7 @@ void FlusherSLSUnittest::TestBuildRequest() { APSARA_TEST_EQUAL(body, req->mBody); APSARA_TEST_EQUAL("test_project." + kAccelerationDataEndpoint, req->mHost); APSARA_TEST_EQUAL(80, req->mPort); - APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_timeout_secs)), req->mTimeout); + APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_timeout_sec)), req->mTimeout); APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_max_try_cnt)), req->mMaxTryCnt); APSARA_TEST_FALSE(req->mFollowRedirects); APSARA_TEST_EQUAL(&item, req->mItem); From 9c499ef90e9a619b3c94a0bf931f88367c24f406 Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Thu, 26 Dec 2024 03:52:50 +0000 Subject: [PATCH 29/41] polish --- core/pipeline/GlobalConfig.cpp | 2 +- core/unittest/pipeline/GlobalConfigUnittest.cpp | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/core/pipeline/GlobalConfig.cpp b/core/pipeline/GlobalConfig.cpp index 15e98c10bd..91a3ed43bd 100644 --- a/core/pipeline/GlobalConfig.cpp +++ b/core/pipeline/GlobalConfig.cpp @@ -94,7 +94,7 @@ bool GlobalConfig::Init(const Json::Value& config, const PipelineContext& ctx, J } // Priority - uint32_t priority = 0; + uint32_t priority = 1; if (!GetOptionalUIntParam(config, "Priority", priority, errorMsg)) { PARAM_WARNING_DEFAULT(ctx.GetLogger(), ctx.GetAlarm(), diff --git a/core/unittest/pipeline/GlobalConfigUnittest.cpp b/core/unittest/pipeline/GlobalConfigUnittest.cpp index 7c7bc555bd..d646d17251 100644 --- a/core/unittest/pipeline/GlobalConfigUnittest.cpp +++ b/core/unittest/pipeline/GlobalConfigUnittest.cpp @@ -43,6 +43,8 @@ void GlobalConfigUnittest::OnSuccessfulInit() const { // only mandatory param config.reset(new GlobalConfig()); + APSARA_TEST_TRUE(config->Init(Json::Value(Json::ValueType::objectValue), ctx, extendedParams)); + APSARA_TEST_TRUE(extendedParams.isNull()); APSARA_TEST_EQUAL(GlobalConfig::TopicType::NONE, config->mTopicType); APSARA_TEST_EQUAL("", config->mTopicFormat); APSARA_TEST_EQUAL(1U, config->mPriority); From 7ec64fc2d610aafd1d2760a666ae9dad238a702c Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Fri, 27 Dec 2024 08:27:38 +0000 Subject: [PATCH 30/41] polish --- core/monitor/Monitor.cpp | 1 - core/plugin/flusher/sls/DiskBufferWriter.cpp | 12 ++++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/core/monitor/Monitor.cpp b/core/monitor/Monitor.cpp index 015c9690d4..b00bbb9594 100644 --- a/core/monitor/Monitor.cpp +++ b/core/monitor/Monitor.cpp @@ -25,7 +25,6 @@ #include "app_config/AppConfig.h" #include "application/Application.h" #include "common/DevInode.h" -#include "common/EncodingUtil.h" #include "common/ExceptionBase.h" #include "common/LogtailCommonFlags.h" #include "common/MachineInfoUtil.h" diff --git a/core/plugin/flusher/sls/DiskBufferWriter.cpp b/core/plugin/flusher/sls/DiskBufferWriter.cpp index 1d9d202802..f613da5f30 100644 --- a/core/plugin/flusher/sls/DiskBufferWriter.cpp +++ b/core/plugin/flusher/sls/DiskBufferWriter.cpp @@ -76,8 +76,12 @@ static sls_logs::EndpointMode GetEndpointMode(EndpointMode mode) { } return sls_logs::EndpointMode::DEFAULT; } + +static const string kAKErrorMsg = "can not get valid access key"; #endif +static const string kNoHostErrorMsg = "can not get available host"; + static const string& GetSLSCompressTypeString(sls_logs::SlsCompressType compressType) { switch (compressType) { case sls_logs::SLS_CMP_NONE: { @@ -545,7 +549,7 @@ void DiskBufferWriter::SendEncryptionBuffer(const std::string& filename, int32_t break; case SEND_NETWORK_ERROR: case SEND_SERVER_ERROR: - if (response.mErrorMsg != "can not get available host") { + if (response.mErrorMsg != kNoHostErrorMsg) { LOG_WARNING( sLogger, ("send data to SLS fail", "retry later")("request id", response.mRequestId)( @@ -583,7 +587,7 @@ void DiskBufferWriter::SendEncryptionBuffer(const std::string& filename, int32_t #ifdef __ENTERPRISE__ if (sendRes != SEND_NETWORK_ERROR && sendRes != SEND_SERVER_ERROR) { bool hasAuthError - = sendRes == SEND_UNAUTHORIZED && response.mErrorMsg != "can not get valid access key"; + = sendRes == SEND_UNAUTHORIZED && response.mErrorMsg != kAKErrorMsg; EnterpriseSLSClientManager::GetInstance()->UpdateAccessKeyStatus(bufferMeta.aliuid(), !hasAuthError); EnterpriseSLSClientManager::GetInstance()->UpdateProjectAnonymousWriteStatus( @@ -829,7 +833,7 @@ SLSResponse DiskBufferWriter::SendBufferFileData(const sls_logs::LogtailBufferMe bufferMeta.project(), type, accessKeyId, accessKeySecret)) { SLSResponse response; response.mErrorCode = LOGE_UNAUTHORIZED; - response.mErrorMsg = "can not get valid access key"; + response.mErrorMsg = kAKErrorMsg; return response; } #endif @@ -848,7 +852,7 @@ SLSResponse DiskBufferWriter::SendBufferFileData(const sls_logs::LogtailBufferMe if (host.empty()) { SLSResponse response; response.mErrorCode = LOGE_REQUEST_ERROR; - response.mErrorMsg = "can not get available host"; + response.mErrorMsg = kNoHostErrorMsg; return response; } #else From 23ba530f4b75c8abf229a9954b48140188bbd163 Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Fri, 27 Dec 2024 09:35:41 +0000 Subject: [PATCH 31/41] polish --- core/plugin/flusher/sls/FlusherSLS.cpp | 52 +++++++++++++++++++------ core/runner/sink/http/HttpSinkRequest.h | 2 +- 2 files changed, 42 insertions(+), 12 deletions(-) diff --git a/core/plugin/flusher/sls/FlusherSLS.cpp b/core/plugin/flusher/sls/FlusherSLS.cpp index c9fe1274e6..a236c6ec61 100644 --- a/core/plugin/flusher/sls/FlusherSLS.cpp +++ b/core/plugin/flusher/sls/FlusherSLS.cpp @@ -132,12 +132,14 @@ shared_ptr FlusherSLS::GetLogstoreConcurrencyLimiter(const s auto iter = sLogstoreConcurrencyLimiterMap.find(key); if (iter == sLogstoreConcurrencyLimiterMap.end()) { - auto limiter = make_shared(sName + "#quota#logstore#" + key, AppConfig::GetInstance()->GetSendRequestConcurrency()); + auto limiter = make_shared(sName + "#quota#logstore#" + key, + AppConfig::GetInstance()->GetSendRequestConcurrency()); sLogstoreConcurrencyLimiterMap.try_emplace(key, limiter); return limiter; } if (iter->second.expired()) { - auto limiter = make_shared(sName + "#quota#logstore#" + key, AppConfig::GetInstance()->GetSendRequestConcurrency()); + auto limiter = make_shared(sName + "#quota#logstore#" + key, + AppConfig::GetInstance()->GetSendRequestConcurrency()); iter->second = limiter; return limiter; } @@ -148,12 +150,14 @@ shared_ptr FlusherSLS::GetProjectConcurrencyLimiter(const st lock_guard lock(sMux); auto iter = sProjectConcurrencyLimiterMap.find(project); if (iter == sProjectConcurrencyLimiterMap.end()) { - auto limiter = make_shared(sName + "#quota#project#" + project, AppConfig::GetInstance()->GetSendRequestConcurrency()); + auto limiter = make_shared(sName + "#quota#project#" + project, + AppConfig::GetInstance()->GetSendRequestConcurrency()); sProjectConcurrencyLimiterMap.try_emplace(project, limiter); return limiter; } if (iter->second.expired()) { - auto limiter = make_shared(sName + "#quota#project#" + project, AppConfig::GetInstance()->GetSendRequestConcurrency()); + auto limiter = make_shared(sName + "#quota#project#" + project, + AppConfig::GetInstance()->GetSendRequestConcurrency()); iter->second = limiter; return limiter; } @@ -164,12 +168,20 @@ shared_ptr FlusherSLS::GetRegionConcurrencyLimiter(const str lock_guard lock(sMux); auto iter = sRegionConcurrencyLimiterMap.find(region); if (iter == sRegionConcurrencyLimiterMap.end()) { - auto limiter = make_shared(sName + "#network#region#" + region, AppConfig::GetInstance()->GetSendRequestConcurrency(), AppConfig::GetInstance()->GetSendRequestConcurrency()*AppConfig::GetInstance()->GetGlobalConcurrencyFreePercentageForOneRegion()); + auto limiter = make_shared( + sName + "#network#region#" + region, + AppConfig::GetInstance()->GetSendRequestConcurrency(), + AppConfig::GetInstance()->GetSendRequestConcurrency() + * AppConfig::GetInstance()->GetGlobalConcurrencyFreePercentageForOneRegion()); sRegionConcurrencyLimiterMap.try_emplace(region, limiter); return limiter; } if (iter->second.expired()) { - auto limiter = make_shared(sName + "#network#region#" + region, AppConfig::GetInstance()->GetSendRequestConcurrency(), AppConfig::GetInstance()->GetSendRequestConcurrency()*AppConfig::GetInstance()->GetGlobalConcurrencyFreePercentageForOneRegion()); + auto limiter = make_shared( + sName + "#network#region#" + region, + AppConfig::GetInstance()->GetSendRequestConcurrency(), + AppConfig::GetInstance()->GetSendRequestConcurrency() + * AppConfig::GetInstance()->GetGlobalConcurrencyFreePercentageForOneRegion()); iter->second = limiter; return limiter; } @@ -889,7 +901,7 @@ void FlusherSLS::OnSendDone(const HttpResponse& response, SenderQueueItem* item) EnterpriseSLSClientManager::GetInstance()->UpdateHostStatus( mProject, mCandidateHostsInfo->GetMode(), data->mCurrentHost, !hasNetworkError); mCandidateHostsInfo->SelectBestHost(); - + if (!hasNetworkError) { bool hasAuthError = sendResult == SEND_UNAUTHORIZED; EnterpriseSLSClientManager::GetInstance()->UpdateAccessKeyStatus(mAliuid, !hasAuthError); @@ -1188,8 +1200,17 @@ unique_ptr FlusherSLS::CreatePostLogStoreLogsRequest(const stri query, header); bool httpsFlag = SLSClientManager::GetInstance()->UsingHttps(mRegion); - return make_unique( - HTTP_POST, httpsFlag, item->mCurrentHost, httpsFlag ? 443 : 80, path, query, header, item->mData, item); + return make_unique(HTTP_POST, + httpsFlag, + item->mCurrentHost, + httpsFlag ? 443 : 80, + path, + query, + header, + item->mData, + item, + INT32_FLAG(default_http_request_timeout_sec), + 1); } unique_ptr FlusherSLS::CreatePostMetricStoreLogsRequest(const string& accessKeyId, @@ -1211,8 +1232,17 @@ unique_ptr FlusherSLS::CreatePostMetricStoreLogsRequest(const s path, header); bool httpsFlag = SLSClientManager::GetInstance()->UsingHttps(mRegion); - return make_unique( - HTTP_POST, httpsFlag, item->mCurrentHost, httpsFlag ? 443 : 80, path, "", header, item->mData, item); + return make_unique(HTTP_POST, + httpsFlag, + item->mCurrentHost, + httpsFlag ? 443 : 80, + path, + "", + header, + item->mData, + item, + INT32_FLAG(default_http_request_timeout_sec), + 1); } sls_logs::SlsCompressType ConvertCompressType(CompressType type) { diff --git a/core/runner/sink/http/HttpSinkRequest.h b/core/runner/sink/http/HttpSinkRequest.h index f8220f7722..d9ed07ce41 100644 --- a/core/runner/sink/http/HttpSinkRequest.h +++ b/core/runner/sink/http/HttpSinkRequest.h @@ -33,7 +33,7 @@ struct HttpSinkRequest : public AsynHttpRequest { const std::map& header, const std::string& body, SenderQueueItem* item, - uint32_t timeout = static_cast(INT32_FLAG(default_http_request_timeout_secs)), + uint32_t timeout = static_cast(INT32_FLAG(default_http_request_timeout_sec)), uint32_t maxTryCnt = static_cast(INT32_FLAG(default_http_request_max_try_cnt)) ) : AsynHttpRequest(method, httpsFlag, host, port, url, query, header, body, HttpResponse(), timeout, maxTryCnt), mItem(item) {} From 9d61d3e84d5a565ed4a9ae3c81dca68a8761ee12 Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Fri, 27 Dec 2024 09:53:19 +0000 Subject: [PATCH 32/41] polish --- core/plugin/flusher/sls/FlusherSLS.cpp | 2 +- core/unittest/models/MetricEventUnittest.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/plugin/flusher/sls/FlusherSLS.cpp b/core/plugin/flusher/sls/FlusherSLS.cpp index a236c6ec61..faa9b7f7c7 100644 --- a/core/plugin/flusher/sls/FlusherSLS.cpp +++ b/core/plugin/flusher/sls/FlusherSLS.cpp @@ -652,7 +652,7 @@ bool FlusherSLS::BuildRequest(SenderQueueItem* item, unique_ptr } if (data->mCurrentHost.empty()) { if (mCandidateHostsInfo->IsInitialized()) { - GetRegionConcurrencyLimiter(mRegion)->OnFail(); + GetRegionConcurrencyLimiter(mRegion)->OnFail(chrono::system_clock::now()); } *errMsg = "failed to get available host"; *keepItem = true; diff --git a/core/unittest/models/MetricEventUnittest.cpp b/core/unittest/models/MetricEventUnittest.cpp index ada8af3239..b148cce39b 100644 --- a/core/unittest/models/MetricEventUnittest.cpp +++ b/core/unittest/models/MetricEventUnittest.cpp @@ -346,7 +346,7 @@ void MetricEventUnittest::TestTagsIterator() { void MetricEventUnittest::TestCopy() { MetricEvent* oldMetricEvent = mEventGroup->AddMetricEvent(); oldMetricEvent->SetValue(map{{"test-1", 10.0}, {"test-2", 2.0}}); - APSARA_TEST_EQUAL(1, mEventGroup->GetEvents().size()); + APSARA_TEST_EQUAL(1U, mEventGroup->GetEvents().size()); PipelineEventGroup newGroup = mEventGroup->Copy(); MetricEvent newMetricEvent = newGroup.GetEvents().at(0).Cast(); From b916dd7c45a67d068efa1c42503ceca0d051510b Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Sat, 28 Dec 2024 03:57:07 +0000 Subject: [PATCH 33/41] polish --- core/unittest/flusher/FlusherSLSUnittest.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/core/unittest/flusher/FlusherSLSUnittest.cpp b/core/unittest/flusher/FlusherSLSUnittest.cpp index a52e386ff8..f968f9f007 100644 --- a/core/unittest/flusher/FlusherSLSUnittest.cpp +++ b/core/unittest/flusher/FlusherSLSUnittest.cpp @@ -771,7 +771,7 @@ void FlusherSLSUnittest::TestBuildRequest() { APSARA_TEST_EQUAL(443, req->mPort); #endif APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_timeout_sec)), req->mTimeout); - APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_max_try_cnt)), req->mMaxTryCnt); + APSARA_TEST_EQUAL(1U, req->mMaxTryCnt); APSARA_TEST_FALSE(req->mFollowRedirects); APSARA_TEST_EQUAL(&item, req->mItem); APSARA_TEST_FALSE(item.mRealIpFlag); @@ -834,7 +834,7 @@ void FlusherSLSUnittest::TestBuildRequest() { APSARA_TEST_EQUAL(443, req->mPort); #endif APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_timeout_sec)), req->mTimeout); - APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_max_try_cnt)), req->mMaxTryCnt); + APSARA_TEST_EQUAL(1U, req->mMaxTryCnt); APSARA_TEST_FALSE(req->mFollowRedirects); APSARA_TEST_EQUAL(&item, req->mItem); #ifdef __ENTERPRISE__ @@ -897,7 +897,7 @@ void FlusherSLSUnittest::TestBuildRequest() { APSARA_TEST_EQUAL(443, req->mPort); #endif APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_timeout_sec)), req->mTimeout); - APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_max_try_cnt)), req->mMaxTryCnt); + APSARA_TEST_EQUAL(1U, req->mMaxTryCnt); APSARA_TEST_FALSE(req->mFollowRedirects); APSARA_TEST_EQUAL(&item, req->mItem); APSARA_TEST_FALSE(item.mRealIpFlag); @@ -966,7 +966,7 @@ void FlusherSLSUnittest::TestBuildRequest() { APSARA_TEST_EQUAL(443, req->mPort); #endif APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_timeout_sec)), req->mTimeout); - APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_max_try_cnt)), req->mMaxTryCnt); + APSARA_TEST_EQUAL(1U, req->mMaxTryCnt); APSARA_TEST_FALSE(req->mFollowRedirects); APSARA_TEST_EQUAL(&item, req->mItem); APSARA_TEST_FALSE(item.mRealIpFlag); @@ -1024,7 +1024,7 @@ void FlusherSLSUnittest::TestBuildRequest() { APSARA_TEST_EQUAL(443, req->mPort); #endif APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_timeout_sec)), req->mTimeout); - APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_max_try_cnt)), req->mMaxTryCnt); + APSARA_TEST_EQUAL(1U, req->mMaxTryCnt); APSARA_TEST_FALSE(req->mFollowRedirects); APSARA_TEST_EQUAL(&item, req->mItem); APSARA_TEST_FALSE(item.mRealIpFlag); @@ -1080,7 +1080,7 @@ void FlusherSLSUnittest::TestBuildRequest() { APSARA_TEST_EQUAL("192.168.0.1", req->mHost); APSARA_TEST_EQUAL(80, req->mPort); APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_timeout_sec)), req->mTimeout); - APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_max_try_cnt)), req->mMaxTryCnt); + APSARA_TEST_EQUAL(1U, req->mMaxTryCnt); APSARA_TEST_FALSE(req->mFollowRedirects); APSARA_TEST_EQUAL(&item, req->mItem); APSARA_TEST_TRUE(item.mRealIpFlag); @@ -1107,7 +1107,7 @@ void FlusherSLSUnittest::TestBuildRequest() { APSARA_TEST_EQUAL("test_project." + kAccelerationDataEndpoint, req->mHost); APSARA_TEST_EQUAL(80, req->mPort); APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_timeout_sec)), req->mTimeout); - APSARA_TEST_EQUAL(static_cast(INT32_FLAG(default_http_request_max_try_cnt)), req->mMaxTryCnt); + APSARA_TEST_EQUAL(1U, req->mMaxTryCnt); APSARA_TEST_FALSE(req->mFollowRedirects); APSARA_TEST_EQUAL(&item, req->mItem); APSARA_TEST_FALSE(item.mRealIpFlag); From bc0cd0dc25d662d70a06ed8a7035d3e60921777c Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Sat, 28 Dec 2024 04:42:26 +0000 Subject: [PATCH 34/41] polish --- core/common/StringPiece.h | 315 -------------------------------------- 1 file changed, 315 deletions(-) delete mode 100644 core/common/StringPiece.h diff --git a/core/common/StringPiece.h b/core/common/StringPiece.h deleted file mode 100644 index 9e59362249..0000000000 --- a/core/common/StringPiece.h +++ /dev/null @@ -1,315 +0,0 @@ -/* - * Copyright 2022 iLogtail Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include -#include - -namespace logtail { -namespace detail { - - template - class StringPieceDetail { - public: - // for standard STL container - typedef size_t size_type; - typedef CharT value_type; - typedef const value_type* pointer; - typedef const value_type& reference; - typedef const value_type& const_reference; - typedef ptrdiff_t difference_type; - typedef const value_type* const_iterator; - typedef std::reverse_iterator const_reverse_iterator; - - static const size_type npos; - - typedef std::basic_string string_type; - typedef typename string_type::traits_type traits_type; - - public: - StringPieceDetail(const value_type* str = NULL) - : mStr(str), mLength((str == NULL) ? 0 : traits_type::length(str)) {} - StringPieceDetail(const string_type& str) : mStr(str.data()), mLength(str.size()) {} - StringPieceDetail(const value_type* str, size_type len) : mStr(str), mLength(len) {} - StringPieceDetail(const typename string_type::const_iterator& begin, - const typename string_type::const_iterator& end) - : mStr((end > begin) ? &(*begin) : NULL), mLength((end > begin) ? (size_type)(end - begin) : 0) {} - - bool operator==(const StringPieceDetail& s) const { return this == &s || this->compare(s) == 0; } - - // Iterators - const_iterator begin() const { return data(); } - const_iterator end() const { return data() + size(); } - const_reverse_iterator rbegin() const { return const_reverse_iterator(data() + size()); } - const_reverse_iterator rend() const { return const_reverse_iterator(data()); } - - // Capacity - size_type size() const { return mLength; } - size_type length() const { return size(); } - size_type max_size() const { return size(); } - size_type capacity() const { return size(); } - void clear() { - mStr = NULL; - mLength = 0; - } - bool empty() const { return size() == 0; } - - // Element access - value_type operator[](size_type pos) const { return data()[pos]; } - value_type at(size_type pos) const { - if (pos >= size()) { - throw std::out_of_range("pos is out of range"); - } - return data()[pos]; - } - - // Modifiers - void swap(StringPieceDetail& sp) { - std::swap(sp.mStr, mStr); - std::swap(sp.mLength, mLength); - } - - // String Operations - const value_type* c_str() const { return mStr; } - // data() may return a pointer to a buffer with embedded NULs, and the - // returned buffer may or may not be null terminated. Therefore it is - // typically a mistake to pass data() to a routine that expects a NUL - // terminated string. - const value_type* data() const { return mStr; } - - // find - size_type find(const StringPieceDetail& sp, size_type pos = 0) const { return find(sp.data(), pos, sp.size()); } - size_type find(const value_type* s, size_type pos = 0) const { - StringPieceDetail sp(s); - return find(sp, pos); - } - size_type find(const value_type* s, size_type pos, size_type n) const { - size_type ret = npos; - const size_type size = this->size(); - if (pos <= size && n <= size - pos) { - const value_type* data = this->data(); - const value_type* p = std::search(data + pos, data + size, s, s + n); - if (p != data + size || n == 0) { - ret = p - data; - } - } - return ret; - } - size_type find(const value_type c, size_type pos = 0) const { - size_type ret = npos; - const size_type size = this->size(); - if (pos < size) { - const value_type* data = this->data(); - const size_type n = size - pos; - const value_type* p = traits_type::find(data + pos, n, c); - if (p != 0) { - ret = p - data; - } - } - return ret; - } - // rfind - size_type rfind(const StringPieceDetail& sp, size_type pos = npos) const { - return rfind(sp.data(), pos, sp.size()); - } - size_type rfind(const value_type* s, size_type pos = npos) const { - StringPieceDetail sp(s); - return rfind(sp, pos); - } - size_type rfind(const value_type* s, size_type pos, size_type n) const { - const size_type size = this->size(); - if (n <= size) { - pos = std::min(size_type(size - n), pos); - const value_type* data = this->data(); - do { - if (traits_type::compare(data + pos, s, n) == 0) { - return pos; - } - } while (pos-- > 0); - } - return npos; - } - size_type rfind(const value_type c, size_type pos = npos) const { - size_type size = this->size(); - if (size) { - if (--size > pos) { - size = pos; - } - for (++size; size-- > 0;) { - if (traits_type::eq(data()[size], c)) { - return size; - } - } - } - return npos; - } - // find_first_of - size_type find_first_of(const StringPieceDetail& s, size_type pos = 0) const { - return find_first_of(s.data(), pos, s.size()); - } - size_type find_first_of(const value_type* s, size_type pos = 0) const { - StringPieceDetail sp(s); - return find_first_of(sp, pos); - } - size_type find_first_of(const value_type* s, size_type pos, size_type n) const { - for (; n && pos < size(); ++pos) { - const value_type* p = traits_type::find(s, n, data()[pos]); - if (p) { - return pos; - } - } - return npos; - } - size_type find_first_of(value_type c, size_type pos = 0) const { return find(c, pos); } - // find_last_of - size_type find_last_of(const StringPieceDetail& s, size_type pos = npos) const { - return find_last_of(s.data(), pos, s.size()); - } - size_type find_last_of(const value_type* s, size_type pos = npos) const { - StringPieceDetail sp(s); - return find_last_of(sp, pos); - } - size_type find_last_of(const value_type* s, size_type pos, size_type n) const { - size_type index = size(); - if (index && n) { - if (--index > pos) { - index = pos; - } - do { - if (traits_type::find(s, n, data()[index])) { - return index; - } - } while (index-- != 0); - } - return npos; - } - size_type find_last_of(value_type c, size_type pos = npos) const { return rfind(c, pos); } - // find_first_not_of - size_type find_first_not_of(const StringPieceDetail& s, size_type pos = 0) const { - return find_first_not_of(s.data(), pos, s.size()); - } - size_type find_first_not_of(const value_type* s, size_type pos = 0) const { - StringPieceDetail sp(s); - return find_first_not_of(sp, pos); - } - size_type find_first_not_of(const value_type* s, size_type pos, size_type n) const { - for (; pos < size(); ++pos) { - if (!traits_type::find(s, n, data()[pos])) { - return pos; - } - } - return npos; - } - size_type find_first_not_of(value_type c, size_type pos = 0) const { - StringPieceDetail sp(&c, 1); - return find_first_not_of(sp, pos); - } - // find_last_not_of - size_type find_last_not_of(const StringPieceDetail& s, size_type pos = npos) const { - return find_last_not_of(s.data(), pos, s.size()); - } - size_type find_last_not_of(const value_type* s, size_type pos = npos) const { - StringPieceDetail sp(s); - return find_last_not_of(s, pos); - } - size_type find_last_not_of(const value_type* s, size_type pos, size_type n) const { - size_type index = size(); - if (index) { - if (--index > pos) { - index = pos; - } - do { - if (!traits_type::find(s, n, data()[index])) { - return index; - } - } while (index--); - } - return npos; - } - size_type find_last_not_of(value_type c, size_type pos = npos) const { - StringPieceDetail sp(&c, 1); - return find_last_not_of(sp, pos); - } - - void set(const value_type* data, size_type len) { - mStr = data; - mLength = len; - } - void set(const value_type* str) { - mStr = str; - mLength = str ? traits_type::length(str) : 0; - } - void set(const string_type& str) { - mStr = str.data(); - mLength = str.size(); - } - - void remove_prefix(size_type n) { - if (mLength < n) { - throw std::out_of_range("invalid parameter"); - } - mStr += n; - mLength -= n; - } - void remove_suffix(size_type n) { - if (mLength < n) { - throw std::out_of_range("invalid parameter"); - } - mLength -= n; - } - - StringPieceDetail substr(size_t pos = 0, size_t len = npos) const { - if (pos > size()) { - throw std::out_of_range("pos is out of range"); - } - const value_type* p = data() + pos; - if (len == npos) { - len = size() - pos; - } else { - len = std::min(size() - pos, len); - } - return StringPieceDetail(p, len); - } - int compare(const StringPieceDetail& s) const { - const size_type this_size = size(); - const size_type other_size = s.size(); - const size_type len = std::min(this_size, other_size); - - int r = traits_type::compare(data(), s.data(), len); - if (r == 0) { - r = this_size - other_size; - } - return r; - } - - string_type as_string() const { return empty() ? string_type() : string_type(data(), size()); } - - private: - const value_type* mStr; - size_type mLength; - }; - - template - const size_t StringPieceDetail::npos = -1; - -} // namespace detail - -typedef detail::StringPieceDetail StringPiece; -// typedef detail::StringPieceDetail StringPiece16; - -} // namespace logtail From 842453b00e046f7ccfedf11313b6c294c62e6272 Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Sat, 28 Dec 2024 05:45:09 +0000 Subject: [PATCH 35/41] polish --- core/app_config/AppConfig.h | 1 + core/unittest/sender/FlusherRunnerUnittest.cpp | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/core/app_config/AppConfig.h b/core/app_config/AppConfig.h index 72d8339123..bde8099bbf 100644 --- a/core/app_config/AppConfig.h +++ b/core/app_config/AppConfig.h @@ -534,6 +534,7 @@ class AppConfig { friend class InputContainerStdioUnittest; friend class BatcherUnittest; friend class EnterpriseSLSClientManagerUnittest; + friend class FlusherRunnerUnittest; #endif }; diff --git a/core/unittest/sender/FlusherRunnerUnittest.cpp b/core/unittest/sender/FlusherRunnerUnittest.cpp index 076b51fb07..9c2b751d66 100644 --- a/core/unittest/sender/FlusherRunnerUnittest.cpp +++ b/core/unittest/sender/FlusherRunnerUnittest.cpp @@ -31,6 +31,10 @@ class FlusherRunnerUnittest : public ::testing::Test { void TestPushToHttpSink(); protected: + static void SetUpTestCase() { + AppConfig::GetInstance()->mSendRequestGlobalConcurrency = 10; + } + void TearDown() override { SenderQueueManager::GetInstance()->Clear(); HttpSink::GetInstance()->mQueue.Clear(); @@ -47,8 +51,6 @@ void FlusherRunnerUnittest::TestDispatch() { flusher->SetMetricsRecordRef("name", "1"); flusher->Init(Json::Value(), tmp); - AppConfig::GetInstance()->mSendRequestGlobalConcurrency = 10; - auto item = make_unique("content", 10, flusher.get(), flusher->GetQueueKey()); auto realItem = item.get(); flusher->PushToQueue(std::move(item)); From 9d76a1e49551acef8732764c8ee7c8f2dce19ce7 Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Sat, 28 Dec 2024 05:53:41 +0000 Subject: [PATCH 36/41] polish --- core/unittest/flusher/FlusherSLSUnittest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/unittest/flusher/FlusherSLSUnittest.cpp b/core/unittest/flusher/FlusherSLSUnittest.cpp index f968f9f007..3639b7b44c 100644 --- a/core/unittest/flusher/FlusherSLSUnittest.cpp +++ b/core/unittest/flusher/FlusherSLSUnittest.cpp @@ -714,7 +714,7 @@ void FlusherSLSUnittest::TestBuildRequest() { APSARA_TEST_FALSE(flusher.BuildRequest(&item, req, &keepItem, &errMsg)); APSARA_TEST_EQUAL(nullptr, req); APSARA_TEST_TRUE(keepItem); - APSARA_TEST_EQUAL(static_cast(AppConfig::GetInstance()->GetSendRequestConcurrency()) / 2, + APSARA_TEST_EQUAL(static_cast(AppConfig::GetInstance()->GetSendRequestConcurrency()), FlusherSLS::GetRegionConcurrencyLimiter(flusher.mRegion)->GetCurrentLimit()); } EnterpriseSLSClientManager::GetInstance()->UpdateHostLatency("test_project", From b216e853a060e42743418c520dd8beae216b5edc Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Sat, 28 Dec 2024 07:11:30 +0000 Subject: [PATCH 37/41] polish --- core/common/CompressTools.cpp | 171 ---------------------------------- core/common/CompressTools.h | 28 +----- 2 files changed, 3 insertions(+), 196 deletions(-) diff --git a/core/common/CompressTools.cpp b/core/common/CompressTools.cpp index fc7d753ee5..4aacdfe487 100644 --- a/core/common/CompressTools.cpp +++ b/core/common/CompressTools.cpp @@ -15,141 +15,9 @@ #include "CompressTools.h" #include -#ifdef __ANDROID__ -#include -#else -#include -#endif -#include - -#include - -#include "protobuf/sls/sls_logs.pb.h" namespace logtail { -const int32_t ZSTD_DEFAULT_LEVEL = 1; - -bool UncompressData(sls_logs::SlsCompressType compressType, - const std::string& src, - uint32_t rawSize, - std::string& dst) { - switch (compressType) { - case sls_logs::SLS_CMP_NONE: - dst = src; - return true; - case sls_logs::SLS_CMP_LZ4: - return UncompressLz4(src, rawSize, dst); - case sls_logs::SLS_CMP_DEFLATE: - return UncompressDeflate(src, rawSize, dst); - case sls_logs::SLS_CMP_ZSTD: - return UncompressZstd(src, rawSize, dst); - default: - return false; - } -} - -bool CompressData(sls_logs::SlsCompressType compressType, const std::string& src, std::string& dst) { - switch (compressType) { - case sls_logs::SLS_CMP_NONE: - dst = src; - return true; - case sls_logs::SLS_CMP_LZ4: - return CompressLz4(src, dst); - case sls_logs::SLS_CMP_DEFLATE: - return CompressDeflate(src, dst); - case sls_logs::SLS_CMP_ZSTD: - return CompressZstd(src, dst, ZSTD_DEFAULT_LEVEL); - default: - return false; - } -} - -bool CompressData(sls_logs::SlsCompressType compressType, const char* src, uint32_t size, std::string& dst) { - switch (compressType) { - case sls_logs::SLS_CMP_NONE: { - dst.assign(src, size); - return true; - } - case sls_logs::SLS_CMP_LZ4: - return CompressLz4(src, size, dst); - case sls_logs::SLS_CMP_DEFLATE: - return CompressDeflate(src, size, dst); - case sls_logs::SLS_CMP_ZSTD: - return CompressZstd(src, size, dst, ZSTD_DEFAULT_LEVEL); - default: - return false; - } -} - -bool UncompressLz4(const std::string& src, const uint32_t rawSize, char* dst) { - uint32_t length = 0; - try { - length = LZ4_decompress_safe(src.c_str(), dst, src.length(), rawSize); - } catch (...) { - return false; - } - if (length != rawSize) { - return false; - } - return true; -} - -bool UncompressLz4(const char* srcPtr, const uint32_t srcSize, const uint32_t rawSize, std::string& dst) { - dst.resize(rawSize); - char* unCompressed = const_cast(dst.c_str()); - uint32_t length = 0; - try { - length = LZ4_decompress_safe(srcPtr, unCompressed, srcSize, rawSize); - } catch (...) { - return false; - } - if (length != rawSize) { - return false; - } - return true; -} -bool CompressDeflate(const char* srcPtr, const uint32_t srcSize, std::string& dst) { - int64_t dstLen = compressBound(srcSize); - dst.resize(dstLen); - if (compress((Bytef*)(dst.c_str()), (uLongf*)&dstLen, (const Bytef*)srcPtr, srcSize) == Z_OK) { - dst.resize(dstLen); - return true; - } - return false; -} - -bool CompressDeflate(const std::string& src, std::string& dst) { - int64_t dstLen = compressBound(src.size()); - dst.resize(dstLen); - if (compress((Bytef*)(dst.c_str()), (uLongf*)&dstLen, (const Bytef*)(src.c_str()), src.size()) == Z_OK) { - dst.resize(dstLen); - return true; - } - return false; -} - -bool UncompressDeflate(const char* srcPtr, const uint32_t srcSize, const int64_t rawSize, std::string& dst) { - static const int64_t MAX_UMCOMPRESS_SIZE = 128 * 1024 * 1024; - if (rawSize > MAX_UMCOMPRESS_SIZE) { - return false; - } - dst.resize(rawSize); - if (uncompress((Bytef*)(dst.c_str()), (uLongf*)&rawSize, (const Bytef*)(srcPtr), srcSize) != Z_OK) { - return false; - } - return true; -} - - -bool UncompressDeflate(const std::string& src, const int64_t rawSize, std::string& dst) { - return UncompressDeflate(src.c_str(), src.size(), rawSize, dst); -} - - -bool UncompressLz4(const std::string& src, const uint32_t rawSize, std::string& dst) { - return UncompressLz4(src.c_str(), src.length(), rawSize, dst); -} bool CompressLz4(const char* srcPtr, const uint32_t srcSize, std::string& dst) { uint32_t encodingSize = LZ4_compressBound(srcSize); dst.resize(encodingSize); @@ -169,43 +37,4 @@ bool CompressLz4(const std::string& src, std::string& dst) { return CompressLz4(src.c_str(), src.length(), dst); } -bool UncompressZstd(const std::string& src, const uint32_t rawSize, std::string& dst) { - return UncompressZstd(src.c_str(), src.length(), rawSize, dst); -} - -bool UncompressZstd(const char* srcPtr, const uint32_t srcSize, const uint32_t rawSize, std::string& dst) { - dst.resize(rawSize); - char* unCompressed = const_cast(dst.c_str()); - uint32_t length = 0; - try { - length = ZSTD_decompress(unCompressed, rawSize, srcPtr, srcSize); - } catch (...) { - return false; - } - if (length != rawSize) { - return false; - } - return true; -} - -bool CompressZstd(const char* srcPtr, const uint32_t srcSize, std::string& dst, int32_t level) { - uint32_t encodingSize = ZSTD_compressBound(srcSize); - dst.resize(encodingSize); - char* compressed = const_cast(dst.c_str()); - try { - size_t const cmp_size = ZSTD_compress(compressed, encodingSize, srcPtr, srcSize, level); - if (ZSTD_isError(cmp_size)) { - return false; - } - dst.resize(cmp_size); - return true; - } catch (...) { - } - return false; -} - -bool CompressZstd(const std::string& src, std::string& dst, int32_t level) { - return CompressZstd(src.c_str(), src.length(), dst, level); -} - } // namespace logtail diff --git a/core/common/CompressTools.h b/core/common/CompressTools.h index 9293150ff9..1ccf1fe041 100644 --- a/core/common/CompressTools.h +++ b/core/common/CompressTools.h @@ -15,36 +15,14 @@ */ #pragma once -#include -#include -#include "protobuf/sls/sls_logs.pb.h" - -namespace logtail { - -extern const int32_t ZSTD_DEFAULT_LEVEL; - -bool UncompressData(sls_logs::SlsCompressType compressType, const std::string& src, uint32_t rawSize, std::string& dst); -bool CompressData(sls_logs::SlsCompressType compressType, const std::string& src, std::string& dst); -bool CompressData(sls_logs::SlsCompressType compressType, const char* src, uint32_t size, std::string& dst); - -bool UncompressDeflate(const std::string& src, const int64_t rawSize, std::string& dst); -bool UncompressDeflate(const char* srcPtr, const uint32_t srcSize, const int64_t rawSize, std::string& dst); +#include -bool CompressDeflate(const std::string& src, std::string& dst); -bool CompressDeflate(const char* srcPtr, const uint32_t srcSize, std::string& dst); +#include -bool UncompressLz4(const std::string& src, const uint32_t rawSize, std::string& dst); -bool UncompressLz4(const std::string& src, const uint32_t rawSize, char* dst); -bool UncompressLz4(const char* srcPtr, const uint32_t srcSize, const uint32_t rawSize, std::string& dst); +namespace logtail { bool CompressLz4(const std::string& src, std::string& dst); bool CompressLz4(const char* srcPtr, const uint32_t srcSize, std::string& dest); -bool UncompressZstd(const std::string& src, const uint32_t rawSize, std::string& dst); -bool UncompressZstd(const char* srcPtr, const uint32_t srcSize, const uint32_t rawSize, std::string& dst); - -bool CompressZstd(const char* srcPtr, const uint32_t srcSize, std::string& dst, int32_t level); -bool CompressZstd(const std::string& src, std::string& dst, int32_t level); - } // namespace logtail \ No newline at end of file From 4aa1c7dabe0eff5b9d6176631375a318cb29cf20 Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Mon, 30 Dec 2024 06:21:50 +0000 Subject: [PATCH 38/41] polish --- core/plugin/flusher/sls/SLSClientManager.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/plugin/flusher/sls/SLSClientManager.cpp b/core/plugin/flusher/sls/SLSClientManager.cpp index 0eca46f717..be9e57291e 100644 --- a/core/plugin/flusher/sls/SLSClientManager.cpp +++ b/core/plugin/flusher/sls/SLSClientManager.cpp @@ -45,8 +45,8 @@ SLSClientManager* SLSClientManager::GetInstance() { #ifdef __ENTERPRISE__ return EnterpriseSLSClientManager::GetInstance(); #else - static auto ptr = unique_ptr(new SLSClientManager()); - return ptr.get(); + static SLSClientManager instance; + return &instance; #endif } From 4ac9671eb3215c6fa421894d0d1055a4365962bf Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Mon, 30 Dec 2024 07:45:27 +0000 Subject: [PATCH 39/41] polish --- core/unittest/pipeline/HttpSinkMock.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/unittest/pipeline/HttpSinkMock.h b/core/unittest/pipeline/HttpSinkMock.h index 7ab010bbc2..38a3a9882e 100644 --- a/core/unittest/pipeline/HttpSinkMock.h +++ b/core/unittest/pipeline/HttpSinkMock.h @@ -66,7 +66,7 @@ class HttpSinkMock : public HttpSink { } request->mResponse.SetNetworkStatus(NetworkCode::Ok, ""); request->mResponse.SetStatusCode(200); - request->mResponse.SetResponseTime(chrono::milliseconds(10)); + request->mResponse.SetResponseTime(std::chrono::milliseconds(10)); // for sls only request->mResponse.mHeader[X_LOG_REQUEST_ID] = "request_id"; static_cast(request->mItem->mFlusher)->OnSendDone(request->mResponse, request->mItem); From 848b7f37d4086d00ad13d36b04bda355972dd544 Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Mon, 30 Dec 2024 08:02:17 +0000 Subject: [PATCH 40/41] polish --- core/unittest/pipeline/PipelineUpdateUnittest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/unittest/pipeline/PipelineUpdateUnittest.cpp b/core/unittest/pipeline/PipelineUpdateUnittest.cpp index 6e5a4be888..a2e948853f 100644 --- a/core/unittest/pipeline/PipelineUpdateUnittest.cpp +++ b/core/unittest/pipeline/PipelineUpdateUnittest.cpp @@ -63,7 +63,7 @@ class FlusherSLSMock : public FlusherSLS { public: static const std::string sName; - bool BuildRequest(SenderQueueItem* item, std::unique_ptr& req, bool* keepItem) const override { + bool BuildRequest(SenderQueueItem* item, std::unique_ptr& req, bool* keepItem, std::string* errMsg) override { auto data = static_cast(item); std::map header; req = std::make_unique( From 8c3276f793fbc738b483deef97e9189ca80612aa Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Mon, 30 Dec 2024 08:19:10 +0000 Subject: [PATCH 41/41] polish --- core/unittest/pipeline/PipelineUpdateUnittest.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/core/unittest/pipeline/PipelineUpdateUnittest.cpp b/core/unittest/pipeline/PipelineUpdateUnittest.cpp index a2e948853f..1301e4046d 100644 --- a/core/unittest/pipeline/PipelineUpdateUnittest.cpp +++ b/core/unittest/pipeline/PipelineUpdateUnittest.cpp @@ -33,6 +33,9 @@ #include "unittest/pipeline/HttpSinkMock.h" #include "unittest/pipeline/LogtailPluginMock.h" #include "unittest/plugin/PluginMock.h" +#ifdef __ENTERPRISE__ +#include "config/provider/EnterpriseConfigProvider.h" +#endif using namespace std; @@ -372,10 +375,12 @@ class PipelineUpdateUnittest : public testing::Test { "Type": "flusher_stdout2" })"; - size_t builtinPipelineCnt = 0; + static size_t builtinPipelineCnt; bool isFileServerStart = false; }; +size_t PipelineUpdateUnittest::builtinPipelineCnt = 0; + void PipelineUpdateUnittest::TestFileServerStart() { isFileServerStart = true; Json::Value nativePipelineConfigJson