From 1738a58448cb3211cd79c644552b28fc5849df08 Mon Sep 17 00:00:00 2001 From: Lanzheng Liu Date: Thu, 16 May 2024 11:45:36 +0800 Subject: [PATCH 1/2] support customized UserAgent in http client (#479) Signed-off-by: liulanzheng --- net/http/client.cpp | 5 ++-- net/http/client.h | 4 +++ net/http/test/client_function_test.cpp | 40 ++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/net/http/client.cpp b/net/http/client.cpp index 475b579c..79964d79 100644 --- a/net/http/client.cpp +++ b/net/http/client.cpp @@ -224,9 +224,10 @@ class ClientImpl : public Client { LOG_ERROR_RETURN(EINVAL, ROUNDTRIP_FAILED, "Content-Length and Transfer-Encoding conflicted"); } - op->req.headers.insert("User-Agent", USERAGENT); - op->req.headers.insert("Connection", "keep-alive"); op->req.headers.merge(m_common_headers); + op->req.headers.insert("User-Agent", m_user_agent.empty() ? std::string_view(USERAGENT) + : std::string_view(m_user_agent)); + op->req.headers.insert("Connection", "keep-alive"); if (m_cookie_jar && m_cookie_jar->set_cookies_to_headers(&op->req) != 0) LOG_ERROR_RETURN(0, -1, "set_cookies_to_headers failed"); Timeout tmo(std::min(op->timeout.timeout(), m_timeout)); diff --git a/net/http/client.h b/net/http/client.h index 350766e3..e2b52ab6 100644 --- a/net/http/client.h +++ b/net/http/client.h @@ -109,6 +109,9 @@ class Client : public Object { m_proxy_url.from_string(proxy); m_proxy = true; } + void set_user_agent(std::string_view user_agent) { + m_user_agent = std::string(user_agent); + } StoredURL* get_proxy() { return &m_proxy_url; } @@ -129,6 +132,7 @@ class Client : public Object { bool secure = false, uint64_t timeout = -1UL) = 0; protected: StoredURL m_proxy_url; + std::string m_user_agent; uint64_t m_timeout = -1UL; bool m_proxy = false; }; diff --git a/net/http/test/client_function_test.cpp b/net/http/test/client_function_test.cpp index 1e81851a..2c9f5741 100644 --- a/net/http/test/client_function_test.cpp +++ b/net/http/test/client_function_test.cpp @@ -512,6 +512,46 @@ TEST(http_client, partial_body) { EXPECT_EQ(true, buf == "http_clien"); } +int ua_check_handler(void*, Request &req, Response &resp, std::string_view) { + auto ua = req.headers["User-Agent"]; + LOG_DEBUG(VALUE(ua)); + EXPECT_EQ(ua, "TEST_UA"); + resp.set_result(200); + std::string str = "success"; + resp.headers.content_length(7); + resp.write((void*)str.data(), str.size()); + return 0; +} + + +TEST(http_client, user_agent) { + auto tcpserver = new_tcp_socket_server(); + DEFER(delete tcpserver); + tcpserver->bind(18731); + tcpserver->listen(); + auto server = new_http_server(); + DEFER(delete server); + server->add_handler({nullptr, &ua_check_handler}); + tcpserver->set_handler(server->get_connection_handler()); + tcpserver->start_loop(); + + std::string target_get = "http://localhost:18731/file"; + auto client = new_http_client(); + client->set_user_agent("TEST_UA"); + DEFER(delete client); + auto op = client->new_operation(Verb::GET, target_get); + DEFER(delete op); + op->req.headers.content_length(0); + client->call(op); + EXPECT_EQ(op->status_code, 200); + std::string buf; + buf.resize(op->resp.headers.content_length()); + op->resp.read((void*)buf.data(), op->resp.headers.content_length()); + LOG_DEBUG(VALUE(buf)); + EXPECT_EQ(true, buf == "success"); +} + + TEST(url, url_escape_unescape) { EXPECT_EQ( url_escape("?a=x:b&b=cd&c= feg&d=2/1[+]@alibaba.com&e='!bad';"), From 33252d3091ec6950458528d2624942ec23d5ab4e Mon Sep 17 00:00:00 2001 From: Coldwings Date: Sat, 8 Jun 2024 11:15:55 +0800 Subject: [PATCH 2/2] FIX: lockfree MPMC queue should not fail to pop/push when queue is not empty/full (#504) * FIX: lockfree MPMC queue should not fail to pop/push when queue is not empty/full Signed-off-by: Coldwings * Old CI image not able to access repo, change to preset image Signed-off-by: Coldwings --------- Signed-off-by: Coldwings --- .github/workflows/ci.linux.arm.yml | 36 ++++++++++----------- .github/workflows/ci.linux.x86.yml | 52 +++++++++++++++--------------- .github/workflows/ci.macos.arm.yml | 12 +++---- common/lockfree_queue.h | 28 +++++----------- 4 files changed, 58 insertions(+), 70 deletions(-) diff --git a/.github/workflows/ci.linux.arm.yml b/.github/workflows/ci.linux.arm.yml index c7766b4b..98275b07 100644 --- a/.github/workflows/ci.linux.arm.yml +++ b/.github/workflows/ci.linux.arm.yml @@ -11,7 +11,7 @@ jobs: runs-on: [self-hosted, Linux, ARM64] container: - image: dokken/centos-stream-8:sha-40294ce + image: ghcr.io/coldwings/photon-ut-base:latest options: --cpus 4 steps: @@ -23,14 +23,14 @@ jobs: - uses: actions/checkout@v3 - - name: Install Dependencies - run: | - dnf install -y git gcc-c++ cmake 'dnf-command(config-manager)' - dnf install -y gcc-toolset-9-gcc-c++ - dnf install -y openssl-devel libcurl-devel libaio-devel - dnf install -y epel-release - dnf config-manager --set-enabled powertools - dnf install -y gtest-devel gmock-devel gflags-devel fuse-devel libgsasl-devel e2fsprogs-devel + # - name: Install Dependencies + # run: | + # dnf install -y git gcc-c++ cmake 'dnf-command(config-manager)' + # dnf install -y gcc-toolset-9-gcc-c++ + # dnf install -y openssl-devel libcurl-devel libaio-devel + # dnf install -y epel-release + # dnf config-manager --set-enabled powertools + # dnf install -y gtest-devel gmock-devel gflags-devel fuse-devel libgsasl-devel e2fsprogs-devel - name: Build run: | @@ -51,7 +51,7 @@ jobs: runs-on: [self-hosted, Linux, ARM64] container: - image: dokken/centos-stream-8:sha-40294ce + image: ghcr.io/coldwings/photon-ut-base:latest options: --cpus 4 steps: @@ -63,14 +63,14 @@ jobs: - uses: actions/checkout@v3 - - name: Install Dependencies - run: | - dnf install -y git gcc-c++ cmake 'dnf-command(config-manager)' - dnf install -y gcc-toolset-9-gcc-c++ - dnf install -y openssl-devel libcurl-devel libaio-devel - dnf install -y epel-release - dnf config-manager --set-enabled powertools - dnf install -y gtest-devel gmock-devel gflags-devel fuse-devel libgsasl-devel e2fsprogs-devel + # - name: Install Dependencies + # run: | + # dnf install -y git gcc-c++ cmake 'dnf-command(config-manager)' + # dnf install -y gcc-toolset-9-gcc-c++ + # dnf install -y openssl-devel libcurl-devel libaio-devel + # dnf install -y epel-release + # dnf config-manager --set-enabled powertools + # dnf install -y gtest-devel gmock-devel gflags-devel fuse-devel libgsasl-devel e2fsprogs-devel - name: Build run: | diff --git a/.github/workflows/ci.linux.x86.yml b/.github/workflows/ci.linux.x86.yml index 940ac7ca..ba225852 100644 --- a/.github/workflows/ci.linux.x86.yml +++ b/.github/workflows/ci.linux.x86.yml @@ -11,7 +11,7 @@ jobs: runs-on: [self-hosted, Linux, X64] container: - image: dokken/centos-stream-8:sha-40294ce + image: ghcr.io/coldwings/photon-ut-base:latest options: --cpus 4 steps: @@ -23,14 +23,14 @@ jobs: - uses: actions/checkout@v3 - - name: Install Dependencies - run: | - dnf install -y git gcc-c++ cmake 'dnf-command(config-manager)' - dnf install -y gcc-toolset-9-gcc-c++ - dnf install -y openssl-devel libcurl-devel libaio-devel - dnf install -y epel-release - dnf config-manager --set-enabled powertools - dnf install -y gtest-devel gmock-devel gflags-devel fuse-devel libgsasl-devel e2fsprogs-devel + # - name: Install Dependencies + # run: | + # dnf install -y git gcc-c++ cmake 'dnf-command(config-manager)' + # dnf install -y gcc-toolset-9-gcc-c++ + # dnf install -y openssl-devel libcurl-devel libaio-devel + # dnf install -y epel-release + # dnf config-manager --set-enabled powertools + # dnf install -y gtest-devel gmock-devel gflags-devel fuse-devel libgsasl-devel e2fsprogs-devel - name: Build run: | @@ -51,7 +51,7 @@ jobs: runs-on: [self-hosted, Linux, X64] container: - image: dokken/centos-stream-8:sha-40294ce + image: ghcr.io/coldwings/photon-ut-base:latest options: --cpus 4 steps: @@ -63,14 +63,14 @@ jobs: - uses: actions/checkout@v3 - - name: Install Dependencies - run: | - dnf install -y git gcc-c++ cmake 'dnf-command(config-manager)' - dnf install -y gcc-toolset-9-gcc-c++ - dnf install -y openssl-devel libcurl-devel libaio-devel - dnf install -y epel-release - dnf config-manager --set-enabled powertools - dnf install -y gtest-devel gmock-devel gflags-devel fuse-devel libgsasl-devel e2fsprogs-devel + # - name: Install Dependencies + # run: | + # dnf install -y git gcc-c++ cmake 'dnf-command(config-manager)' + # dnf install -y gcc-toolset-9-gcc-c++ + # dnf install -y openssl-devel libcurl-devel libaio-devel + # dnf install -y epel-release + # dnf config-manager --set-enabled powertools + # dnf install -y gtest-devel gmock-devel gflags-devel fuse-devel libgsasl-devel e2fsprogs-devel - name: Build run: | @@ -91,7 +91,7 @@ jobs: runs-on: [self-hosted, Linux, X64] container: - image: dokken/centos-stream-8:sha-40294ce + image: ghcr.io/coldwings/photon-ut-base:latest # In order to run io_uring, the docker daemon should add --default-ulimit memlock=-1:-1 options: --cpus 4 @@ -104,13 +104,13 @@ jobs: - uses: actions/checkout@v3 - - name: Install Dependencies - run: | - dnf install -y git gcc-c++ cmake - dnf install -y gcc-toolset-9-gcc-c++ - dnf install -y openssl-devel libcurl-devel - dnf install -y epel-release - dnf install -y fuse-devel libgsasl-devel e2fsprogs-devel + # - name: Install Dependencies + # run: | + # dnf install -y git gcc-c++ cmake + # dnf install -y gcc-toolset-9-gcc-c++ + # dnf install -y openssl-devel libcurl-devel + # dnf install -y epel-release + # dnf install -y fuse-devel libgsasl-devel e2fsprogs-devel - name: Build run: | diff --git a/.github/workflows/ci.macos.arm.yml b/.github/workflows/ci.macos.arm.yml index c8e17718..45558cab 100644 --- a/.github/workflows/ci.macos.arm.yml +++ b/.github/workflows/ci.macos.arm.yml @@ -8,14 +8,14 @@ on: jobs: macOS-clang-debug: - runs-on: [self-hosted, macOS, ARM64] + runs-on: macos-14 steps: -# - uses: szenius/set-timezone@v1.2 -# with: -# timezoneLinux: "Asia/Shanghai" -# timezoneMacos: "Asia/Shanghai" -# timezoneWindows: "China Standard Time" + - uses: szenius/set-timezone@v1.2 + with: + timezoneLinux: "Asia/Shanghai" + timezoneMacos: "Asia/Shanghai" + timezoneWindows: "China Standard Time" - uses: actions/checkout@v3 diff --git a/common/lockfree_queue.h b/common/lockfree_queue.h index 8bdcaac3..51ce0074 100644 --- a/common/lockfree_queue.h +++ b/common/lockfree_queue.h @@ -179,7 +179,7 @@ class LockfreeMPMCRingQueue : public LockfreeRingQueueBase { using Base::empty; using Base::full; - bool push_weak(const T& x) { + bool push(const T& x) { auto t = tail.load(std::memory_order_acquire); for (;;) { auto& slot = slots[idx(t)]; @@ -192,15 +192,16 @@ class LockfreeMPMCRingQueue : public LockfreeRingQueueBase { } } else { auto const prevTail = t; + auto h = head.load(std::memory_order_acquire); t = tail.load(std::memory_order_acquire); - if (t == prevTail) { + if (t == prevTail && Base::check_full(h, t)) { return false; } } } } - bool pop_weak(T& x) { + bool pop(T& x) { auto h = head.load(std::memory_order_acquire); for (;;) { auto& slot = slots[idx(h)]; @@ -213,28 +214,15 @@ class LockfreeMPMCRingQueue : public LockfreeRingQueueBase { } } else { auto const prevHead = h; + auto t = tail.load(std::memory_order_acquire); h = head.load(std::memory_order_acquire); - if (h == prevHead) { + if (h == prevHead && Base::check_empty(h, t)) { return false; } } } } - bool push(const T& x) { - do { - if (push_weak(x)) return true; - } while (!full()); - return false; - } - - bool pop(T& x) { - do { - if (pop_weak(x)) return true; - } while (!empty()); - return false; - } - template void send(const T& x) { static_assert(std::is_base_of::value, @@ -536,8 +524,8 @@ namespace common { * and load balancing. * Watch out that `recv` should run in photon environment (because it has to) * use photon semaphore to be notified that new item has sended. `send` could - * running in photon or std::thread environment (needs to set template `Pause` as - * `ThreadPause`). + * running in photon or std::thread environment (needs to set template `Pause` + * as `ThreadPause`). * * @tparam QueueType shoulde be one of LockfreeMPMCRingQueue, * LockfreeBatchMPMCRingQueue, or LockfreeSPSCRingQueue, with their own template