From 4349145a0879ef9853b24e69d85963cc5ff13200 Mon Sep 17 00:00:00 2001
From: Bob Chen <beef9999@qq.com>
Date: Mon, 23 Oct 2023 16:00:01 +0800
Subject: [PATCH 01/76] ipv6 (#192)

* IPv6
---
 CMakeLists.txt                       |   2 +-
 common/test/test_alog.cpp            |  19 ++-
 include/photon/net/utils-stdstring.h |   1 +
 net/base_socket.h                    |  59 +++++++++
 net/basic_socket.cpp                 |  11 +-
 net/datagram_socket.cpp              |  29 ++--
 net/http/client.cpp                  |  16 ++-
 net/http/client.h                    |   2 +-
 net/kernel_socket.cpp                |  88 ++++++++-----
 net/socket.h                         | 190 +++++++++++++++++++--------
 net/test/CMakeLists.txt              |   5 +-
 net/test/test-ipv6.cpp               | 167 +++++++++++++++++++++++
 net/test/test.cpp                    |  14 +-
 net/utils-stdstring.h                |  48 +++++++
 net/utils.cpp                        |  70 +++++-----
 net/utils.h                          |   6 +-
 rpc/rpc.cpp                          |  10 +-
 rpc/rpc.h                            |   3 +-
 thread/test/test.cpp                 |   4 -
 19 files changed, 575 insertions(+), 169 deletions(-)
 create mode 120000 include/photon/net/utils-stdstring.h
 create mode 100644 net/test/test-ipv6.cpp
 create mode 100644 net/utils-stdstring.h

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 2cd793c0..8626da46 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -55,7 +55,7 @@ set(CMAKE_CXX_EXTENSIONS OFF)
 set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g")
 set(CMAKE_CXX_FLAGS_RELEASE "-O2 -DNDEBUG")
 set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -DNDEBUG -g")
-set(CMAKE_CXX_FLAGS_MINSIZEREL "-O3 -g")    # Only for CI test
+set(CMAKE_CXX_FLAGS_MINSIZEREL "-O2 -g")    # Only for CI test
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 set(CMAKE_BUILD_RPATH_USE_ORIGIN ON)
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
diff --git a/common/test/test_alog.cpp b/common/test/test_alog.cpp
index bf0f6fbb..ce5f3dd4 100644
--- a/common/test/test_alog.cpp
+++ b/common/test/test_alog.cpp
@@ -19,7 +19,9 @@ limitations under the License.
 #include "../alog-functionptr.h"
 #include "../alog-audit.h"
 #include <gtest/gtest.h>
-#include "photon/thread/thread.h"
+#include <photon/thread/thread.h>
+#include <photon/net/socket.h>
+#include <photon/net/utils-stdstring.h>
 #include <chrono>
 #include <vector>
 #include <stdint.h>
@@ -556,6 +558,21 @@ TEST(ALog, LOG_LIMIT) {
     EXPECT_LE(x, suppose + 3);
 }
 
+TEST(ALOG, IPAddr) {
+    log_output = &log_output_test;
+    DEFER(log_output = log_output_stdout);
+
+    photon::net::IPAddr ip("192.168.12.34");
+    EXPECT_STREQ(photon::net::to_string(ip).c_str(), "192.168.12.34");
+    LOG_INFO(ip);
+    EXPECT_STREQ("192.168.12.34", log_output_test.log_start());
+
+    ip = photon::net::IPAddr("abcd:1111:222:33:4:5:6:7");
+    EXPECT_STREQ(photon::net::to_string(ip).c_str(), "abcd:1111:222:33:4:5:6:7");
+    LOG_INFO(ip);
+    EXPECT_STREQ("abcd:1111:222:33:4:5:6:7", log_output_test.log_start());
+}
+
 int main(int argc, char **argv)
 {
     photon::vcpu_init();
diff --git a/include/photon/net/utils-stdstring.h b/include/photon/net/utils-stdstring.h
new file mode 120000
index 00000000..ee064c39
--- /dev/null
+++ b/include/photon/net/utils-stdstring.h
@@ -0,0 +1 @@
+../../../net/utils-stdstring.h
\ No newline at end of file
diff --git a/net/base_socket.h b/net/base_socket.h
index b26b9e08..26bf2148 100644
--- a/net/base_socket.h
+++ b/net/base_socket.h
@@ -20,6 +20,7 @@ limitations under the License.
 Internal header provides abstract socket base class
 ***/
 
+#include <netinet/in.h>
 #include <vector>
 
 #include <photon/net/socket.h>
@@ -37,6 +38,64 @@ Internal header provides abstract socket base class
 namespace photon {
 namespace net {
 
+// sockaddr_storage is the container for any socket address type
+struct sockaddr_storage {
+    sockaddr_storage() = default;
+    explicit sockaddr_storage(const EndPoint& ep) {
+        if (ep.is_ipv4()) {
+            auto* in4 = (sockaddr_in*) &store;
+            in4->sin_family = AF_INET;
+            in4->sin_port = htons(ep.port);
+            in4->sin_addr.s_addr = ep.addr.to_nl();
+        } else {
+            auto* in6 = (sockaddr_in6*) &store;
+            in6->sin6_family = AF_INET6;
+            in6->sin6_port = htons(ep.port);
+            in6->sin6_addr = ep.addr.addr;
+        }
+    }
+    explicit sockaddr_storage(const sockaddr_in& addr) {
+        *((sockaddr_in*) &store) = addr;
+    }
+    explicit sockaddr_storage(const sockaddr_in6& addr) {
+        *((sockaddr_in6*) &store) = addr;
+    }
+    explicit sockaddr_storage(const sockaddr& addr) {
+        *((sockaddr*) &store) = addr;
+    }
+    EndPoint to_endpoint() const {
+        EndPoint ep;
+        if (store.ss_family == AF_INET6) {
+            auto s6 = (sockaddr_in6*) &store;
+            ep.addr = IPAddr(s6->sin6_addr);
+            ep.port = ntohs(s6->sin6_port);
+        } else if (store.ss_family == AF_INET) {
+            auto s4 = (sockaddr_in*) &store;
+            ep.addr = IPAddr(s4->sin_addr);
+            ep.port = ntohs(s4->sin_port);
+        }
+        return ep;
+    }
+    sockaddr* get_sockaddr() const {
+        return (sockaddr*) &store;
+    }
+    socklen_t get_socklen() const {
+        switch (store.ss_family) {
+            case AF_INET:
+                return sizeof(sockaddr_in);
+            case AF_INET6:
+                return sizeof(sockaddr_in6);
+            default:
+                return 0;
+        }
+    }
+    socklen_t get_max_socklen() const {
+        return sizeof(store);
+    }
+    // store must be zero initialized
+    ::sockaddr_storage store = {};
+};
+
 struct SocketOpt {
     int level;
     int opt_name;
diff --git a/net/basic_socket.cpp b/net/basic_socket.cpp
index d583454d..b5b2ec83 100644
--- a/net/basic_socket.cpp
+++ b/net/basic_socket.cpp
@@ -33,6 +33,7 @@ limitations under the License.
 #include <photon/common/alog.h>
 #include <photon/common/iovector.h>
 #include <photon/common/utility.h>
+#include "base_socket.h"
 
 #ifndef MSG_ZEROCOPY
 #define MSG_ZEROCOPY    0x4000000
@@ -262,11 +263,11 @@ bool ISocketStream::skip_read(size_t count) {
 }
 
 int do_get_name(int fd, Getter getter, EndPoint& addr) {
-    struct sockaddr_in addr_in;
-    socklen_t len = sizeof(addr_in);
-    int ret = getter(fd, (struct sockaddr*) &addr_in, &len);
-    if (ret < 0 || len != sizeof(addr_in)) return -1;
-    addr.from_sockaddr_in(addr_in);
+    sockaddr_storage storage;
+    socklen_t len = storage.get_max_socklen();
+    int ret = getter(fd, storage.get_sockaddr(), &len);
+    if (ret < 0 || len > storage.get_max_socklen()) return -1;
+    addr = storage.to_endpoint();
     return 0;
 }
 
diff --git a/net/datagram_socket.cpp b/net/datagram_socket.cpp
index dee6504e..8f42f403 100644
--- a/net/datagram_socket.cpp
+++ b/net/datagram_socket.cpp
@@ -16,14 +16,15 @@ limitations under the License.
 
 #include "datagram_socket.h"
 
+#include <sys/fcntl.h>
+#include <sys/un.h>
+#include <unistd.h>
 #include <photon/common/alog.h>
 #include <photon/common/utility.h>
 #include <photon/io/fd-events.h>
 #include <photon/net/basic_socket.h>
 #include <photon/net/socket.h>
-#include <sys/fcntl.h>
-#include <sys/un.h>
-#include <unistd.h>
+#include "base_socket.h"
 
 namespace photon {
 namespace net {
@@ -139,14 +140,14 @@ class UDP : public DatagramSocketBase {
     virtual int connect(const Addr* addr, size_t addr_len) override {
         auto ep = (EndPoint*)addr;
         assert(ep && addr_len == sizeof(*ep));
-        auto in = ep->to_sockaddr_in();
-        return do_connect((sockaddr*)&in, sizeof(in));
+        sockaddr_storage s(*ep);
+        return do_connect(s.get_sockaddr(), s.get_socklen());
     }
     virtual int bind(const Addr* addr, size_t addr_len) override {
         auto ep = (EndPoint*)addr;
         assert(ep && addr_len == sizeof(*ep));
-        auto in = ep->to_sockaddr_in();
-        return do_bind((sockaddr*)&in, sizeof(in));
+        sockaddr_storage s(*ep);
+        return do_bind(s.get_sockaddr(), s.get_socklen());
     }
     virtual ssize_t send(const struct iovec* iov, int iovcnt, const Addr* addr,
                          size_t addr_len, int flags = 0) override {
@@ -154,8 +155,8 @@ class UDP : public DatagramSocketBase {
         if (likely(!ep) || unlikely(addr_len != sizeof(*ep)))
             return do_send(iov, iovcnt, nullptr, 0, flags);
         assert(addr_len == sizeof(*ep));
-        auto in = ep->to_sockaddr_in();
-        return do_send(iov, iovcnt, (sockaddr*)&in, sizeof(in), flags);
+        sockaddr_storage s(*ep);
+        return do_send(iov, iovcnt, s.get_sockaddr(), s.get_socklen(), flags);
     }
     virtual ssize_t recv(const struct iovec* iov, int iovcnt, Addr* addr,
                          size_t* addr_len, int flags) override {
@@ -164,12 +165,12 @@ class UDP : public DatagramSocketBase {
             return do_recv(iov, iovcnt, nullptr, 0, flags);
         }
 
-        sockaddr_in in;
-        size_t alen = sizeof(in);
-        auto ret = do_recv(iov, iovcnt, (sockaddr*)&in, &alen, flags);
+        sockaddr_storage s(*ep);
+        size_t alen = s.get_socklen();
+        auto ret = do_recv(iov, iovcnt, s.get_sockaddr(), &alen, flags);
         if (ret >= 0) {
-            ep->from(in);
-            *addr_len = sizeof(*ep);
+            *ep = s.to_endpoint();
+            *addr_len = alen;
         }
         return ret;
     }
diff --git a/net/http/client.cpp b/net/http/client.cpp
index a66d6e57..efb6b3b0 100644
--- a/net/http/client.cpp
+++ b/net/http/client.cpp
@@ -40,11 +40,13 @@ class PooledDialer {
     std::unique_ptr<Resolver> resolver;
 
     //etsocket seems not support multi thread very well, use tcp_socket now. need to find out why
-    PooledDialer() :
+    explicit PooledDialer(bool ipv6) :
             tls_ctx(new_tls_context(nullptr, nullptr, nullptr)),
-            tcpsock(new_tcp_socket_pool(new_tcp_socket_client(), -1, true)),
-            tlssock(new_tcp_socket_pool(new_tls_client(tls_ctx, new_tcp_socket_client(), true), -1, true)),
             resolver(new_default_resolver(kDNSCacheLife)) {
+        auto tcp_cli = ipv6 ? new_tcp_socket_client_ipv6() : new_tcp_socket_client();
+        auto tls_cli = ipv6 ? new_tcp_socket_client_ipv6() : new_tcp_socket_client();
+        tcpsock.reset(new_tcp_socket_pool(tcp_cli, -1, true));
+        tlssock.reset(new_tcp_socket_pool(new_tls_client(tls_ctx, tls_cli, true), -1, true));
     }
 
     ~PooledDialer() { delete tls_ctx; }
@@ -77,7 +79,7 @@ ISocketStream* PooledDialer::dial(std::string_view host, uint16_t port, bool sec
         return sock;
     }
     LOG_DEBUG("connect ssl : ` ep : `  host : ` failed", secure, ep, host);
-    if (ipaddr.addr == 0) LOG_DEBUG("No connectable resolve result");
+    if (ipaddr.undefined()) LOG_DEBUG("No connectable resolve result");
     // When failed, remove resolved result from dns cache so that following retries can try
     // different ips.
     resolver->discard_cache(strhost.c_str());
@@ -107,8 +109,8 @@ class ClientImpl : public Client {
     PooledDialer m_dialer;
     CommonHeaders<> m_common_headers;
     ICookieJar *m_cookie_jar;
-    ClientImpl(ICookieJar *cookie_jar) :
-        m_cookie_jar(cookie_jar) {}
+    ClientImpl(ICookieJar *cookie_jar, bool ipv6) :
+            m_dialer(ipv6), m_cookie_jar(cookie_jar) {}
 
     using SocketStream_ptr = std::unique_ptr<ISocketStream>;
     int redirect(Operation* op) {
@@ -258,7 +260,7 @@ class ClientImpl : public Client {
     }
 };
 
-Client* new_http_client(ICookieJar *cookie_jar) { return new ClientImpl(cookie_jar); }
+Client* new_http_client(ICookieJar *cookie_jar, bool ipv6) { return new ClientImpl(cookie_jar, ipv6); }
 
 } // namespace http
 } // namespace net
diff --git a/net/http/client.h b/net/http/client.h
index 7a767786..2d68b27c 100644
--- a/net/http/client.h
+++ b/net/http/client.h
@@ -133,7 +133,7 @@ class Client : public Object {
 };
 
 //A Client without cookie_jar would ignore all response-header "Set-Cookies"
-Client* new_http_client(ICookieJar *cookie_jar = nullptr);
+Client* new_http_client(ICookieJar *cookie_jar = nullptr, bool ipv6 = false);
 
 ICookieJar* new_simple_cookie_jar();
 
diff --git a/net/kernel_socket.cpp b/net/kernel_socket.cpp
index 6ce6cf5d..323cddf1 100644
--- a/net/kernel_socket.cpp
+++ b/net/kernel_socket.cpp
@@ -60,20 +60,19 @@ limitations under the License.
 #endif
 
 LogBuffer& operator<<(LogBuffer& log, const in_addr& iaddr) {
-    return log << photon::net::IPAddr(ntohl(iaddr.s_addr));
+    return log << photon::net::IPAddr(iaddr);
+}
+LogBuffer& operator<<(LogBuffer& log, const in6_addr& iaddr) {
+    return log << photon::net::IPAddr(iaddr);
 }
-
 LogBuffer& operator<<(LogBuffer& log, const sockaddr_in& addr) {
-    return log << photon::net::EndPoint(addr);
+    return log << photon::net::sockaddr_storage(addr).to_endpoint();
+}
+LogBuffer& operator<<(LogBuffer& log, const sockaddr_in6& addr) {
+    return log << photon::net::sockaddr_storage(addr).to_endpoint();
 }
-
 LogBuffer& operator<<(LogBuffer& log, const sockaddr& addr) {
-    if (addr.sa_family == AF_INET) {
-        log << (const sockaddr_in&)addr;
-    } else {
-        log.printf("<sockaddr>");
-    }
-    return log;
+    return log << photon::net::sockaddr_storage(addr).to_endpoint();
 }
 
 namespace photon {
@@ -93,7 +92,7 @@ class KernelSocketStream : public SocketStreamBase {
         } else {
             fd = ::socket(socket_family, SOCK_STREAM, 0);
         }
-        if (fd >= 0 && socket_family == AF_INET) {
+        if (fd >= 0 && (socket_family == AF_INET || socket_family == AF_INET6)) {
             int val = 1;
             ::setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &val, (socklen_t) sizeof(val));
         }
@@ -214,18 +213,16 @@ class KernelSocketClient : public SocketClientBase {
     ISocketStream* connect(const char* path, size_t count) override {
         struct sockaddr_un addr_un;
         if (fill_uds_path(addr_un, path, count) != 0) return nullptr;
-        return do_connect((const sockaddr*) &addr_un, nullptr, sizeof(addr_un));
+        return do_connect((const sockaddr*) &addr_un, sizeof(addr_un));
     }
 
     ISocketStream* connect(EndPoint remote, EndPoint local = EndPoint()) override {
-        sockaddr_in addr_remote = remote.to_sockaddr_in();
-        auto r = (sockaddr*) &addr_remote;
-        if (local.empty()) {
-            return do_connect(r, nullptr, sizeof(addr_remote));
+        sockaddr_storage r(remote);
+        if (local.undefined()) {
+            return do_connect(r.get_sockaddr(), r.get_socklen());
         }
-        sockaddr_in addr_local = local.to_sockaddr_in();
-        auto l = (sockaddr*) &addr_local;
-        return do_connect(r, l, sizeof(addr_remote));
+        sockaddr_storage l(local);
+        return do_connect(r.get_sockaddr(), r.get_socklen(), l.get_sockaddr(), l.get_socklen());
     }
 
 protected:
@@ -240,7 +237,8 @@ class KernelSocketClient : public SocketClientBase {
         return net::connect(fd, remote, addrlen, m_timeout);
     }
 
-    ISocketStream* do_connect(const sockaddr* remote, const sockaddr* local, socklen_t addrlen) {
+    ISocketStream* do_connect(const sockaddr* remote, socklen_t len_remote,
+                              const sockaddr* local = nullptr, socklen_t len_local = 0) {
         auto stream = create_stream();
         std::unique_ptr<KernelSocketStream> ptr(stream);
         if (!ptr || ptr->fd < 0) {
@@ -251,11 +249,11 @@ class KernelSocketClient : public SocketClientBase {
         }
         ptr->timeout(m_timeout);
         if (local != nullptr) {
-            if (::bind(ptr->fd, local, addrlen) != 0) {
+            if (::bind(ptr->fd, local, len_local) != 0) {
                 LOG_ERRNO_RETURN(0, nullptr, "fail to bind socket");
             }
         }
-        auto ret = fd_connect(ptr->fd, remote, addrlen);
+        auto ret = fd_connect(ptr->fd, remote, len_remote);
         if (ret < 0) {
             LOG_ERRNO_RETURN(0, nullptr, "Failed to connect socket");
         }
@@ -300,7 +298,7 @@ class KernelSocketServer : public SocketServerBase {
         if (m_listen_fd < 0) {
             LOG_ERRNO_RETURN(0, -1, "fail to setup listen fd");
         }
-        if (m_socket_family == AF_INET) {
+        if (m_socket_family == AF_INET || m_socket_family == AF_INET6) {
             if (setsockopt<int>(IPPROTO_TCP, TCP_NODELAY, 1) != 0) {
                 LOG_ERRNO_RETURN(EINVAL, -1, "failed to setsockopt of TCP_NODELAY");
             }
@@ -336,8 +334,11 @@ class KernelSocketServer : public SocketServerBase {
     }
 
     int bind(uint16_t port, IPAddr addr) override {
-        auto addr_in = EndPoint(addr, port).to_sockaddr_in();
-        return ::bind(m_listen_fd, (struct sockaddr*)&addr_in, sizeof(addr_in));
+        if (m_socket_family == AF_INET6 && addr.undefined()) {
+            addr = IPAddr::V6Any();
+        }
+        sockaddr_storage s(EndPoint(addr, port));
+        return ::bind(m_listen_fd, s.get_sockaddr(), s.get_socklen());
     }
 
     int bind(const char* path, size_t count) override {
@@ -419,11 +420,12 @@ class KernelSocketServer : public SocketServerBase {
     int do_accept() { return fd_accept(m_listen_fd, nullptr, nullptr); }
 
     int do_accept(EndPoint& remote_endpoint) {
-        struct sockaddr_in addr_in;
-        socklen_t len = sizeof(addr_in);
-        int cfd = fd_accept(m_listen_fd, (struct sockaddr*) &addr_in, &len);
-        if (cfd < 0 || len != sizeof(addr_in)) return -1;
-        remote_endpoint.from_sockaddr_in(addr_in);
+        sockaddr_storage s;
+        socklen_t len = s.get_max_socklen();
+        int cfd = fd_accept(m_listen_fd, s.get_sockaddr(), &len);
+        if (cfd < 0 || len > s.get_max_socklen())
+            return -1;
+        remote_endpoint = s.to_endpoint();
         return cfd;
     }
 
@@ -900,7 +902,8 @@ class ETKernelSocketStream : public KernelSocketStream, public NotifyContext {
         if (fd >= 0) etpoller.register_notifier(fd, this);
     }
 
-    ETKernelSocketStream() : KernelSocketStream(AF_INET, true) {
+    ETKernelSocketStream(int socket_family, bool nonblocking) :
+            KernelSocketStream(socket_family, nonblocking) {
         if (fd >= 0) etpoller.register_notifier(fd, this);
     }
 
@@ -947,7 +950,7 @@ class ETKernelSocketClient : public KernelSocketClient {
 
 protected:
     KernelSocketStream* create_stream() override {
-        return new ETKernelSocketStream();
+        return new ETKernelSocketStream(m_socket_family, m_nonblocking);
     }
 };
 
@@ -981,19 +984,36 @@ class ETKernelSocketServer : public KernelSocketServer, public NotifyContext {
 /* ET Socket - End */
 
 LogBuffer& operator<<(LogBuffer& log, const IPAddr addr) {
-    return log.printf(addr.d, '.', addr.c, '.', addr.b, '.', addr.a);
+    if (addr.is_ipv4())
+        return log.printf(addr.a, '.', addr.b, '.', addr.c, '.', addr.d);
+    else {
+        if (log.size < INET6_ADDRSTRLEN)
+            return log;
+        inet_ntop(AF_INET6, &addr.addr, log.ptr, INET6_ADDRSTRLEN);
+        log.consume(strlen(log.ptr));
+        return log;
+    }
 }
 
 LogBuffer& operator<<(LogBuffer& log, const EndPoint ep) {
-    return log << ep.addr << ':' << ep.port;
+    if (ep.is_ipv4())
+        return log << ep.addr << ':' << ep.port;
+    else
+        return log << '[' << ep.addr << "]:" << ep.port;
 }
 
 extern "C" ISocketClient* new_tcp_socket_client() {
     return new KernelSocketClient(AF_INET, true);
 }
+extern "C" ISocketClient* new_tcp_socket_client_ipv6() {
+    return new KernelSocketClient(AF_INET6, true);
+}
 extern "C" ISocketServer* new_tcp_socket_server() {
     return NewObj<KernelSocketServer>(AF_INET, false, true)->init();
 }
+extern "C" ISocketServer* new_tcp_socket_server_ipv6() {
+    return NewObj<KernelSocketServer>(AF_INET6, false, true)->init();
+}
 extern "C" ISocketClient* new_uds_client() {
     return new KernelSocketClient(AF_UNIX, true);
 }
diff --git a/net/socket.h b/net/socket.h
index 55b576fd..15a89c6f 100644
--- a/net/socket.h
+++ b/net/socket.h
@@ -26,78 +26,146 @@ limitations under the License.
 #include <photon/common/callback.h>
 #include <photon/common/object.h>
 
+#ifdef __linux__
+#define _in_addr_field s6_addr32
+#else // macOS
+#define _in_addr_field __u6_addr.__u6_addr32
+#endif
+
 struct LogBuffer;
 LogBuffer& operator << (LogBuffer& log, const in_addr& iaddr);
 LogBuffer& operator << (LogBuffer& log, const sockaddr_in& addr);
+LogBuffer& operator << (LogBuffer& log, const in6_addr& iaddr);
+LogBuffer& operator << (LogBuffer& log, const sockaddr_in6& addr);
 LogBuffer& operator << (LogBuffer& log, const sockaddr& addr);
+
 namespace photon {
-namespace net
-{
-    union IPAddr
-    {
-        uint32_t addr = 0;
-        struct { uint8_t a, b, c, d; };
-        explicit IPAddr(uint32_t nl)
-        {
-            from_nl(nl);
-        }
-        explicit IPAddr(const char* s)
-        {
-            struct in_addr addr;
-            if (inet_aton(s, &addr) == 0)
-                return;  // invalid IPv4 address
-            from_nl(addr.s_addr);
-        }
-        IPAddr() = default;
-        IPAddr(const IPAddr& rhs) = default;
-        uint32_t to_nl() const
-        {
-            return htonl(addr);
-        }
-        void from_nl(uint32_t nl)
-        {
-            addr = ntohl(nl);
+namespace net {
+
+    struct __attribute__ ((packed)) IPAddr {
+    public:
+        union {
+            in6_addr addr = {};
+            struct { uint16_t _1, _2, _3, _4, _5, _6; uint8_t a, b, c, d; };
+        };
+        // For compatibility, the default constructor is still 0.0.0.0 (IPv4)
+        IPAddr() {
+            map_v4(htonl(INADDR_ANY));
+        }
+        // V6 constructor (Internet Address)
+        explicit IPAddr(in6_addr internet_addr) {
+            addr = internet_addr;
+        }
+        // V6 constructor (Network byte order)
+        IPAddr(uint32_t nl1, uint32_t nl2, uint32_t nl3, uint32_t nl4) {
+            addr._in_addr_field[0] = nl1;
+            addr._in_addr_field[1] = nl2;
+            addr._in_addr_field[2] = nl3;
+            addr._in_addr_field[3] = nl4;
+        }
+        // V4 constructor (Internet Address)
+        explicit IPAddr(in_addr internet_addr) {
+            map_v4(internet_addr);
+        }
+        // V4 constructor (Network byte order)
+        explicit IPAddr(uint32_t nl) {
+            map_v4(nl);
+        }
+        // String constructor
+        explicit IPAddr(const char* s) {
+            if (inet_pton(AF_INET6, s, &addr) > 0) {
+                return;
+            }
+            in_addr v4_addr;
+            if (inet_pton(AF_INET, s, &v4_addr) > 0) {
+                map_v4(v4_addr);
+                return;
+            }
+            // Invalid string, make it a default value
+            *this = IPAddr();
+        }
+        // Check if it's actually an IPv4 address mapped in IPV6
+        bool is_ipv4() const {
+            if (ntohl(addr._in_addr_field[2]) != 0x0000ffff) {
+                return false;
+            }
+            if (addr._in_addr_field[0] != 0 || addr._in_addr_field[1] != 0) {
+                return false;
+            }
+            return true;
+        }
+        // We regard the default IPv4 0.0.0.0 as undefined
+        bool undefined() const {
+            return *this == V4Any();
+        }
+        // Should ONLY be used for IPv4 address
+        uint32_t to_nl() const {
+            return addr._in_addr_field[3];
+        }
+        bool is_loopback() const {
+            return is_ipv4() ? (*this == V4Loopback()) : (*this == V6Loopback());
+        }
+        bool is_broadcast() const {
+            // IPv6 does not support broadcast
+            return is_ipv4() && (*this == V4Broadcast());
+        }
+        bool is_link_local() const {
+            if (is_ipv4()) {
+                return (to_nl() & htonl(0xffff0000)) == htonl(0xa9fe0000);
+            } else {
+                return (addr._in_addr_field[0] & htonl(0xffc00000)) == htonl(0xfe800000);
+            }
+        }
+        bool operator==(const IPAddr& rhs) const {
+            return memcmp(this, &rhs, sizeof(rhs)) == 0;
+        }
+        bool operator!=(const IPAddr& rhs) const {
+            return !(*this == rhs);
+        }
+    public:
+        static IPAddr V6None() {
+            return IPAddr(htonl(0xffffffff), htonl(0xffffffff), htonl(0xffffffff), htonl(0xffffffff));
+        }
+        static IPAddr V6Any() { return IPAddr(in6addr_any); }
+        static IPAddr V6Loopback() { return IPAddr(in6addr_loopback); }
+        static IPAddr V4Broadcast() { return IPAddr(htonl(INADDR_BROADCAST)); }
+        static IPAddr V4Any() { return IPAddr(htonl(INADDR_ANY)); }
+        static IPAddr V4Loopback() { return IPAddr(htonl(INADDR_LOOPBACK)); }
+    private:
+        void map_v4(in_addr addr_) {
+            map_v4(addr_.s_addr);
+        }
+        void map_v4(uint32_t nl) {
+            addr._in_addr_field[0] = 0x00000000;
+            addr._in_addr_field[1] = 0x00000000;
+            addr._in_addr_field[2] = htonl(0xffff);
+            addr._in_addr_field[3] = nl;
         }
     };
 
-    struct EndPoint
-    {
+    static_assert(sizeof(IPAddr) == 16, "IPAddr size incorrect");
+
+    struct __attribute__ ((packed)) EndPoint {
         IPAddr addr;
         uint16_t port = 0;
         EndPoint() = default;
-        EndPoint(IPAddr ip, uint16_t port): addr(ip), port(port) {}
-        EndPoint(const struct sockaddr_in& addr_in)
-        {
-            from(addr_in);
-        }
-        sockaddr_in to_sockaddr_in() const
-        {
-            struct sockaddr_in addr_in;
-            addr_in.sin_family = AF_INET;
-            addr_in.sin_addr.s_addr = addr.to_nl();
-            addr_in.sin_port = htons(port);
-            return addr_in;
-        }
-        void from_sockaddr_in(const struct sockaddr_in& addr_in)
-        {
-            addr.from_nl(addr_in.sin_addr.s_addr);
-            port = ntohs(addr_in.sin_port);
-        }
-        void from(const struct sockaddr_in& addr_in)
-        {
-            from_sockaddr_in(addr_in);
-        }
+        EndPoint(IPAddr ip, uint16_t port) : addr(ip), port(port) {}
+        bool is_ipv4() const {
+            return addr.is_ipv4();
+        };
         bool operator==(const EndPoint& rhs) const {
-            return rhs.addr.addr == addr.addr && rhs.port == port;
+            return rhs.addr == addr && rhs.port == port;
         }
         bool operator!=(const EndPoint& rhs) const {
             return !operator==(rhs);
         }
-        bool empty() const {
-            return addr.addr == 0 && port == 0;
+        bool undefined() const {
+            return addr.undefined() && port == 0;
         }
     };
 
+    static_assert(sizeof(EndPoint) == 18, "Endpoint size incorrect");
+
     // operators to help with logging IP addresses
     LogBuffer& operator << (LogBuffer& log, const IPAddr addr);
     LogBuffer& operator << (LogBuffer& log, const EndPoint ep);
@@ -187,6 +255,8 @@ namespace net
 
     extern "C" ISocketClient* new_tcp_socket_client();
     extern "C" ISocketServer* new_tcp_socket_server();
+    extern "C" ISocketClient* new_tcp_socket_client_ipv6();
+    extern "C" ISocketServer* new_tcp_socket_server_ipv6();
     extern "C" ISocketClient* new_uds_client();
     extern "C" ISocketServer* new_uds_server(bool autoremove = false);
     extern "C" ISocketClient* new_tcp_socket_pool(ISocketClient* client, uint64_t expiration = -1UL,
@@ -208,11 +278,21 @@ namespace net
 }
 
 namespace std {
+
 template<>
 struct hash<photon::net::EndPoint> {
-    hash<uint64_t> hasher;
     size_t operator()(const photon::net::EndPoint& x) const {
-        return hasher((x.addr.to_nl() << 16) | x.port);
+        hash<std::string_view> hasher;
+        return hasher(std::string_view((const char*) &x, sizeof(x)));
     }
 };
+
+template<>
+struct hash<photon::net::IPAddr> {
+    size_t operator()(const photon::net::IPAddr& x) const {
+        hash<std::string_view> hasher;
+        return hasher(std::string_view((const char*) &x, sizeof(x)));
+    }
+};
+
 }
diff --git a/net/test/CMakeLists.txt b/net/test/CMakeLists.txt
index d287383a..93862a4f 100644
--- a/net/test/CMakeLists.txt
+++ b/net/test/CMakeLists.txt
@@ -22,4 +22,7 @@ add_executable(test-server test-server.cpp)
 target_link_libraries(test-server PRIVATE photon_shared ${testing_libs})
 
 add_executable(test-client test-client.cpp)
-target_link_libraries(test-client PRIVATE photon_shared ${testing_libs})
\ No newline at end of file
+target_link_libraries(test-client PRIVATE photon_shared ${testing_libs})
+
+add_executable(test-ipv6 test-ipv6.cpp)
+target_link_libraries(test-ipv6 PRIVATE photon_shared ${testing_libs})
\ No newline at end of file
diff --git a/net/test/test-ipv6.cpp b/net/test/test-ipv6.cpp
new file mode 100644
index 00000000..9400b5b8
--- /dev/null
+++ b/net/test/test-ipv6.cpp
@@ -0,0 +1,167 @@
+#include <vector>
+#include <gtest/gtest.h>
+
+#include <photon/photon.h>
+#include <photon/thread/thread11.h>
+#include <photon/net/socket.h>
+#include <photon/net/utils.h>
+#include <photon/common/alog.h>
+
+TEST(ipv6, addr) {
+    EXPECT_NO_THROW(photon::net::IPAddr a("::1"));
+    EXPECT_NO_THROW(photon::net::IPAddr a("1.2.3.4"));
+    EXPECT_NO_THROW(photon::net::IPAddr a("fdbd:dc01:ff:312:9641:f71:10c4:2378"));
+    EXPECT_NO_THROW(photon::net::IPAddr a("fdbd:dc01:ff:312:9641:f71::2378"));
+    EXPECT_NO_THROW(photon::net::IPAddr a("fdbd:dc01:ff:312:9641::2378"));
+
+    auto c = photon::net::IPAddr("zfdbd:dq01:8:165::158");
+    EXPECT_TRUE(c.undefined());
+    c = photon::net::IPAddr("fdbd::ff:312:9641:f71::2378");
+    EXPECT_TRUE(c.undefined());
+    c = photon::net::IPAddr("::1z");
+    EXPECT_TRUE(c.undefined());
+    c = photon::net::IPAddr("::ffffffff");
+    EXPECT_TRUE(c.undefined());
+    c = photon::net::IPAddr("1.256.3.4");
+    EXPECT_TRUE(c.undefined());
+    c = photon::net::IPAddr("1.2.3.zzzz");
+    EXPECT_TRUE(c.undefined());
+
+    EXPECT_TRUE(photon::net::IPAddr() == photon::net::IPAddr::V4Any());
+    EXPECT_TRUE(photon::net::IPAddr("0.0.0.0") == photon::net::IPAddr::V4Any());
+    EXPECT_TRUE(photon::net::IPAddr("127.0.0.1") == photon::net::IPAddr::V4Loopback());
+    EXPECT_TRUE(photon::net::IPAddr("255.255.255.255") == photon::net::IPAddr::V4Broadcast());
+
+    EXPECT_TRUE(photon::net::IPAddr("::") == photon::net::IPAddr::V6Any());
+    EXPECT_TRUE(photon::net::IPAddr("::1") == photon::net::IPAddr::V6Loopback());
+    EXPECT_TRUE(photon::net::IPAddr("ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff") == photon::net::IPAddr::V6None());
+
+    photon::net::IPAddr b("fe80::216:3eff:fe7c:39e0");
+    EXPECT_TRUE(b.is_link_local());
+}
+
+TEST(ipv6, get_host_by_peer) {
+    auto peer = photon::net::gethostbypeer(photon::net::IPAddr("2001:4860:4860::8888"));
+    ASSERT_TRUE(!peer.undefined());
+    ASSERT_TRUE(!peer.is_ipv4());
+    LOG_INFO(peer);
+}
+
+TEST(ipv6, dns_lookup) {
+    std::vector<photon::net::IPAddr> ret;
+    int num = photon::net::gethostbyname("github.com", ret);
+    ASSERT_GT(num, 0);
+    ASSERT_EQ(num, ret.size());
+    bool has_v6 = false;
+    for (auto& each : ret) {
+        LOG_INFO("github.com IP addr `", each);
+        if (!each.is_ipv4()) {
+            has_v6 = true;
+            break;
+        }
+    }
+    ASSERT_TRUE(has_v6);
+}
+
+class DualStackTest : public ::testing::Test {
+public:
+    void run() {
+        auto server = photon::net::new_tcp_socket_server_ipv6();
+        ASSERT_NE(nullptr, server);
+        DEFER(delete server);
+        int ret = server->setsockopt(SOL_SOCKET, SO_REUSEPORT, 1);
+        ASSERT_EQ(0, ret);
+
+        ret = server->bind(9527, photon::net::IPAddr::V6Any());
+        ASSERT_EQ(0, ret);
+        ret = server->listen();
+        ASSERT_EQ(0, ret);
+
+        photon::thread_create11([&] {
+            auto client = get_client();
+            if (!client) abort();
+            DEFER(delete client);
+
+            photon::net::EndPoint ep(get_server_ip(), 9527);
+            auto stream = client->connect(ep);
+            if (!stream) abort();
+            DEFER(delete stream);
+            LOG_INFO("Server endpoint: ", ep);
+
+            photon::net::EndPoint ep2;
+            int ret = stream->getsockname(ep2);
+            if (ret) abort();
+            if(ep2.undefined()) abort();
+            LOG_INFO("Client endpoint: ", ep2);
+            if (ep2.port < 10000) abort();
+
+            photon::net::EndPoint ep3;
+            ret = stream->getpeername(ep3);
+            if (ret) abort();
+            if (ep3.undefined()) abort();
+            if (ep != ep3) abort();
+        });
+
+        photon::net::EndPoint ep;
+        auto stream = server->accept(&ep);
+        ASSERT_NE(nullptr, stream);
+        LOG_INFO("Client endpoint: ", ep);
+
+        photon::net::EndPoint ep4;
+        ret = stream->getsockname(ep4);
+        ASSERT_EQ(0, ret);
+
+        if (is_ipv6_client()) {
+            ASSERT_TRUE(!ep4.is_ipv4());
+        } else {
+            ASSERT_TRUE(ep4.is_ipv4());
+        }
+        ASSERT_EQ(9527, ep4.port);
+
+        // Wait client close
+        photon::thread_sleep(2);
+        DEFER(delete stream);
+    }
+protected:
+    virtual photon::net::ISocketClient* get_client() = 0;
+    virtual photon::net::IPAddr get_server_ip() = 0;
+    virtual bool is_ipv6_client() = 0;
+};
+
+class V6ToV6Test : public DualStackTest {
+protected:
+    photon::net::ISocketClient* get_client() override {
+        return photon::net::new_tcp_socket_client_ipv6();
+    }
+    photon::net::IPAddr get_server_ip() override {
+        return photon::net::IPAddr::V6Loopback();
+    }
+    bool is_ipv6_client() override { return true; }
+};
+
+class V4ToV6Test : public DualStackTest {
+protected:
+    photon::net::ISocketClient* get_client() override {
+        return photon::net::new_tcp_socket_client();
+    }
+    photon::net::IPAddr get_server_ip() override {
+        return photon::net::IPAddr::V4Loopback();
+    }
+    bool is_ipv6_client() override { return false; }
+};
+
+TEST_F(V6ToV6Test, run) {
+    run();
+}
+
+TEST_F(V4ToV6Test, run) {
+    run();
+}
+
+int main(int argc, char** arg) {
+    ::testing::InitGoogleTest(&argc, arg);
+    if (photon::init() != 0)
+        LOG_ERROR_RETURN(0, -1, "error init");
+    DEFER(photon::fini());
+    return RUN_ALL_TESTS();
+}
\ No newline at end of file
diff --git a/net/test/test.cpp b/net/test/test.cpp
index a2869a51..a61d6f72 100644
--- a/net/test/test.cpp
+++ b/net/test/test.cpp
@@ -180,18 +180,20 @@ class LogOutputTest final : public ILogOutput {
 TEST(Socket, endpoint) {
     EndPoint ep;
     struct in_addr inaddr;
-    struct sockaddr_in saddrin, rsai;
+    struct sockaddr_in saddrin;
     inet_aton("12.34.56.78", &inaddr);
     saddrin.sin_family = AF_INET;
     saddrin.sin_port = htons(4321);
     saddrin.sin_addr = inaddr;
 
-    ep.from_sockaddr_in(saddrin);
-    rsai = ep.to_sockaddr_in();
+    photon::net::sockaddr_storage s(saddrin);
+    ep = s.to_endpoint();
+    EXPECT_TRUE(ep == EndPoint(IPAddr("12.34.56.78"), 4321));
 
-    EXPECT_EQ(saddrin.sin_addr.s_addr, rsai.sin_addr.s_addr);
-    EXPECT_EQ(saddrin.sin_family, rsai.sin_family);
-    EXPECT_EQ(saddrin.sin_port, rsai.sin_port);
+    auto rsai = (sockaddr_in*) s.get_sockaddr();
+    EXPECT_EQ(saddrin.sin_addr.s_addr, rsai->sin_addr.s_addr);
+    EXPECT_EQ(saddrin.sin_family, rsai->sin_family);
+    EXPECT_EQ(saddrin.sin_port, rsai->sin_port);
 
     log_output = &log_output_test;
     LOG_DEBUG(ep);
diff --git a/net/utils-stdstring.h b/net/utils-stdstring.h
new file mode 100644
index 00000000..4a5fa4fc
--- /dev/null
+++ b/net/utils-stdstring.h
@@ -0,0 +1,48 @@
+/*
+Copyright 2022 The Photon Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#pragma once
+
+#include <string>
+#include <photon/net/socket.h>
+
+namespace photon {
+namespace net {
+
+inline std::string to_string(const photon::net::IPAddr& addr) {
+    std::string str;
+    char text[INET6_ADDRSTRLEN];
+    if (addr.is_ipv4()) {
+        in_addr ip4;
+        ip4.s_addr = addr.to_nl();
+        inet_ntop(AF_INET, &ip4, text, INET_ADDRSTRLEN);
+    } else {
+        inet_ntop(AF_INET6, &addr, text, INET6_ADDRSTRLEN);
+    }
+    str.assign(text, strlen(text));
+    return str;
+}
+
+inline std::string to_string(const photon::net::EndPoint& ep) {
+    if (ep.is_ipv4()) {
+        return to_string(ep.addr) + ":" + std::to_string(ep.port);
+    } else {
+        return "[" + to_string(ep.addr) + "]:" + std::to_string(ep.port);
+    }
+}
+
+}
+}
\ No newline at end of file
diff --git a/net/utils.cpp b/net/utils.cpp
index 3820e65a..28827d77 100644
--- a/net/utils.cpp
+++ b/net/utils.cpp
@@ -25,11 +25,11 @@ limitations under the License.
 #include <string>
 
 #include <photon/common/alog.h>
-#include <photon/io/signal.h>
 #include <photon/thread/thread11.h>
 #include <photon/common/utility.h>
 #include <photon/common/expirecontainer.h>
 #include "socket.h"
+#include "base_socket.h"
 
 namespace photon {
 namespace net {
@@ -41,21 +41,23 @@ IPAddr gethostbypeer(IPAddr remote) {
     // but let os select the interface to connect,
     // then get its ip
     constexpr uint16_t UDP_IP_DETECTE_PORT = 8080;
+    int sock_family = remote.is_ipv4() ? AF_INET : AF_INET6;
 
-    int sockfd = ::socket(AF_INET, SOCK_DGRAM, 0);
+    int sockfd = ::socket(sock_family, SOCK_DGRAM, 0);
     if (sockfd < 0) LOG_ERRNO_RETURN(0, IPAddr(), "Cannot create udp socket");
     DEFER(::close(sockfd));
-    struct sockaddr_in addr_in =
-        EndPoint{remote, UDP_IP_DETECTE_PORT}.to_sockaddr_in();
-    auto ret =
-        ::connect(sockfd, (sockaddr *)&addr_in, sizeof(struct sockaddr_in));
+
+    EndPoint ep_remote(remote, UDP_IP_DETECTE_PORT);
+    sockaddr_storage s_remote(ep_remote);
+
+    auto ret = ::connect(sockfd, s_remote.get_sockaddr(), s_remote.get_socklen());
     if (ret < 0) LOG_ERRNO_RETURN(0, IPAddr(), "Cannot connect remote");
-    struct sockaddr_in addr_local;
-    socklen_t len = sizeof(struct sockaddr_in);
-    ::getsockname(sockfd, (sockaddr *)&addr_local, &len);
-    IPAddr result;
-    result.from_nl(addr_local.sin_addr.s_addr);
-    return result;
+
+    sockaddr_storage s_local;
+    socklen_t len = s_local.get_max_socklen();
+    ::getsockname(sockfd, s_local.get_sockaddr(), &len);
+
+    return s_local.to_endpoint().addr;
 }
 
 IPAddr gethostbypeer(const char *domain) {
@@ -67,29 +69,35 @@ IPAddr gethostbypeer(const char *domain) {
     return gethostbypeer(remote);
 }
 
-int _gethostbyname(const char *name, Delegate<int, IPAddr> append_op) {
-    struct hostent *ent = nullptr;
-#ifdef __APPLE__
-    ent = ::gethostbyname(name);
-#else
-    int err;
-    struct hostent hbuf;
-    char buf[32 * 1024];
-
-    // only retval 0 means successful
-    if (::gethostbyname_r(name, &hbuf, buf, sizeof(buf), &ent, &err) != 0)
-        LOG_ERRNO_RETURN(0, -1, "Failed to gethostbyname", VALUE(err));
-#endif
+int _gethostbyname(const char* name, Delegate<int, IPAddr> append_op) {
+    assert(name);
     int idx = 0;
-    if (ent && ent->h_addrtype == AF_INET) {
-        // can only support IPv4
-        auto addrlist = (struct in_addr **)ent->h_addr_list;
-        while (*addrlist != nullptr) {
-            if (append_op(IPAddr((*addrlist)->s_addr)) < 0) break;
+    addrinfo* result = nullptr;
+    addrinfo hints = {};
+    hints.ai_socktype = SOCK_STREAM;
+    hints.ai_family = AF_UNSPEC;
+
+    int ret = getaddrinfo(name, nullptr, &hints, &result);
+    if (ret != 0) {
+        LOG_ERROR_RETURN(0, -1, "Fail to getaddrinfo: `", gai_strerror(ret));
+    }
+    assert(result);
+    for (auto* cur = result; cur != nullptr; cur = cur->ai_next) {
+        if (cur->ai_family == AF_INET6) {
+            auto sock_addr = (sockaddr_in6*) cur->ai_addr;
+            if (append_op(IPAddr(sock_addr->sin6_addr)) < 0) {
+                break;
+            }
+            idx++;
+        } else if (cur->ai_family == AF_INET) {
+            auto sock_addr = (sockaddr_in*) cur->ai_addr;
+            if (append_op(IPAddr(sock_addr->sin_addr)) < 0) {
+                break;
+            }
             idx++;
-            addrlist++;
         }
     }
+    freeaddrinfo(result);
     return idx;
 }
 
diff --git a/net/utils.h b/net/utils.h
index 452944dc..9420686e 100644
--- a/net/utils.h
+++ b/net/utils.h
@@ -69,7 +69,7 @@ int _gethostbyname(const char* name, Callback<IPAddr> append_op);
  * @param name Host name to resolve
  * @return first resolved address.
  */
- inline IPAddr gethostbyname(const char* name) {
+inline IPAddr gethostbyname(const char* name) {
     IPAddr ret;
     auto cb = [&](IPAddr addr) {
         ret = addr;
@@ -88,7 +88,7 @@ int _gethostbyname(const char* name, Callback<IPAddr> append_op);
  * @param name Host name to resolve
  * @param buf IPAddr buffer pointer
  * @param bufsize size of `buf`, takes `sizeof(IPAddr)` as unit
- * @return sum of resolved address number. result will be filled into `buf`
+ * @return sum of resolved address number. -1 means error. result will be filled into `buf`
  */
 inline int gethostbyname(const char* name, IPAddr* buf, int bufsize = 1) {
     int i = 0;
@@ -107,7 +107,7 @@ inline int gethostbyname(const char* name, IPAddr* buf, int bufsize = 1) {
  *
  * @param name Host name to resolve
  * @param ret `std::vector<IPAddr>` reference to get results
- * @return sum of resolved address number.
+ * @return sum of resolved address number. -1 means error.
  */
 inline int gethostbyname(const char* name, std::vector<IPAddr>& ret) {
     ret.clear();
diff --git a/rpc/rpc.cpp b/rpc/rpc.cpp
index 18ac63aa..2ded983a 100644
--- a/rpc/rpc.cpp
+++ b/rpc/rpc.cpp
@@ -427,9 +427,9 @@ namespace rpc {
 
     class StubPoolImpl : public StubPool {
     public:
-        explicit StubPoolImpl(uint64_t expiration, uint64_t connect_timeout, uint64_t rpc_timeout) {
+        explicit StubPoolImpl(uint64_t expiration, uint64_t connect_timeout, uint64_t rpc_timeout, bool ipv6) {
             tls_ctx = net::new_tls_context(nullptr, nullptr, nullptr);
-            tcpclient = net::new_tcp_socket_client();
+            tcpclient = ipv6 ? net::new_tcp_socket_client_ipv6() : net::new_tcp_socket_client();
             tcpclient->timeout(connect_timeout);
             m_pool = new ObjectCache<net::EndPoint, rpc::Stub*>(expiration);
             m_rpc_timeout = rpc_timeout;
@@ -489,7 +489,7 @@ namespace rpc {
     public:
         explicit UDSStubPoolImpl(const char* path, uint64_t expiration,
                                  uint64_t connect_timeout, uint64_t rpc_timeout)
-            : StubPoolImpl(expiration, connect_timeout, rpc_timeout),
+            : StubPoolImpl(expiration, connect_timeout, rpc_timeout, false),
               m_path(path), m_client(net::new_uds_client()) {
                   m_client->timeout(connect_timeout);
               }
@@ -515,8 +515,8 @@ namespace rpc {
         net::ISocketClient * m_client;
     };
 
-    StubPool* new_stub_pool(uint64_t expiration, uint64_t connect_timeout, uint64_t rpc_timeout) {
-        return new StubPoolImpl(expiration, connect_timeout, rpc_timeout);
+    StubPool* new_stub_pool(uint64_t expiration, uint64_t connect_timeout, uint64_t rpc_timeout, bool ipv6) {
+        return new StubPoolImpl(expiration, connect_timeout, rpc_timeout, ipv6);
     }
 
     StubPool* new_uds_stub_pool(const char* path, uint64_t expiration,
diff --git a/rpc/rpc.h b/rpc/rpc.h
index 0b2b19fa..b5e6b083 100644
--- a/rpc/rpc.h
+++ b/rpc/rpc.h
@@ -234,7 +234,8 @@ namespace rpc
     };
 
     extern "C" Stub* new_rpc_stub(IStream* stream, bool ownership = false);
-    extern "C" StubPool* new_stub_pool(uint64_t expiration, uint64_t connect_timeout, uint64_t rpc_timeout);
+    extern "C" StubPool* new_stub_pool(uint64_t expiration, uint64_t connect_timeout, uint64_t rpc_timeout,
+                                       bool ipv6 = false);
     extern "C" StubPool* new_uds_stub_pool(const char* path, uint64_t expiration,
                                 uint64_t connect_timeout,
                                 uint64_t rpc_timeout);
diff --git a/thread/test/test.cpp b/thread/test/test.cpp
index 1e1b4c5a..4fd2a171 100644
--- a/thread/test/test.cpp
+++ b/thread/test/test.cpp
@@ -712,7 +712,6 @@ TEST(Sleep, sleep_only_thread) {    //Sleep_sleep_only_thread_Test::TestBody
 void *func1(void *)
 {
     photon::thread_sleep(rand()%5);
-    LOG_INFO("hello");
     return nullptr;
 }
 
@@ -757,14 +756,11 @@ TEST(ThreadPool, multithread) {
         wp.call(
             [&] { ths[i] = pool.thread_create_ex(&::func1, nullptr, true); });
     }
-    LOG_INFO("----------");
     for (int i = 0; i < FLAGS_ths_total; i++) {
         wp.call([&] {
-            LOG_DEBUG("wait thread: `", ths[i]->th);
             pool.join(ths[i]);
         });
     }
-    LOG_INFO("???????????????");
 }
 
 thread_local uint64_t rw_count;

From fe71b2053b18e80c05f9c22ca68a20b87d43b63e Mon Sep 17 00:00:00 2001
From: Bob Chen <beef9999@qq.com>
Date: Tue, 24 Oct 2023 13:05:25 +0800
Subject: [PATCH 02/76] Prioritize epoll engine (#201)

---
 net/http/client.cpp | 2 +-
 photon.cpp          | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/net/http/client.cpp b/net/http/client.cpp
index efb6b3b0..de5f6a88 100644
--- a/net/http/client.cpp
+++ b/net/http/client.cpp
@@ -29,7 +29,7 @@ namespace photon {
 namespace net {
 namespace http {
 static const uint64_t kDNSCacheLife = 3600UL * 1000 * 1000;
-static constexpr char USERAGENT[] = "EASE/0.21.6";
+static constexpr char USERAGENT[] = "PhotonLibOS_HTTP";
 
 
 class PooledDialer {
diff --git a/photon.cpp b/photon.cpp
index a17cfe72..cafc770c 100644
--- a/photon.cpp
+++ b/photon.cpp
@@ -38,7 +38,7 @@ static thread_local uint64_t g_event_engine = 0, g_io_engine = 0;
 
 // Try to init master engine with the recommended order
 #if defined(__linux__)
-static const int recommended_order[] = {INIT_EVENT_IOURING, INIT_EVENT_EPOLL, INIT_EVENT_SELECT};
+static const int recommended_order[] = {INIT_EVENT_EPOLL, INIT_EVENT_IOURING, INIT_EVENT_SELECT};
 #else   // macOS, FreeBSD ...
 static const int recommended_order[] = {INIT_EVENT_KQUEUE, INIT_EVENT_SELECT};
 #endif

From 3ebddbbfec3f74c72c43cee9eb2d92966a676ccf Mon Sep 17 00:00:00 2001
From: Bob Chen <beef9999@qq.com>
Date: Thu, 26 Oct 2023 00:55:27 +0800
Subject: [PATCH 03/76] Auto PR

---
 .github/workflows/auto-pr.yml | 38 +++++++++++++++++++++++++++++++++++
 README.md                     |  1 +
 2 files changed, 39 insertions(+)
 create mode 100644 .github/workflows/auto-pr.yml

diff --git a/.github/workflows/auto-pr.yml b/.github/workflows/auto-pr.yml
new file mode 100644
index 00000000..58e430c1
--- /dev/null
+++ b/.github/workflows/auto-pr.yml
@@ -0,0 +1,38 @@
+name: Auto PR
+
+on:
+  push:
+    branches:
+      - 'release/*'
+
+jobs:
+  auto-pr:
+    runs-on: ubuntu-latest
+
+    env:
+      GH_TOKEN: ${{ github.token }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Create Pull Request to the next higher release version
+
+        run: |        
+          git fetch
+          RELEASE_VERSIONS=$(git branch -r | grep 'release/' | cut -d '/' -f 3 | sort -V)
+          CURRENT_VERSION=$(echo $GITHUB_REF | rev | cut -d '/' -f 1 | rev)
+          ME_AND_MY_NEXT=$(echo "$RELEASE_VERSIONS" | grep -w $CURRENT_VERSION -A 1)        
+          NUM=$(echo "$ME_AND_MY_NEXT" | wc -l)            
+          if (( NUM > 1 )); then
+            NEXT_VERSION=$(echo "$ME_AND_MY_NEXT" | tail -n 1)        
+            set -x
+            gh pr create --base "release/$NEXT_VERSION" --head "release/$CURRENT_VERSION" \
+                --title "Auto PR from release/$CURRENT_VERSION to release/$NEXT_VERSION" \
+                --body 'Created by Github action'
+          else
+            echo "No more higher release versions, will merge to main"
+            set -x
+            gh pr create --base main --head "release/$CURRENT_VERSION" \
+                --title "Auto PR from release/$CURRENT_VERSION to main" \
+                --body 'Created by Github action'
+          fi
diff --git a/README.md b/README.md
index e5336aeb..3112c56a 100644
--- a/README.md
+++ b/README.md
@@ -7,6 +7,7 @@
 [PhotonlibOS.github.io](https://photonlibos.github.io)
 
 ## What's New
+* Since 0.7, Photon will use release branches to enhance the reliability of software delivery. Bugfix will be merged into a stable release at first, then to higher release versions, and finally main.
 * Since version 0.6, Photon can run with a userspace TCP/IP stack on top of `DPDK`.
 [En](https://developer.aliyun.com/article/1208512) / [中文](https://developer.aliyun.com/article/1208390).
 * How to transform `RocksDB` from multi-threads to coroutines by only 200 lines of code?

From 13cae28d9dee435d47b1315b28143b9192a13ffb Mon Sep 17 00:00:00 2001
From: Coldwings <coldwings@me.com>
Date: Thu, 26 Oct 2023 10:05:18 +0800
Subject: [PATCH 04/76] Fix lockfree queue size and full check (#203)

Signed-off-by: Coldwings <coldwings@me.com>
---
 common/lockfree_queue.h | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/common/lockfree_queue.h b/common/lockfree_queue.h
index eb19262f..8bdcaac3 100644
--- a/common/lockfree_queue.h
+++ b/common/lockfree_queue.h
@@ -58,6 +58,9 @@ struct Capacity_2expN<0> {
 template <>
 struct Capacity_2expN<1> : public Capacity_2expN<0> {};
 
+template <>
+struct Capacity_2expN<2> : public Capacity_2expN<0> {};
+
 struct PauseBase {};
 
 struct CPUPause : PauseBase {
@@ -136,7 +139,7 @@ class LockfreeRingQueueBase {
     bool check_empty(size_t h, size_t t) const { return h == t; }
 
     bool check_full(size_t h, size_t t) const {
-        return check_mask_equal(h, t + 1);
+        return h != t && check_mask_equal(h, t);
     }
 
     size_t idx(size_t x) const { return x & mask; }

From 37055c70694aaa546d339409649eef87d9c2444a Mon Sep 17 00:00:00 2001
From: Bob Chen <beef9999@qq.com>
Date: Sat, 28 Oct 2023 00:04:56 +0800
Subject: [PATCH 05/76] Add slack and dingtalk link

---
 README.md                   |  6 ++++++
 doc/docusaurus.config.js    | 11 ++++++++---
 doc/static/img/dingtalk.svg |  1 +
 doc/static/img/slack.svg    |  1 +
 4 files changed, 16 insertions(+), 3 deletions(-)
 create mode 100644 doc/static/img/dingtalk.svg
 create mode 100644 doc/static/img/slack.svg

diff --git a/README.md b/README.md
index 3112c56a..2b79868c 100644
--- a/README.md
+++ b/README.md
@@ -29,3 +29,9 @@
 * Made the first tag on 27 Jul 2022. Fix the compatibility for ARM CPU. Throughly compared the TCP echo server performance with other libs.
 
 </p></details>
+
+## Community
+
+<img src="/doc/static/img/slack.svg" width="20"> Join Slack: [link](https://join.slack.com/t/photonlibos/shared_invite/zt-25wauq8g1-iK_oHMrXetcvWNNhIt8Nkg)
+
+<img src="/doc/static/img/dingtalk.svg" width="20"> Join DingTalk group: 55690000272
diff --git a/doc/docusaurus.config.js b/doc/docusaurus.config.js
index 38be8820..59bbb85d 100644
--- a/doc/docusaurus.config.js
+++ b/doc/docusaurus.config.js
@@ -101,9 +101,14 @@ const config = {
             title: 'Community',
             items: [
               {
-                label: 'Slack',
-                href: 'https://photonlibos.slack.com',
-              },              
+                html: `
+                    <a href="https://join.slack.com/t/photonlibos/shared_invite/zt-25wauq8g1-iK_oHMrXetcvWNNhIt8Nkg" target="_blank" rel="noreferrer noopener">
+                        <img src="/img/slack.png" alt="Slack" width="30" />
+                    </a>
+                    <a href="https://www.dingtalk.com/download?action=joingroup&code=v1,k1,Q3fyZvf3qFx7aB+9j4FkrK2K45E2g9SiufbbSueS8h0=&_dt_no_comment=1&origin=11" target="_blank" rel="noreferrer noopener" style="padding-left: 10px;">
+                        <img src="/img/dingtalk.svg" alt="Dingtalk" width="30" />
+                    </a>`
+              },
             ],
           },
           {
diff --git a/doc/static/img/dingtalk.svg b/doc/static/img/dingtalk.svg
new file mode 100644
index 00000000..51009643
--- /dev/null
+++ b/doc/static/img/dingtalk.svg
@@ -0,0 +1 @@
+<svg version="1.1" xmlns="http://www.w3.org/2000/svg" fill-rule="evenodd" clip-rule="evenodd" height="128" width="128" viewBox="0 0 1024 1024" style="width: 128px; height: 128px;"><path d="M1024 155.733333c0-85.333333-70.4-155.733333-155.733333-155.733333h-712.533334c-85.333333 0-155.733333 70.4-155.733333 155.733333v712.533334c0 85.333333 70.4 155.733333 155.733333 155.733333h712.533334c85.333333 0 155.733333-70.4 155.733333-155.733333v-712.533334z" fill="#4285F4"></path><path d="M810.666667 422.4c-2.133333 6.4-4.266667 14.933333-8.533334 23.466667-27.733333 59.733333-100.266667 174.933333-100.266666 174.933333l-21.333334 36.266667h102.4l-194.133333 258.133333 44.8-174.933333h-78.933333l27.733333-115.2c-23.466667 6.4-49.066667 12.8-81.066667 23.466666 0 0-42.666667 25.6-121.6-46.933333 0 0-53.333333-46.933333-23.466666-59.733333 12.8-4.266667 64-10.666667 104.533333-17.066667 55.466667-6.4 87.466667-10.666667 87.466667-10.666667s-168.533333 2.133333-206.933334-4.266666c-40.533333-6.4-89.6-72.533333-100.266666-132.266667 0 0-17.066667-32 36.266666-17.066667s268.8 59.733333 268.8 59.733334-281.6-87.466667-300.8-108.8c-19.2-21.333333-55.466667-115.2-51.2-172.8 0 0 2.133333-14.933333 17.066667-10.666667 0 0 209.066667 96 352 147.2 140.8 53.333333 264.533333 81.066667 247.466667 147.2z" fill="#FFFFFF"></path></svg>
\ No newline at end of file
diff --git a/doc/static/img/slack.svg b/doc/static/img/slack.svg
new file mode 100644
index 00000000..69a4eb6a
--- /dev/null
+++ b/doc/static/img/slack.svg
@@ -0,0 +1 @@
+<svg enable-background="new 0 0 2447.6 2452.5" viewBox="0 0 2447.6 2452.5" xmlns="http://www.w3.org/2000/svg"><g clip-rule="evenodd" fill-rule="evenodd"><path d="m897.4 0c-135.3.1-244.8 109.9-244.7 245.2-.1 135.3 109.5 245.1 244.8 245.2h244.8v-245.1c.1-135.3-109.5-245.1-244.9-245.3.1 0 .1 0 0 0m0 654h-652.6c-135.3.1-244.9 109.9-244.8 245.2-.2 135.3 109.4 245.1 244.7 245.3h652.7c135.3-.1 244.9-109.9 244.8-245.2.1-135.4-109.5-245.2-244.8-245.3z" fill="#36c5f0"/><path d="m2447.6 899.2c.1-135.3-109.5-245.1-244.8-245.2-135.3.1-244.9 109.9-244.8 245.2v245.3h244.8c135.3-.1 244.9-109.9 244.8-245.3zm-652.7 0v-654c.1-135.2-109.4-245-244.7-245.2-135.3.1-244.9 109.9-244.8 245.2v654c-.2 135.3 109.4 245.1 244.7 245.3 135.3-.1 244.9-109.9 244.8-245.3z" fill="#2eb67d"/><path d="m1550.1 2452.5c135.3-.1 244.9-109.9 244.8-245.2.1-135.3-109.5-245.1-244.8-245.2h-244.8v245.2c-.1 135.2 109.5 245 244.8 245.2zm0-654.1h652.7c135.3-.1 244.9-109.9 244.8-245.2.2-135.3-109.4-245.1-244.7-245.3h-652.7c-135.3.1-244.9 109.9-244.8 245.2-.1 135.4 109.4 245.2 244.7 245.3z" fill="#ecb22e"/><path d="m0 1553.2c-.1 135.3 109.5 245.1 244.8 245.2 135.3-.1 244.9-109.9 244.8-245.2v-245.2h-244.8c-135.3.1-244.9 109.9-244.8 245.2zm652.7 0v654c-.2 135.3 109.4 245.1 244.7 245.3 135.3-.1 244.9-109.9 244.8-245.2v-653.9c.2-135.3-109.4-245.1-244.7-245.3-135.4 0-244.9 109.8-244.8 245.1 0 0 0 .1 0 0" fill="#e01e5a"/></g></svg>
\ No newline at end of file

From c55212cc6fb919a02df74a616dd106e1d552d40e Mon Sep 17 00:00:00 2001
From: Bob Chen <beef9999@qq.com>
Date: Sat, 28 Oct 2023 00:50:29 +0800
Subject: [PATCH 06/76] fix auto pr merge

---
 .github/workflows/auto-pr.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/auto-pr.yml b/.github/workflows/auto-pr.yml
index 58e430c1..743fc2fe 100644
--- a/.github/workflows/auto-pr.yml
+++ b/.github/workflows/auto-pr.yml
@@ -26,13 +26,13 @@ jobs:
           if (( NUM > 1 )); then
             NEXT_VERSION=$(echo "$ME_AND_MY_NEXT" | tail -n 1)        
             set -x
-            gh pr create --base "release/$NEXT_VERSION" --head "release/$CURRENT_VERSION" \
+            gh pr create --base "origin/release/$NEXT_VERSION" --head "origin/release/$CURRENT_VERSION" \
                 --title "Auto PR from release/$CURRENT_VERSION to release/$NEXT_VERSION" \
                 --body 'Created by Github action'
           else
             echo "No more higher release versions, will merge to main"
             set -x
-            gh pr create --base main --head "release/$CURRENT_VERSION" \
+            gh pr create --base origin/main --head "origin/release/$CURRENT_VERSION" \
                 --title "Auto PR from release/$CURRENT_VERSION to main" \
                 --body 'Created by Github action'
           fi

From 62a80e97d781df300a6edae6fe8c1edd3fea1d40 Mon Sep 17 00:00:00 2001
From: Bob Chen <beef9999@qq.com>
Date: Sat, 28 Oct 2023 00:52:20 +0800
Subject: [PATCH 07/76] Revert "fix auto pr merge"

This reverts commit 3f8a6a58d49a09c59874f5dd5fa983aaf1636384.
---
 .github/workflows/auto-pr.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/auto-pr.yml b/.github/workflows/auto-pr.yml
index 743fc2fe..58e430c1 100644
--- a/.github/workflows/auto-pr.yml
+++ b/.github/workflows/auto-pr.yml
@@ -26,13 +26,13 @@ jobs:
           if (( NUM > 1 )); then
             NEXT_VERSION=$(echo "$ME_AND_MY_NEXT" | tail -n 1)        
             set -x
-            gh pr create --base "origin/release/$NEXT_VERSION" --head "origin/release/$CURRENT_VERSION" \
+            gh pr create --base "release/$NEXT_VERSION" --head "release/$CURRENT_VERSION" \
                 --title "Auto PR from release/$CURRENT_VERSION to release/$NEXT_VERSION" \
                 --body 'Created by Github action'
           else
             echo "No more higher release versions, will merge to main"
             set -x
-            gh pr create --base origin/main --head "origin/release/$CURRENT_VERSION" \
+            gh pr create --base main --head "release/$CURRENT_VERSION" \
                 --title "Auto PR from release/$CURRENT_VERSION to main" \
                 --body 'Created by Github action'
           fi

From 52f2892c6b3174ef67ab8784d8576d1499e06166 Mon Sep 17 00:00:00 2001
From: Bob Chen <beef9999@qq.com>
Date: Sat, 28 Oct 2023 00:54:05 +0800
Subject: [PATCH 08/76] fix website logo

---
 doc/docusaurus.config.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/docusaurus.config.js b/doc/docusaurus.config.js
index 59bbb85d..95103743 100644
--- a/doc/docusaurus.config.js
+++ b/doc/docusaurus.config.js
@@ -103,7 +103,7 @@ const config = {
               {
                 html: `
                     <a href="https://join.slack.com/t/photonlibos/shared_invite/zt-25wauq8g1-iK_oHMrXetcvWNNhIt8Nkg" target="_blank" rel="noreferrer noopener">
-                        <img src="/img/slack.png" alt="Slack" width="30" />
+                        <img src="/img/slack.svg" alt="Slack" width="30" />
                     </a>
                     <a href="https://www.dingtalk.com/download?action=joingroup&code=v1,k1,Q3fyZvf3qFx7aB+9j4FkrK2K45E2g9SiufbbSueS8h0=&_dt_no_comment=1&origin=11" target="_blank" rel="noreferrer noopener" style="padding-left: 10px;">
                         <img src="/img/dingtalk.svg" alt="Dingtalk" width="30" />

From 59fd6203e735b3f33022a495f9ffb660da9a1a67 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Sat, 28 Oct 2023 01:03:37 +0800
Subject: [PATCH 09/76] Auto PR from release/0.6 to main (#221)

---
 doc/README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/README.md b/doc/README.md
index 73b98790..b04e2806 100644
--- a/doc/README.md
+++ b/doc/README.md
@@ -43,3 +43,4 @@ $ GIT_USER=photonlibos yarn deploy
 ```
 
 Will push to the `main` branch and use the GitHub pages for hosting
+

From 1e9750b93bcd8f0fb3dc2ae836d866625e9a2ba6 Mon Sep 17 00:00:00 2001
From: Bob Chen <beef9999@qq.com>
Date: Sat, 28 Oct 2023 22:19:57 +0800
Subject: [PATCH 10/76] fix-http-resolve-for-ipv6

---
 .github/workflows/ci.linux.arm.yml     |  4 +-
 .github/workflows/ci.linux.x86.yml     |  4 +-
 .github/workflows/ci.macos.arm.yml     |  4 +-
 .github/workflows/ci.macos.yml         |  4 +-
 doc/docs/api/network.md                | 65 ++++++++++++++++++--------
 doc/docs/introduction/how-to-build.md  | 10 +++-
 net/http/test/client_function_test.cpp |  5 +-
 net/utils.cpp                          | 26 ++++++++---
 net/utils.h                            |  5 +-
 9 files changed, 87 insertions(+), 40 deletions(-)

diff --git a/.github/workflows/ci.linux.arm.yml b/.github/workflows/ci.linux.arm.yml
index d352064e..c7766b4b 100644
--- a/.github/workflows/ci.linux.arm.yml
+++ b/.github/workflows/ci.linux.arm.yml
@@ -2,9 +2,9 @@ name: Linux ARM
 
 on:
   push:
-    branches: [ "main" ]
+    branches: [ "main", "release/*" ]
   pull_request:
-    branches: [ "main" ]
+    branches: [ "main", "release/*" ]
 
 jobs:
   centos8-gcc921-epoll-release:
diff --git a/.github/workflows/ci.linux.x86.yml b/.github/workflows/ci.linux.x86.yml
index 79cd2cb6..940ac7ca 100644
--- a/.github/workflows/ci.linux.x86.yml
+++ b/.github/workflows/ci.linux.x86.yml
@@ -2,9 +2,9 @@ name: Linux x86
 
 on:
   push:
-    branches: [ "main" ]
+    branches: [ "main", "release/*" ]
   pull_request:
-    branches: [ "main" ]
+    branches: [ "main", "release/*" ]
 
 jobs:
   centos8-gcc921-epoll-release:
diff --git a/.github/workflows/ci.macos.arm.yml b/.github/workflows/ci.macos.arm.yml
index 391ed891..c8e17718 100644
--- a/.github/workflows/ci.macos.arm.yml
+++ b/.github/workflows/ci.macos.arm.yml
@@ -2,9 +2,9 @@ name: macOS ARM
 
 on:
   push:
-    branches: [ "main" ]
+    branches: [ "main", "release/*" ]
   pull_request:
-    branches: [ "main" ]
+    branches: [ "main", "release/*" ]
 
 jobs:
   macOS-clang-debug:
diff --git a/.github/workflows/ci.macos.yml b/.github/workflows/ci.macos.yml
index 2b40153e..5e75d552 100644
--- a/.github/workflows/ci.macos.yml
+++ b/.github/workflows/ci.macos.yml
@@ -2,9 +2,9 @@ name: macOS
 
 on:
   push:
-    branches: [ "main" ]
+    branches: [ "main", "release/*" ]
   pull_request:
-    branches: [ "main" ]
+    branches: [ "main", "release/*" ]
 
 jobs:
   macOS-12-Monterey-debug:
diff --git a/doc/docs/api/network.md b/doc/docs/api/network.md
index abd63e13..d555cfd8 100644
--- a/doc/docs/api/network.md
+++ b/doc/docs/api/network.md
@@ -23,6 +23,9 @@ Network lib provides non-blocking socket implementations for clients and servers
 ISocketClient* new_tcp_socket_client();
 ISocketServer* new_tcp_socket_server();
 
+ISocketClient* new_tcp_socket_client_ipv6();
+ISocketServer* new_tcp_socket_server_ipv6();
+
 ISocketClient* new_uds_client();
 ISocketServer* new_uds_server(bool autoremove = false);
 
@@ -42,6 +45,11 @@ ISocketClient* new_fstack_dpdk_socket_client();
 ISocketServer* new_fstack_dpdk_socket_server();
 ```
 
+:::note
+An IPv6 socket server listening on `::0` can handle both v4 and v6 socket client.
+:::
+
+
 #### Class Method
 
 ```cpp
@@ -95,36 +103,55 @@ namespace net {
 }
 ```
 
-#### IPv4 Address
+#### Address and Endpoint
 ```cpp
 namespace photon {
 namespace net {
-    union IPAddr {
-        // Save IP address as a 32 bits integer, or 4 bytes.
-        uint32_t addr = 0;
-        struct { uint8_t a, b, c, d; };
-
-        // Create from the Internet host address, in network byte order, or from string
+    struct IPAddr {
+        in6_addr addr;
+        // For compatibility, the default constructor is still 0.0.0.0 (IPv4)
+        IPAddr();
+        // V6 constructor (Internet Address)
+        explicit IPAddr(in6_addr internet_addr);
+        // V6 constructor (Network byte order)
+        IPAddr(uint32_t nl1, uint32_t nl2, uint32_t nl3, uint32_t nl4);
+        // V4 constructor (Internet Address)
+        explicit IPAddr(in_addr internet_addr);
+        // V4 constructor (Network byte order)
         explicit IPAddr(uint32_t nl);
-        explicit IPAddr(const char* s)
-        
+        // String constructor
+        explicit IPAddr(const char* s);
+        // Check if it's actually an IPv4 address mapped in IPV6
+        bool is_ipv4();
+        // We regard the default IPv4 0.0.0.0 as undefined
+        bool undefined();
+        // Should ONLY be used for IPv4 address
         uint32_t to_nl() const;
-        void from_nl(uint32_t nl);
+        bool is_loopback() const;
+        bool is_broadcast() const;
+        bool is_link_local() const;
+        bool operator==(const IPAddr& rhs) const;
+        bool operator!=(const IPAddr& rhs) const;
+        static IPAddr V6None();
+        static IPAddr V6Any();
+        static IPAddr V6Loopback();
+        static IPAddr V4Broadcast();
+        static IPAddr V4Any();
+        static IPAddr V4Loopback();
     };
 
     // EndPoint represents IP address and port
+    // A default endpoint is undefined (0.0.0.0:0)
     struct EndPoint {
         IPAddr addr;
-        uint16_t port;
-
-        // convert from the typical struct sockaddr_in
-        sockaddr_in to_sockaddr_in() const;
-        void from_sockaddr_in(const struct sockaddr_in& addr_in);
+        uint16_t port = 0;
+        EndPoint() = default;
+        EndPoint(IPAddr ip, uint16_t port);
+        bool is_ipv4() const;
+        bool operator==(const EndPoint& rhs) const;
+        bool operator!=(const EndPoint& rhs) const;
+        bool undefined() const;
     };
-
-    // operators to help logging IP addresses with alog
-    LogBuffer& operator << (LogBuffer& log, const IPAddr addr);
-    LogBuffer& operator << (LogBuffer& log, const EndPoint ep);
 }
 }
 ```
diff --git a/doc/docs/introduction/how-to-build.md b/doc/docs/introduction/how-to-build.md
index 2562bf0a..963a048a 100644
--- a/doc/docs/introduction/how-to-build.md
+++ b/doc/docs/introduction/how-to-build.md
@@ -85,8 +85,7 @@ cmake --build build -j
 
 ```bash
 cd PhotonLibOS
-# Use `brew info openssl` to find openssl path
-cmake -B build -D OPENSSL_ROOT_DIR=/path/to/openssl/
+cmake -B build
 cmake --build build -j
 ```
 
@@ -178,3 +177,10 @@ ctest
 | PHOTON_ENABLE_FSTACK_DPDK |   OFF   |          Enable F-Stack and DPDK. Requires both.          |
 |    PHOTON_ENABLE_EXTFS    |   OFF   |             Enable extfs. Requires `libe2fs`              |
 
+#### Example
+
+If there is any shared lib you don't want Photon to link to on local host, build its static from source.
+
+```bash
+cmake -B build -D PHOTON_BUILD_DEPENDENCIES=ON -D PHOTON_GFLAGS_SOURCE=https://github.com/gflags/gflags/archive/refs/tags/v2.2.2.tar.gz
+```
diff --git a/net/http/test/client_function_test.cpp b/net/http/test/client_function_test.cpp
index 29151c0d..7af2d2df 100644
--- a/net/http/test/client_function_test.cpp
+++ b/net/http/test/client_function_test.cpp
@@ -84,11 +84,12 @@ TEST(http_client, get) {
     auto op2 = client->new_operation(Verb::GET, target);
     DEFER(delete op2);
     op2->req.headers.content_length(0);
-    client->call(op2);
+    int ret = client->call(op2);
+    GTEST_ASSERT_EQ(0, ret);
 
     char resp_body_buf[1024];
     EXPECT_EQ(sizeof(socket_buf), op2->resp.resource_size());
-    auto ret = op2->resp.read(resp_body_buf, sizeof(socket_buf));
+    ret = op2->resp.read(resp_body_buf, sizeof(socket_buf));
     EXPECT_EQ(sizeof(socket_buf), ret);
     resp_body_buf[sizeof(socket_buf) - 1] = '\0';
     LOG_DEBUG(resp_body_buf);
diff --git a/net/utils.cpp b/net/utils.cpp
index 28827d77..2137b47e 100644
--- a/net/utils.cpp
+++ b/net/utils.cpp
@@ -256,23 +256,34 @@ bool Base64Decode(std::string_view in, std::string &out) {
 
 class DefaultResolver : public Resolver {
 public:
-    DefaultResolver(uint64_t cache_ttl, uint64_t resolve_timeout)
-        : dnscache_(cache_ttl), resolve_timeout_(resolve_timeout) {}
+    DefaultResolver(uint64_t cache_ttl, uint64_t resolve_timeout, bool ipv6)
+        : dnscache_(cache_ttl), resolve_timeout_(resolve_timeout), ipv6_(ipv6) {}
     ~DefaultResolver() { dnscache_.clear(); }
 
     IPAddr resolve(const char *host) override {
         auto ctr = [&]() -> IPAddr * {
-            auto *ip = new IPAddr();
+            std::vector<IPAddr> addrs;
             photon::semaphore sem;
             std::thread([&]() {
-                *ip = gethostbyname(host);
+                int ret = gethostbyname(host, addrs);
+                if (ret < 0) {
+                    addrs.clear();
+                }
                 sem.signal(1);
             }).detach();
             auto ret = sem.wait(1, resolve_timeout_);
             if (ret < 0 && errno == ETIMEDOUT) {
                 LOG_WARN("Domain resolution for ` timeout!", host);
+                return new IPAddr;  // undefined addr
+            } else if (addrs.empty()) {
+                LOG_WARN("Domain resolution for ` failed", host);
+                return new IPAddr;  // undefined addr
             }
-            return ip;
+            for (auto& each : addrs) {
+                if ((each.is_ipv4() ^ !ipv6_) == 0)
+                    return new IPAddr(each);
+            }
+            return new IPAddr;      // undefined addr
         };
         return *(dnscache_.borrow(host, ctr));
     }
@@ -287,10 +298,11 @@ class DefaultResolver : public Resolver {
 private:
     ObjectCache<std::string, IPAddr *> dnscache_;
     uint64_t resolve_timeout_;
+    bool ipv6_;
 };
 
-Resolver* new_default_resolver(uint64_t cache_ttl, uint64_t resolve_timeout) {
-    return new DefaultResolver(cache_ttl, resolve_timeout);
+Resolver* new_default_resolver(uint64_t cache_ttl, uint64_t resolve_timeout, bool ipv6) {
+    return new DefaultResolver(cache_ttl, resolve_timeout, ipv6);
 }
 
 }  // namespace net
diff --git a/net/utils.h b/net/utils.h
index 9420686e..b5457c3a 100644
--- a/net/utils.h
+++ b/net/utils.h
@@ -153,7 +153,7 @@ bool zerocopy_available();
  */
 class Resolver : public Object {
 public:
-    // When failed, IPAddr(0) should be returned.
+    // When failed, return an Undefined IPAddr
     // Normally dns servers return multiple ips in random order, choosing the first one should suffice.
     virtual IPAddr resolve(const char* host) = 0;
     virtual void resolve(const char* host, Delegate<void, IPAddr> func) = 0;
@@ -165,9 +165,10 @@ class Resolver : public Object {
  *
  * @param cache_ttl cache's lifetime in microseconds.
  * @param resolve_timeout timeout in microseconds for domain resolution.
+ * @param ipv6 specify v4 or v6 domain name
  * @return Resolver*
  */
-Resolver* new_default_resolver(uint64_t cache_ttl = 3600UL * 1000000, uint64_t resolve_timeout = -1);
+Resolver* new_default_resolver(uint64_t cache_ttl = 3600UL * 1000000, uint64_t resolve_timeout = -1, bool ipv6 = false);
 
 }  // namespace net
 }

From 2f547d24c7ca3649055f70ead91fbd67e1fe2fde Mon Sep 17 00:00:00 2001
From: Huiba Li <lihuiba@gmail.com>
Date: Mon, 30 Oct 2023 13:41:24 +0800
Subject: [PATCH 11/76] add a new function recv_at_least() to socket stream
 (#224)

add new functions recv_at_least() and recv_at_least_mutable() to socket stream
---
 net/basic_socket.cpp | 27 +++++++++++++++++++++++++++
 net/socket.h         |  4 ++++
 2 files changed, 31 insertions(+)

diff --git a/net/basic_socket.cpp b/net/basic_socket.cpp
index b5b2ec83..bd54f8fd 100644
--- a/net/basic_socket.cpp
+++ b/net/basic_socket.cpp
@@ -262,6 +262,33 @@ bool ISocketStream::skip_read(size_t count) {
     return true;
 }
 
+ssize_t ISocketStream::recv_at_least(void* buf, size_t count, size_t least, int flags) {
+    size_t n = 0;
+    do {
+        ssize_t ret = this->recv(buf, count, flags);
+        if (ret < 0) return ret;
+        if (ret == 0) break;    // EOF
+        if ((n += ret) >= least) break;
+        count -= ret;
+    } while (count);
+    return n;
+}
+
+ssize_t ISocketStream::recv_at_least_mutable(struct iovec *iov, int iovcnt,
+                                             size_t least, int flags /*=0*/) {
+    size_t n = 0;
+    iovector_view v(iov, iovcnt);
+    do {
+        ssize_t ret = this->recv(v.iov, v.iovcnt, flags);
+        if (ret < 0) return ret;
+        if (ret == 0) break;    // EOF
+        if ((n += ret) >= least) break;
+        auto r = v.extract_front(ret);
+        assert(r == ret); (void)r;
+    } while (v.iovcnt && v.iov->iov_len);
+    return n;
+}
+
 int do_get_name(int fd, Getter getter, EndPoint& addr) {
     sockaddr_storage storage;
     socklen_t len = storage.get_max_socklen();
diff --git a/net/socket.h b/net/socket.h
index 15a89c6f..719e2436 100644
--- a/net/socket.h
+++ b/net/socket.h
@@ -217,6 +217,10 @@ namespace net {
         virtual ssize_t recv(void *buf, size_t count, int flags = 0) = 0;
         virtual ssize_t recv(const struct iovec *iov, int iovcnt, int flags = 0) = 0;
 
+        // recv at `least` bytes to buffer (`buf`, `count`)
+        ssize_t recv_at_least(void* buf, size_t count, size_t least, int flags = 0);
+        ssize_t recv_at_least_mutable(struct iovec *iov, int iovcnt, size_t least, int flags = 0);
+
         // read count bytes and drop them
         // return true/false for success/failure
         bool skip_read(size_t count);

From 16d75b539820cf4b9c9d874a2a7ce6e9ba57893b Mon Sep 17 00:00:00 2001
From: Coldwings <coldwings@me.com>
Date: Fri, 3 Nov 2023 13:32:48 +0800
Subject: [PATCH 12/76] Fix thread-pool join threads in multiple vCPUs (#228)

Signed-off-by: Coldwings <coldwings@me.com>
---
 common/identity-pool.cpp |  2 +-
 thread/thread-pool.cpp   | 75 +++++++++++++++++++++++++---------------
 thread/thread-pool.h     |  3 ++
 3 files changed, 51 insertions(+), 29 deletions(-)

diff --git a/common/identity-pool.cpp b/common/identity-pool.cpp
index e523869a..d942d597 100644
--- a/common/identity-pool.cpp
+++ b/common/identity-pool.cpp
@@ -49,8 +49,8 @@ void IdentityPoolBase::put(void* obj)
             m_mtx.lock();
         }
         --m_refcnt;
+        m_cvar.notify_all();
     }
-    m_cvar.notify_all();
     assert(m_size <= m_capacity);
 }
 
diff --git a/thread/thread-pool.cpp b/thread/thread-pool.cpp
index 7c0f7698..64a3e511 100644
--- a/thread/thread-pool.cpp
+++ b/thread/thread-pool.cpp
@@ -29,62 +29,81 @@ namespace photon
             pCtrl->joining = false;
             pCtrl->start = start;
             pCtrl->arg = arg;
+            pCtrl->cvar.notify_one();
         }
-        pCtrl->cvar.notify_one();
         return pCtrl;
     }
+    bool ThreadPoolBase::wait_for_work(TPControl &ctrl)
+    {
+        SCOPED_LOCK(ctrl.m_mtx);
+        while (!ctrl.start)                     // wait for `create()` to give me
+            ctrl.cvar.wait(ctrl.m_mtx);           // thread_entry and argument
+
+        if (ctrl.start == &stub)
+            return false;
+
+        ((partial_thread*) CURRENT)->tls = nullptr;
+        return true;
+    }
+    bool ThreadPoolBase::after_work_done(TPControl &ctrl)
+    {
+        SCOPED_LOCK(ctrl.m_mtx);
+        auto ret = !ctrl.joining;
+        if (ctrl.joining) {
+            assert(ctrl.joinable);
+            ctrl.cvar.notify_all();
+        } else if (ctrl.joinable) {
+            ctrl.joining = true;
+            ctrl.cvar.wait(ctrl.m_mtx);
+        }
+        ctrl.joinable = false;
+        ctrl.joining = false;
+        ctrl.start = nullptr;
+        return ret;
+    }
     void* ThreadPoolBase::stub(void* arg)
     {
         TPControl ctrl;
         auto th = *(thread**)arg;
         *(TPControl**)arg = &ctrl;              // tell ctor where my `ctrl` is
         thread_yield_to(th);
-        while(true)
+        while(wait_for_work(ctrl))
         {
-            {
-                SCOPED_LOCK(ctrl.m_mtx);
-                while (!ctrl.start)                     // wait for `create()` to give me
-                    ctrl.cvar.wait(ctrl.m_mtx);           // thread_entry and argument
-
-                if (ctrl.start == &stub)
-                    break;
-                ((partial_thread*) CURRENT)->tls = nullptr;
-            }
             ctrl.start(ctrl.arg);
             deallocate_tls();
-            {
-                SCOPED_LOCK(ctrl.m_mtx);
-                if (ctrl.joining) {
-                    assert(ctrl.joinable);
-                    ctrl.cvar.notify_all();
-                } else if (ctrl.joinable) {
-                    ctrl.joining = true;
-                    ctrl.cvar.wait(ctrl.m_mtx);
-                }
-                ctrl.joinable = false;
-                ctrl.joining = false;
-                ctrl.start = nullptr;
+            auto should_put = after_work_done(ctrl);
+            if (should_put) {
+                // if has no other joiner waiting
+                // collect it into pool
+                ctrl.pool->put(&ctrl);
             }
-            ctrl.pool->put(&ctrl);
         }
         return nullptr;
     }
-    void ThreadPoolBase::join(TPControl* pCtrl)
+    bool ThreadPoolBase::do_thread_join(TPControl* pCtrl)
     {
         SCOPED_LOCK(pCtrl->m_mtx);
         if (!pCtrl->joinable)
-            LOG_ERROR_RETURN(EINVAL, , "thread is not joinable");
+            LOG_ERROR_RETURN(EINVAL, false, "thread is not joinable");
         if (!pCtrl->start)
-            LOG_ERROR_RETURN(EINVAL, , "thread is not running");
+            LOG_ERROR_RETURN(EINVAL, false, "thread is not running");
         if (pCtrl->start == &stub)
-            LOG_ERROR_RETURN(EINVAL, , "thread is dying");
+            LOG_ERROR_RETURN(EINVAL, false, "thread is dying");
 
+        auto ret = !pCtrl->joining;
         if (pCtrl->joining) {
             pCtrl->cvar.notify_one();
         } else {
             pCtrl->joining = true;
             pCtrl->cvar.wait(pCtrl->m_mtx);
         }
+        return ret;
+    }
+    void ThreadPoolBase::join(TPControl* pCtrl)
+    {
+        auto should_put = do_thread_join(pCtrl);
+        if (should_put)
+            pCtrl->pool->put(pCtrl);
     }
     int ThreadPoolBase::ctor(ThreadPoolBase* pool, TPControl** out)
     {
diff --git a/thread/thread-pool.h b/thread/thread-pool.h
index a41405be..448f29c9 100644
--- a/thread/thread-pool.h
+++ b/thread/thread-pool.h
@@ -67,6 +67,9 @@ namespace photon
         static void* stub(void* arg);
         static int ctor(ThreadPoolBase*, TPControl**);
         static int dtor(ThreadPoolBase*, TPControl*);
+        static bool wait_for_work(TPControl &ctrl);
+        static bool after_work_done(TPControl &ctrl);
+        static bool do_thread_join(TPControl* pCtrl);
         void init(uint64_t stack_size)
         {
             set_ctor({this, &ctor});

From ec0f92d0e96a8055a0bfc17546e7bcc74e4a8830 Mon Sep 17 00:00:00 2001
From: Bob Chen <beef9999@qq.com>
Date: Mon, 6 Nov 2023 10:29:46 +0800
Subject: [PATCH 13/76] add-sync-primitive-example (#232)

---
 examples/CMakeLists.txt                    |   3 +
 examples/sync-primitive/sync-primitive.cpp | 144 +++++++++++++++++++++
 2 files changed, 147 insertions(+)
 create mode 100644 examples/sync-primitive/sync-primitive.cpp

diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index 814ede66..fd5056b6 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -13,6 +13,9 @@ target_link_libraries(rpc-example-client PRIVATE photon_static ${testing_libs})
 add_executable(rpc-example-server rpc/server.cpp rpc/server_main.cpp)
 target_link_libraries(rpc-example-server PRIVATE photon_static ${testing_libs})
 
+add_executable(sync-primitive sync-primitive/sync-primitive.cpp)
+target_link_libraries(sync-primitive PRIVATE photon_static ${testing_libs})
+
 if (ENABLE_FSTACK_DPDK)
     add_executable(fstack-dpdk-demo fstack-dpdk/fstack-dpdk-demo.cpp)
     target_link_libraries(fstack-dpdk-demo PRIVATE fstack_dpdk photon_static ${testing_libs})
diff --git a/examples/sync-primitive/sync-primitive.cpp b/examples/sync-primitive/sync-primitive.cpp
new file mode 100644
index 00000000..61977b9a
--- /dev/null
+++ b/examples/sync-primitive/sync-primitive.cpp
@@ -0,0 +1,144 @@
+/*
+Copyright 2022 The Photon Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// This is an example of synchronization primitives, also comparing the performance of
+// std::condition_variable and photon::semaphore. Note that std::binary_semaphore was not added
+// until C++20. Before that, cv is the only alternative.
+//
+// Test results:
+// std:     QPS 1231796, latency 8303 ns
+// Photon:  QPS 1102088, latency 10051 ns
+
+#include <mutex>
+#include <condition_variable>
+#include <chrono>
+
+#include <photon/photon.h>
+#include <photon/common/alog.h>
+#include <photon/common/lockfree_queue.h>
+#include <photon/thread/workerpool.h>
+#include <photon/thread/thread11.h>
+
+// Comment/uncomment this macro to trigger std or photon
+#define PHOTON
+
+struct Message {
+    photon::semaphore sem;
+    std::mutex mu;
+    std::condition_variable cv;
+    bool done = false;
+    std::chrono::time_point<std::chrono::steady_clock> start;
+};
+
+static const int num_producers = 16, num_consumers = 16;
+static LockfreeMPMCRingQueue<Message*, 1024 * 1024> ring;
+static std::atomic<uint64_t> qps{0}, latency{0};
+
+__attribute__ ((noinline))
+static void do_fill(char* buf, size_t size) {
+    memset(buf, 0, size);
+}
+
+// A function that costs approximately 1 us
+static void func_1us() {
+    for (int i = 0; i < 5; ++i) {
+        constexpr size_t size = 32 * 1024;
+        char buf[size];
+        do_fill(buf, size);
+    }
+}
+
+int main() {
+    photon::init(photon::INIT_EVENT_DEFAULT, photon::INIT_IO_NONE);
+    photon::WorkPool wp(num_producers, photon::INIT_EVENT_DEFAULT, photon::INIT_IO_NONE, 128 * 1024);
+
+    // Setup some producer vCPUs/threads, continuously adding messages into the lock-free ring queue.
+    // Use semaphore.wait() for cv.wait() to wait for the messages been processed by the consumers.
+#ifdef PHOTON
+    for (int i = 0; i < num_producers; ++i) {
+        wp.async_call(new auto([] {
+            while (true) {
+                func_1us();
+                Message message;
+                ring.push(&message);
+                message.sem.wait(1);
+                auto end = std::chrono::steady_clock::now();
+                auto duration_us = std::chrono::duration_cast<std::chrono::nanoseconds>(end - message.start).count();
+                latency.fetch_add(duration_us, std::memory_order::memory_order_relaxed);
+                qps.fetch_add(1, std::memory_order::memory_order_relaxed);
+            }
+        }));
+    }
+#else
+    for (int i = 0; i < num_producers; ++i) {
+        new std::thread([] {
+            while (true) {
+                func_1us();
+                Message message;
+                ring.push(&message);
+                {
+                    std::unique_lock l(message.mu);
+                    message.cv.wait(l, [&] { return message.done; });
+                }
+                auto end = std::chrono::steady_clock::now();
+                auto duration_us = std::chrono::duration_cast<std::chrono::nanoseconds>(end - message.start).count();
+                latency.fetch_add(duration_us, std::memory_order::memory_order_relaxed);
+                qps.fetch_add(1, std::memory_order::memory_order_relaxed);
+            }
+        });
+    }
+#endif
+
+    // Setup some consumer std::threads, busy polling.
+    // Continuously process messages from the ring queue, and then notify the consumer.
+    for (int i = 0; i < num_consumers; ++i) {
+        new std::thread([&] {
+            while (true) {
+                func_1us();
+                Message* m;
+                bool ok = ring.pop_weak(m);
+                if (!ok)
+                    continue;
+                m->start = std::chrono::steady_clock::now();
+#ifdef PHOTON
+                m->sem.signal(1);
+#else
+                {
+                    std::unique_lock l(m->mu);
+                    m->done = true;
+                    m->cv.notify_one();
+                }
+#endif
+            }
+        });
+    }
+
+    // Show QPS and latency
+    photon::thread_create11([&] {
+        photon::init(photon::INIT_EVENT_DEFAULT, photon::INIT_IO_NONE);
+        while (true) {
+            photon::thread_sleep(1);
+            auto prev_qps = qps.exchange(0, std::memory_order_seq_cst);
+            auto prev_lat = latency.exchange(0, std::memory_order_seq_cst);
+            if (prev_qps != 0)
+                prev_lat = prev_lat / prev_qps;
+            LOG_INFO("QPS `, latency ` ns", prev_qps, prev_lat);
+        }
+    });
+
+    photon::thread_sleep(-1);
+}
+

From 8a48daeca22b2c4d033a20475d9054c0df4927de Mon Sep 17 00:00:00 2001
From: Huiba Li <lihuiba@gmail.com>
Date: Tue, 7 Nov 2023 09:33:03 +0800
Subject: [PATCH 14/76] minor improvements to http (#235)

* minor improvements to http
---
 common/estring.h     |   8 +---
 net/http/headers.cpp |   5 ---
 net/http/headers.h   |   6 ++-
 net/http/message.cpp |  35 +++++++--------
 net/http/message.h   |   2 +-
 net/http/status.cpp  | 101 ++++++++++++++++++++++++++++++++++++++++++-
 6 files changed, 123 insertions(+), 34 deletions(-)

diff --git a/common/estring.h b/common/estring.h
index 10464b21..9e57214f 100644
--- a/common/estring.h
+++ b/common/estring.h
@@ -321,12 +321,8 @@ class rstring_view  // relative string_view, that stores values relative
     }
 
 public:
-    constexpr rstring_view() = default; // note: don't give _offset or _length
-                                        // an initial value and do not
-                                        // assigned them in default ctor()
-                                        // it would caused verb_init() error
-    rstring_view(uint64_t offset, uint64_t length)
-    {
+    constexpr rstring_view() = default;
+    constexpr rstring_view(uint64_t offset, uint64_t length) {
         assert(offset <= std::numeric_limits<OffsetType>::max());
         assert(length <= std::numeric_limits<LengthType>::max());
         _offset = (OffsetType)offset;
diff --git a/net/http/headers.cpp b/net/http/headers.cpp
index de85a649..39f262e4 100644
--- a/net/http/headers.cpp
+++ b/net/http/headers.cpp
@@ -64,11 +64,6 @@ HeadersBase::iterator HeadersBase::find(std::string_view key) const {
     return {this, (uint16_t)(it - kv_begin())};
 }
 
-void buf_append(char*& ptr, std::string_view sv) {
-    memcpy(ptr, sv.data(), sv.size());
-    ptr += sv.size();
-}
-
 void buf_append(char*& ptr, uint64_t x) {
     auto begin = ptr;
     do {
diff --git a/net/http/headers.h b/net/http/headers.h
index 3153d165..05b2fd64 100644
--- a/net/http/headers.h
+++ b/net/http/headers.h
@@ -29,9 +29,13 @@ namespace photon {
 namespace net {
 namespace http {
 
-void buf_append(char*& ptr, std::string_view sv);
 void buf_append(char*& ptr, uint64_t x);
 
+inline void buf_append(char*& ptr, std::string_view sv) {
+    memcpy(ptr, sv.data(), sv.size());
+    ptr += sv.size();
+}
+
 class HeadersBase {
 public:
     HeadersBase() = default;
diff --git a/net/http/message.cpp b/net/http/message.cpp
index 395d10e4..ab8e3179 100644
--- a/net/http/message.cpp
+++ b/net/http/message.cpp
@@ -41,7 +41,7 @@ Message::~Message() {
         delete m_stream;
     }
 
-    if (m_buf_ownership && m_buf)
+    if (m_buf_ownership)
         free(m_buf);
 }
 
@@ -105,11 +105,14 @@ int Message::append_bytes(uint16_t size) {
         return parse_res;
     }
 
-    LOG_DEBUG("add headers, header buf size `", m_buf + m_buf_capacity - p.cur());
-    if (headers.reset(p.cur(), m_buf + m_buf_capacity - p.cur(), m_buf + m_buf_size - p.cur()) < 0)
+    auto buf_cap = m_buf + m_buf_capacity - p.cur();
+    auto buf_size = m_buf + m_buf_size - p.cur();
+    LOG_DEBUG("add headers: ", VALUE(buf_cap), VALUE(buf_size));
+    if (headers.reset(p.cur(), buf_cap, buf_size) < 0)
         LOG_ERRNO_RETURN(0, -1, "failed to parse headers");
 
-    m_abandon = (headers.get_value("Connection") == "close") || (!headers.get_value("Trailer").empty());
+    m_abandon = (headers["Connection"] == "close" ||
+                !headers["Trailer"].empty());
     message_status = HEADER_PARSED;
     LOG_DEBUG("header parsed");
     return 0;
@@ -118,11 +121,9 @@ int Message::append_bytes(uint16_t size) {
 int Message::send_header(net::ISocketStream* stream) {
     if (stream != nullptr) m_stream = stream; // update stream if needed
 
-    if (m_keep_alive)
-        headers.insert("Connection", "keep-alive");
-    else
-        headers.insert("Connection", "close");
-
+    using SV = std::string_view;
+    headers.insert("Connection", m_keep_alive ? SV("keep-alive") :
+                                                SV("close"));
     if (headers.space_remain() < 2)
         LOG_ERRNO_RETURN(ENOBUFS, -1, "no buffer");
 
@@ -322,17 +323,15 @@ int Request::reset(Verb v, std::string_view url, bool enable_proxy) {
 
 int Request::redirect(Verb v, estring_view location, bool enable_proxy) {
     estring full_location;
-    if (!location.starts_with(http_url_scheme) && (!location.starts_with(https_url_scheme))) {
+    if (!location.starts_with(http_url_scheme) &&
+        !location.starts_with(https_url_scheme)) {
         full_location.appends(secure() ? https_url_scheme : http_url_scheme,
                              host(), location);
         location = full_location;
     }
     StoredURL u(location);
-    auto new_request_line_size = verbstr[v].size() + sizeof(" HTTP/1.1\r\n");
-    if (enable_proxy)
-        new_request_line_size += full_url_size(u);
-    else
-        new_request_line_size += u.target().size();
+    auto new_request_line_size = verbstr[v].size() + sizeof(" HTTP/1.1\r\n") +
+        enable_proxy ? full_url_size(u) : u.target().size();
 
     auto delta = new_request_line_size - m_buf_size;
     LOG_DEBUG(VALUE(delta));
@@ -381,11 +380,9 @@ int Response::set_result(int code, std::string_view reason) {
     char* buf = m_buf;
     m_status_code = code;
     buf_append(buf, "HTTP/1.1 ");
-    buf_append(buf, std::to_string(code));
+    buf_append(buf, code);
     buf_append(buf, " ");
-    auto message = reason;
-    if (message.empty()) message = obsolete_reason(code);
-    buf_append(buf, message);
+    buf_append(buf, reason.size() ? reason : obsolete_reason(code));
     buf_append(buf, "\r\n");
     m_buf_size = buf - m_buf;
     headers.reset(m_buf + m_buf_size, m_buf_capacity - m_buf_size);
diff --git a/net/http/message.h b/net/http/message.h
index e8e8859f..95c5e747 100644
--- a/net/http/message.h
+++ b/net/http/message.h
@@ -57,7 +57,7 @@ class Message : public IStream {
     void reset(void* buf, uint16_t buf_capacity, bool buf_ownership = false,
                 ISocketStream* s = nullptr, bool stream_ownership = false) {
         reset(s, stream_ownership);
-        if (m_buf_ownership && m_buf) {
+        if (m_buf_ownership) {
             free(m_buf);
         }
         m_buf = (char*)buf;
diff --git a/net/http/status.cpp b/net/http/status.cpp
index ef0ba126..849474fc 100644
--- a/net/http/status.cpp
+++ b/net/http/status.cpp
@@ -17,7 +17,103 @@ limitations under the License.
 #include "photon/common/string_view.h"
 #include "message.h"
 
+struct SV : public std::string_view {
+	template<size_t N>
+	constexpr SV(const char(&s)[N]) : SV(s, N) { }
+	constexpr SV(const char* s, size_t n) : std::string_view(s, n) { }
+};
+
+constexpr static SV code_str[] = {
+	/*100*/ "Continue",
+	/*101*/ "Switching Protocols",
+	/*102*/ "Processing",
+	/*103*/ "Early Hints",
+
+	/*200*/ "OK",
+	/*201*/ "Created",
+	/*202*/ "Accepted",
+	/*203*/ "Non-Authoritative Information",
+	/*204*/ "No Content",
+	/*205*/ "Reset Content",
+	/*206*/ "Partial Content",
+	/*207*/ "Multi-Status",
+	/*208*/ "Already Reported",
+//	/*226*/ "IM Used",
+
+	/*300*/ "Multiple Choices",
+	/*301*/ "Moved Permanently",
+	/*302*/ "Found",
+	/*303*/ "See Other",
+	/*304*/ "Not Modified",
+	/*305*/ "Use Proxy",
+	/*306*/ {0, 0},
+	/*307*/ "Temporary Redirect",
+	/*308*/ "Permanent Redirect",
+
+	/*400*/ "Bad Request",
+	/*401*/ "Unauthorized",
+	/*402*/ "Payment Required",
+	/*403*/ "Forbidden",
+	/*404*/ "Not Found",
+	/*405*/ "Method Not Allowed",
+	/*406*/ "Not Acceptable",
+	/*407*/ "Proxy Authentication Required",
+	/*408*/ "Request Timeout",
+	/*409*/ "Conflict",
+	/*410*/ "Gone",
+	/*411*/ "Length Required",
+	/*412*/ "Precondition Failed",
+	/*413*/ "Content Too Large",
+	/*414*/ "URI Too Long",
+	/*415*/ "Unsupported Media Type",
+	/*416*/ "Range Not Satisfiable",
+	/*417*/ "Expectation Failed",
+	/*418*/ "I'm a teapot",
+	/*419*/ {0, 0},
+	/*420*/ {0, 0},
+	/*421*/ "Misdirected Request",
+	/*422*/ "Unprocessable Content",
+	/*423*/ "Locked",
+	/*424*/ "Failed Dependency",
+	/*425*/ "Too Early",
+	/*426*/ "Upgrade Required",
+	/*427*/ {0, 0},
+	/*428*/ "Precondition Required",
+	/*429*/ "Too Many Requests",
+	/*430*/ {0, 0},
+	/*431*/ "Request Header Fields Too Large",
+//	/*451*/ "Unavailable For Legal Reasons",
+
+	/*500*/ "Internal Server Error",
+	/*501*/ "Not Implemented",
+	/*502*/ "Bad Gateway",
+	/*503*/ "Service Unavailable",
+	/*504*/ "Gateway Timeout",
+	/*505*/ "HTTP Version Not Supported",
+	/*506*/ "Variant Also Negotiates",
+	/*507*/ "Insufficient Storage",
+	/*508*/ "Loop Detected",
+	/*509*/ {0, 0},
+	/*510*/ "Not Extended",
+	/*511*/ "Network Authentication Required",
+};
+
+const static uint8_t LEN1xx = 4;
+const static uint8_t LEN2xx = 9 + LEN1xx;
+const static uint8_t LEN3xx = 9 + LEN2xx;
+const static uint8_t LEN4xx = 32 + LEN3xx;
+const static uint8_t LEN5xx = 12 + LEN4xx;
+uint8_t entries[] = {0, LEN1xx, LEN2xx, LEN3xx, LEN4xx, LEN5xx};
+
 std::string_view photon::net::http::obsolete_reason(int code) {
+	uint8_t major = code / 100 - 1;
+	uint8_t minor = code % 100;
+	if (unlikely(major > 4)) return {};
+	uint8_t max = entries[major+1] - entries[major];
+	if (unlikely(minor >= max)) return {};
+	return code_str[entries[major] + minor];
+
+/*
 	switch (code){
 
 	case 100: return "Continue";
@@ -87,6 +183,7 @@ std::string_view photon::net::http::obsolete_reason(int code) {
 	case 510: return "Not Extended";
 	case 511: return "Network Authentication Required";
 
-	default: return "";
+	default: return { };
 	}
-}
\ No newline at end of file
+*/
+}

From 277ea3644d5989feba758e09b47a845bb69bda03 Mon Sep 17 00:00:00 2001
From: Bob Chen <beef9999@qq.com>
Date: Wed, 15 Nov 2023 10:15:47 +0800
Subject: [PATCH 15/76] Some fixes for networking (#242)

1. RPC client now supports both v4 and v6. Don't need to create two clients any more.
2. HTTP client removes v6 temporarily. There are still a lot work to be done for HTTP to truly support IPv6 (url parse, domain resolve, single client that supports both sockets), leave this work to the future...
3. Backup errno when KernelSocketStream destruct
4. Endpoint adds a string constructor
5. CMake can build openssl and curl from source now
---
 .github/workflows/ci.linux.x86.yml       |  9 +--
 CMake/build-from-src.cmake               | 48 +++++++++++-
 CMakeLists.txt                           | 62 ++++++++-------
 common/checksum/test/CMakeLists.txt      |  4 +-
 common/test/CMakeLists.txt               | 16 ++--
 examples/CMakeLists.txt                  | 13 ++--
 fs/extfs/test/CMakeLists.txt             |  4 +-
 fs/test/CMakeLists.txt                   | 10 +--
 io/test/CMakeLists.txt                   | 10 +--
 net/http/client.cpp                      | 33 +++++---
 net/http/client.h                        |  2 +-
 net/http/test/CMakeLists.txt             | 16 ++--
 net/kernel_socket.cpp                    | 99 +++++++++++++++---------
 net/security-context/test/CMakeLists.txt |  6 +-
 net/socket.h                             |  1 +
 net/test/CMakeLists.txt                  | 16 ++--
 net/test/test-ipv6.cpp                   | 28 +++++--
 net/utils.cpp                            | 16 ++--
 net/utils.h                              |  3 +-
 rpc/rpc.cpp                              | 15 ++--
 rpc/rpc.h                                |  3 +-
 rpc/test/CMakeLists.txt                  |  8 +-
 thread/test/CMakeLists.txt               | 18 ++---
 tools/libphoton.mri                      |  4 +
 24 files changed, 265 insertions(+), 179 deletions(-)
 create mode 100644 tools/libphoton.mri

diff --git a/.github/workflows/ci.linux.x86.yml b/.github/workflows/ci.linux.x86.yml
index 940ac7ca..74fb7995 100644
--- a/.github/workflows/ci.linux.x86.yml
+++ b/.github/workflows/ci.linux.x86.yml
@@ -108,17 +108,14 @@ jobs:
         run: |
           dnf install -y git gcc-c++ cmake
           dnf install -y gcc-toolset-9-gcc-c++
-          dnf install -y openssl-devel libcurl-devel
-          dnf install -y epel-release
-          dnf install -y fuse-devel libgsasl-devel e2fsprogs-devel
+          dnf install -y autoconf automake libtool
 
       - name: Build
         run: |
           source /opt/rh/gcc-toolset-9/enable
           cmake -B build -D CMAKE_BUILD_TYPE=MinSizeRel \
-            -D PHOTON_BUILD_DEPENDENCIES=ON -D PHOTON_BUILD_TESTING=ON \
-            -D PHOTON_ENABLE_SASL=ON -D PHOTON_ENABLE_FUSE=ON \
-            -D PHOTON_ENABLE_EXTFS=ON \
+            -D PHOTON_BUILD_DEPENDENCIES=ON \
+            -D PHOTON_BUILD_TESTING=ON \
             -D PHOTON_ENABLE_URING=ON
           cmake --build build -j -- VERBOSE=1
 
diff --git a/CMake/build-from-src.cmake b/CMake/build-from-src.cmake
index 752240cd..ee9fcca7 100644
--- a/CMake/build-from-src.cmake
+++ b/CMake/build-from-src.cmake
@@ -1,4 +1,5 @@
 # Note: Be aware of the differences between CMake project and Makefile project.
+# Makefile project is not able to get BINARY_DIR at configuration.
 
 set(actually_built)
 
@@ -53,9 +54,12 @@ function(build_from_src [dep])
                 CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_POSITION_INDEPENDENT_CODE=ON
                 INSTALL_COMMAND ""
         )
+        if (CMAKE_BUILD_TYPE STREQUAL "Debug")
+            set(POSTFIX "_debug")
+        endif ()
         ExternalProject_Get_Property(gflags BINARY_DIR)
         set(GFLAGS_INCLUDE_DIRS ${BINARY_DIR}/include PARENT_SCOPE)
-        set(GFLAGS_LIBRARIES ${BINARY_DIR}/lib/libgflags.a ${BINARY_DIR}/lib/libgflags_nothreads.a PARENT_SCOPE)
+        set(GFLAGS_LIBRARIES ${BINARY_DIR}/lib/libgflags${POSTFIX}.a ${BINARY_DIR}/lib/libgflags_nothreads${POSTFIX}.a PARENT_SCOPE)
 
     elseif (dep STREQUAL "googletest")
         ExternalProject_Add(
@@ -70,6 +74,48 @@ function(build_from_src [dep])
         set(GOOGLETEST_INCLUDE_DIRS ${SOURCE_DIR}/googletest/include ${SOURCE_DIR}/googlemock/include PARENT_SCOPE)
         set(GOOGLETEST_LIBRARIES ${BINARY_DIR}/lib/libgmock.a ${BINARY_DIR}/lib/libgmock_main.a
                 ${BINARY_DIR}/lib/libgtest.a ${BINARY_DIR}/lib/libgtest_main.a PARENT_SCOPE)
+
+    elseif (dep STREQUAL "openssl")
+        set(BINARY_DIR ${PROJECT_BINARY_DIR}/openssl-build)
+        ExternalProject_Add(
+                openssl
+                URL ${PHOTON_OPENSSL_SOURCE}
+                URL_MD5 bad68bb6bd9908da75e2c8dedc536b29
+                BUILD_IN_SOURCE ON
+                CONFIGURE_COMMAND ./config -fPIC no-unit-test no-shared --openssldir=${BINARY_DIR} --prefix=${BINARY_DIR}
+                BUILD_COMMAND make depend -j ${NumCPU} && make -j ${NumCPU}
+                INSTALL_COMMAND make install
+        )
+        ExternalProject_Get_Property(openssl SOURCE_DIR)
+        set(OPENSSL_ROOT_DIR ${SOURCE_DIR} PARENT_SCOPE)
+        set(OPENSSL_INCLUDE_DIRS ${BINARY_DIR}/include PARENT_SCOPE)
+        set(OPENSSL_LIBRARIES ${BINARY_DIR}/lib/libssl.a ${BINARY_DIR}/lib/libcrypto.a PARENT_SCOPE)
+
+    elseif (dep STREQUAL "curl")
+        if (${OPENSSL_ROOT_DIR} STREQUAL "")
+            message(FATAL_ERROR "OPENSSL_ROOT_DIR not exist")
+        endif ()
+        set(BINARY_DIR ${PROJECT_BINARY_DIR}/curl-build)
+        ExternalProject_Add(
+                curl
+                URL ${PHOTON_CURL_SOURCE}
+                URL_MD5 a66270f11e3fbfad709600bbd1686704
+                BUILD_IN_SOURCE ON
+                CONFIGURE_COMMAND export CC=${CMAKE_C_COMPILER} && export CXX=${CMAKE_CXX_COMPILER} &&
+                    export LD=${CMAKE_LINKER} && export CFLAGS=-fPIC &&
+                    autoreconf -i && ./configure --with-ssl=${OPENSSL_ROOT_DIR}
+                    --without-libssh2 --enable-static --enable-shared=no --enable-optimize
+                    --disable-manual --without-libidn
+                    --disable-ftp --disable-file --disable-ldap --disable-ldaps
+                    --disable-rtsp --disable-dict --disable-telnet --disable-tftp
+                    --disable-pop3 --disable-imap --disable-smb --disable-smtp
+                    --disable-gopher --without-nghttp2 --enable-http
+                    --with-pic=PIC --prefix=${BINARY_DIR}
+                BUILD_COMMAND make -j ${NumCPU}
+                INSTALL_COMMAND make install
+        )
+        set(CURL_INCLUDE_DIRS ${BINARY_DIR}/include PARENT_SCOPE)
+        set(CURL_LIBRARIES ${BINARY_DIR}/lib/libcurl.a PARENT_SCOPE)
     endif ()
 
     list(APPEND actually_built ${dep})
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 8626da46..ed48b296 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -28,8 +28,8 @@ option(PHOTON_ENABLE_EXTFS "enable extfs" OFF)
 option(PHOTON_BUILD_DEPENDENCIES "" OFF)
 set(PHOTON_AIO_SOURCE "https://pagure.io/libaio/archive/libaio-0.3.113/libaio-0.3.113.tar.gz" CACHE STRING "")
 set(PHOTON_ZLIB_SOURCE "https://github.com/madler/zlib/releases/download/v1.2.13/zlib-1.2.13.tar.gz" CACHE STRING "")
-set(PHOTON_OPENSSL_SOURCE "" CACHE STRING "")
-set(PHOTON_CURL_SOURCE "" CACHE STRING "")
+set(PHOTON_OPENSSL_SOURCE "https://github.com/openssl/openssl/archive/refs/heads/OpenSSL_1_0_2-stable.tar.gz" CACHE STRING "")
+set(PHOTON_CURL_SOURCE "https://github.com/curl/curl/archive/refs/tags/curl-7_42_1.tar.gz" CACHE STRING "")
 set(PHOTON_URING_SOURCE "https://github.com/axboe/liburing/archive/refs/tags/liburing-2.3.tar.gz" CACHE STRING "")
 set(PHOTON_FUSE_SOURCE "" CACHE STRING "")
 set(PHOTON_GSASL_SOURCE "" CACHE STRING "")
@@ -96,7 +96,7 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/output)
 ################################################################################
 
 # There are two ways to handle dependencies:
-#   1. Find locally installed packages. (Static lib is suggested)
+#   1. Find locally installed packages.
 #   2. Build it from source. (Only when PHOTON_BUILD_DEPENDENCIES is set, and PHOTON_XXX_SOURCE is not empty)
 #
 # The naming conventions MUST obey:
@@ -196,21 +196,22 @@ endif ()
 
 # An object library compiles source files but does not archive or link their object files.
 add_library(photon_obj OBJECT ${PHOTON_SRC})
-target_include_directories(photon_obj PUBLIC include ${OPENSSL_INCLUDE_DIRS} ${AIO_INCLUDE_DIRS}
-        ${ZLIB_INCLUDE_DIRS} ${CURL_INCLUDE_DIRS})
+target_include_directories(photon_obj PRIVATE include ${OPENSSL_INCLUDE_DIRS} ${AIO_INCLUDE_DIRS}
+        ${ZLIB_INCLUDE_DIRS} ${CURL_INCLUDE_DIRS}
+)
 target_compile_definitions(photon_obj PRIVATE _FILE_OFFSET_BITS=64 FUSE_USE_VERSION=29)
 if (PHOTON_ENABLE_URING)
-    target_include_directories(photon_obj PUBLIC ${URING_INCLUDE_DIRS})
+    target_include_directories(photon_obj PRIVATE ${URING_INCLUDE_DIRS})
     target_compile_definitions(photon_obj PRIVATE PHOTON_URING=on)
 endif()
 if (PHOTON_ENABLE_MIMIC_VDSO)
     target_compile_definitions(photon_obj PRIVATE ENABLE_MIMIC_VDSO=on)
 endif()
 if (PHOTON_ENABLE_FSTACK_DPDK)
-    target_include_directories(photon_obj PUBLIC ${FSTACK_INCLUDE_DIRS})
+    target_include_directories(photon_obj PRIVATE ${FSTACK_INCLUDE_DIRS})
 endif()
 if (PHOTON_ENABLE_EXTFS)
-    target_include_directories(photon_obj PUBLIC ${LIBE2FS_INCLUDE_DIRS})
+    target_include_directories(photon_obj PRIVATE ${LIBE2FS_INCLUDE_DIRS})
 endif()
 if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 8.0)
     # This option is enabled by default after -std=c++17
@@ -226,9 +227,9 @@ endif ()
 set(static_deps
         easy_weak
         fstack_weak
-        ${ZLIB_LIBRARIES}
-        ${OPENSSL_LIBRARIES}
         ${CURL_LIBRARIES}
+        ${OPENSSL_LIBRARIES}
+        ${ZLIB_LIBRARIES}
 )
 set(shared_deps
         -lpthread
@@ -256,8 +257,8 @@ if (PHOTON_ENABLE_EXTFS)
     list(APPEND static_deps ${E2FS_LIBRARIES})
 endif ()
 
-# It's OK to have some `shared_deps` duplicated in `static_deps`.
-# Even though .a is suggested, we may still probably find .so in local installed dir.
+# Find out dynamic libs and append to `shared_deps`.
+# Because if not built from source, we won't know the local packages are static or shared.
 # This is for the max compatability.
 if (NOT APPLE)
     set(suffix "\.so$")
@@ -271,9 +272,6 @@ foreach (dep ${static_deps})
             break()
         endif ()
     endforeach ()
-    if (dep MATCHES "\.a$")
-        list(APPEND static_deps_archive ${dep})
-    endif ()
 endforeach ()
 
 set(version_scripts)
@@ -284,6 +282,7 @@ list(APPEND version_scripts "-Wl,--version-script=${PROJECT_SOURCE_DIR}/tools/li
 # Link photon shared lib
 add_library(photon_shared SHARED $<TARGET_OBJECTS:photon_obj>)
 set_target_properties(photon_shared PROPERTIES OUTPUT_NAME photon)
+target_include_directories(photon_shared PUBLIC include ${CURL_INCLUDE_DIRS})
 if (NOT APPLE)
     target_link_libraries(photon_shared
             PRIVATE ${version_scripts} -Wl,--whole-archive ${static_deps} -Wl,--no-whole-archive
@@ -299,32 +298,39 @@ endif ()
 # Link photon static lib
 add_library(photon_static STATIC $<TARGET_OBJECTS:photon_obj>)
 set_target_properties(photon_static PROPERTIES OUTPUT_NAME photon_sole)
+target_include_directories(photon_static PUBLIC include ${CURL_INCLUDE_DIRS})
 target_link_libraries(photon_static
         PUBLIC ${shared_deps}
         PRIVATE ${static_deps}
 )
 
-# Merge static libs into libphoton.a
-# Do NOT use this target directly.
+# Merge static libs into libphoton.a for manual distribution.
+# Do NOT link to this target directly.
 if (NOT APPLE)
-    set(merge_command ar -crT libphoton.a)
+    add_custom_target(_photon_static_archive ALL
+            COMMAND rm -rf libphoton.a
+            COMMAND ar -qcT libphoton.a $<TARGET_FILE:photon_static> $<TARGET_FILE:easy_weak> $<TARGET_FILE:fstack_weak>
+            COMMAND ar -M < ${PROJECT_SOURCE_DIR}/tools/libphoton.mri
+            DEPENDS photon_static
+            WORKING_DIRECTORY ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}
+            VERBATIM
+    )
 else ()
-    set(merge_command libtool -static -o libphoton.a)
+    add_custom_target(_photon_static_archive ALL
+            COMMAND rm -rf libphoton.a
+            COMMAND libtool -static -o libphoton.a $<TARGET_FILE:photon_static> $<TARGET_FILE:easy_weak> $<TARGET_FILE:fstack_weak>
+            DEPENDS photon_static
+            WORKING_DIRECTORY ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}
+            VERBATIM
+    )
 endif ()
-add_custom_target(_photon_static_archive ALL
-        COMMAND rm -rf libphoton.a
-        COMMAND ${merge_command} $<TARGET_FILE:photon_static> $<TARGET_FILE:easy_weak> $<TARGET_FILE:fstack_weak>
-        DEPENDS photon_static
-        WORKING_DIRECTORY ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}
-        VERBATIM
-)
 
 # Build test cases
 if (PHOTON_BUILD_TESTING)
     include(CTest)
 
-    set(testing_libs ${GFLAGS_LIBRARIES} ${GOOGLETEST_LIBRARIES})
-    include_directories(${GFLAGS_INCLUDE_DIRS} ${GOOGLETEST_INCLUDE_DIRS})
+    include_directories(photon_static ${GFLAGS_INCLUDE_DIRS} ${GOOGLETEST_INCLUDE_DIRS})
+    link_libraries(${GFLAGS_LIBRARIES} ${GOOGLETEST_LIBRARIES})
 
     add_subdirectory(examples)
     add_subdirectory(common/checksum/test)
diff --git a/common/checksum/test/CMakeLists.txt b/common/checksum/test/CMakeLists.txt
index 6e4bd9a3..f3abd1c9 100644
--- a/common/checksum/test/CMakeLists.txt
+++ b/common/checksum/test/CMakeLists.txt
@@ -1,8 +1,6 @@
-include_directories(${CMAKE_SOURCE_DIR}/include)
-link_directories(${CMAKE_LIBRARY_OUTPUT_DIRECTORY})
 SET(TEST_WORKING_DIR ${CMAKE_CURRENT_SOURCE_DIR}/)
 ADD_DEFINITIONS(-w -DDATA_DIR=${TEST_WORKING_DIR})
 
 add_executable(test-checksum test_checksum.cpp)
-target_link_libraries(test-checksum PRIVATE photon_shared ${testing_libs})
+target_link_libraries(test-checksum PRIVATE photon_shared)
 add_test(NAME test-checksum COMMAND $<TARGET_FILE:test-checksum>)
\ No newline at end of file
diff --git a/common/test/CMakeLists.txt b/common/test/CMakeLists.txt
index 906a7b4e..09497252 100644
--- a/common/test/CMakeLists.txt
+++ b/common/test/CMakeLists.txt
@@ -1,32 +1,30 @@
-include_directories(${CMAKE_SOURCE_DIR}/include)
-link_directories(${CMAKE_LIBRARY_OUTPUT_DIRECTORY})
 SET(TEST_WORKING_DIR ${CMAKE_CURRENT_SOURCE_DIR}/)
 ADD_DEFINITIONS(-w -DDATA_DIR=${TEST_WORKING_DIR})
 
 add_executable(test-objcache test_objcache.cpp)
-target_link_libraries(test-objcache PRIVATE photon_shared ${testing_libs})
+target_link_libraries(test-objcache PRIVATE photon_shared)
 add_test(NAME test-objcache COMMAND $<TARGET_FILE:test-objcache>)
 
 add_executable(test-common test.cpp)
-target_link_libraries(test-common PRIVATE photon_shared ${testing_libs})
+target_link_libraries(test-common PRIVATE photon_shared)
 add_test(NAME test-common COMMAND $<TARGET_FILE:test-common>)
 
 add_executable(test-scalepool test_scalepool.cpp)
-target_link_libraries(test-scalepool PRIVATE photon_shared ${testing_libs})
+target_link_libraries(test-scalepool PRIVATE photon_shared)
 add_test(NAME test-scalepool COMMAND $<TARGET_FILE:test-scalepool>)
 
 add_executable(test-throttle test_throttle.cpp)
-target_link_libraries(test-throttle PRIVATE photon_shared ${testing_libs})
+target_link_libraries(test-throttle PRIVATE photon_shared)
 add_test(NAME test-throttle COMMAND $<TARGET_FILE:test-throttle>)
 
 add_executable(test-constexprstr test_constexprstr.cpp)
-target_link_libraries(test-constexprstr PRIVATE photon_shared ${testing_libs})
+target_link_libraries(test-constexprstr PRIVATE photon_shared)
 add_test(NAME test-constexprstr COMMAND $<TARGET_FILE:test-constexprstr>)
 
 #add_executable(test-lockfree test_lockfree.cpp)
-#target_link_libraries(test-lockfree PRIVATE photon_shared ${testing_libs})
+#target_link_libraries(test-lockfree PRIVATE photon_shared)
 #add_test(NAME test-lockfree COMMAND $<TARGET_FILE:test-lockfree>)
 
 add_executable(test-alog test_alog.cpp x.cpp)
-target_link_libraries(test-alog PRIVATE photon_shared ${testing_libs})
+target_link_libraries(test-alog PRIVATE photon_shared)
 add_test(NAME test-alog COMMAND $<TARGET_FILE:test-alog>)
diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index fd5056b6..5c8b91e4 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -1,22 +1,19 @@
-include_directories(${CMAKE_SOURCE_DIR}/include)
-link_directories(${CMAKE_LIBRARY_OUTPUT_DIRECTORY})
-
 add_executable(simple-example simple/simple.cpp)
 target_link_libraries(simple-example PRIVATE photon_static)
 
 add_executable(net-perf perf/net-perf.cpp)
-target_link_libraries(net-perf PRIVATE photon_static ${testing_libs})
+target_link_libraries(net-perf PRIVATE photon_static)
 
 add_executable(rpc-example-client rpc/client.cpp rpc/client_main.cpp)
-target_link_libraries(rpc-example-client PRIVATE photon_static ${testing_libs})
+target_link_libraries(rpc-example-client PRIVATE photon_static)
 
 add_executable(rpc-example-server rpc/server.cpp rpc/server_main.cpp)
-target_link_libraries(rpc-example-server PRIVATE photon_static ${testing_libs})
+target_link_libraries(rpc-example-server PRIVATE photon_static)
 
 add_executable(sync-primitive sync-primitive/sync-primitive.cpp)
-target_link_libraries(sync-primitive PRIVATE photon_static ${testing_libs})
+target_link_libraries(sync-primitive PRIVATE photon_static)
 
 if (ENABLE_FSTACK_DPDK)
     add_executable(fstack-dpdk-demo fstack-dpdk/fstack-dpdk-demo.cpp)
-    target_link_libraries(fstack-dpdk-demo PRIVATE fstack_dpdk photon_static ${testing_libs})
+    target_link_libraries(fstack-dpdk-demo PRIVATE fstack_dpdk photon_static)
 endif ()
diff --git a/fs/extfs/test/CMakeLists.txt b/fs/extfs/test/CMakeLists.txt
index 9ea04cdb..2a146c22 100644
--- a/fs/extfs/test/CMakeLists.txt
+++ b/fs/extfs/test/CMakeLists.txt
@@ -1,8 +1,6 @@
-include_directories(${CMAKE_SOURCE_DIR}/include)
-link_directories(${CMAKE_LIBRARY_OUTPUT_DIRECTORY})
 add_definitions(-w)
 
 add_executable(test-extfs test.cpp)
-target_link_libraries(test-extfs PRIVATE photon_shared ${testing_libs})
+target_link_libraries(test-extfs PRIVATE photon_shared)
 
 add_test(NAME test-extfs COMMAND $<TARGET_FILE:test-extfs>)
diff --git a/fs/test/CMakeLists.txt b/fs/test/CMakeLists.txt
index 058d7fbd..6eb5b9c5 100644
--- a/fs/test/CMakeLists.txt
+++ b/fs/test/CMakeLists.txt
@@ -1,19 +1,17 @@
-include_directories(${CMAKE_SOURCE_DIR}/include)
-link_directories(${CMAKE_LIBRARY_OUTPUT_DIRECTORY})
 add_definitions(-w)
 
 add_executable(test-fs test.cpp)
-target_link_libraries(test-fs PRIVATE photon_shared ${testing_libs})
+target_link_libraries(test-fs PRIVATE photon_shared)
 add_test(NAME test-fs COMMAND $<TARGET_FILE:test-fs>)
 
 # add_executable(test-exportfs test_exportfs.cpp)
-# target_link_libraries(test-exportfs PRIVATE photon_static ${testing_libs})
+# target_link_libraries(test-exportfs PRIVATE photon_static)
 # add_test(NAME test-exportfs COMMAND $<TARGET_FILE:test-exportfs>)
 
 add_executable(test-filecopy test_filecopy.cpp)
-target_link_libraries(test-filecopy PRIVATE photon_shared ${testing_libs})
+target_link_libraries(test-filecopy PRIVATE photon_shared)
 add_test(NAME test-filecopy COMMAND $<TARGET_FILE:test-filecopy>)
 
 add_executable(test-throttled test_throttledfile.cpp)
-target_link_libraries(test-throttled PRIVATE photon_shared ${testing_libs})
+target_link_libraries(test-throttled PRIVATE photon_shared)
 add_test(NAME test-throttled COMMAND $<TARGET_FILE:test-throttled>)
\ No newline at end of file
diff --git a/io/test/CMakeLists.txt b/io/test/CMakeLists.txt
index 947e8f80..eea05c43 100644
--- a/io/test/CMakeLists.txt
+++ b/io/test/CMakeLists.txt
@@ -1,21 +1,19 @@
-include_directories(${CMAKE_SOURCE_DIR}/include)
-link_directories(${CMAKE_LIBRARY_OUTPUT_DIRECTORY})
 add_definitions(-w)
 
 add_executable(signalfdtest signalfdtest.cpp)
-target_link_libraries(signalfdtest PRIVATE photon_shared ${testing_libs})
+target_link_libraries(signalfdtest PRIVATE photon_shared)
 add_test(NAME signalfdtest COMMAND $<TARGET_FILE:signalfdtest>)
 
 add_executable(signalfdboom signalfdboom.cpp)
-target_link_libraries(signalfdboom PRIVATE photon_shared ${testing_libs})
+target_link_libraries(signalfdboom PRIVATE photon_shared)
 add_test(NAME signalfdboom COMMAND $<TARGET_FILE:signalfdboom>)
 
 if (NOT APPLE)
 add_executable(test-syncio test-syncio.cpp)
-target_link_libraries(test-syncio PRIVATE photon_shared ${testing_libs})
+target_link_libraries(test-syncio PRIVATE photon_shared)
 add_test(NAME test-syncio COMMAND $<TARGET_FILE:test-syncio>)
 
 add_executable(test-iouring test-iouring.cpp)
-target_link_libraries(test-iouring PRIVATE photon_shared ${testing_libs})
+target_link_libraries(test-iouring PRIVATE photon_shared)
 add_test(NAME test-iouring COMMAND $<TARGET_FILE:test-iouring>)
 endif ()
\ No newline at end of file
diff --git a/net/http/client.cpp b/net/http/client.cpp
index de5f6a88..b0735ad7 100644
--- a/net/http/client.cpp
+++ b/net/http/client.cpp
@@ -40,13 +40,13 @@ class PooledDialer {
     std::unique_ptr<Resolver> resolver;
 
     //etsocket seems not support multi thread very well, use tcp_socket now. need to find out why
-    explicit PooledDialer(bool ipv6) :
+    PooledDialer():
             tls_ctx(new_tls_context(nullptr, nullptr, nullptr)),
             resolver(new_default_resolver(kDNSCacheLife)) {
-        auto tcp_cli = ipv6 ? new_tcp_socket_client_ipv6() : new_tcp_socket_client();
-        auto tls_cli = ipv6 ? new_tcp_socket_client_ipv6() : new_tcp_socket_client();
+        auto tcp_cli = new_tcp_socket_client();
+        auto tls_cli = new_tls_client(tls_ctx, new_tcp_socket_client(), true);
         tcpsock.reset(new_tcp_socket_pool(tcp_cli, -1, true));
-        tlssock.reset(new_tcp_socket_pool(new_tls_client(tls_ctx, tls_cli, true), -1, true));
+        tlssock.reset(new_tcp_socket_pool(tls_cli, -1, true));
     }
 
     ~PooledDialer() { delete tls_ctx; }
@@ -64,6 +64,10 @@ ISocketStream* PooledDialer::dial(std::string_view host, uint16_t port, bool sec
     LOG_DEBUG("Dial to ` `", host, port);
     std::string strhost(host);
     auto ipaddr = resolver->resolve(strhost.c_str());
+    if (ipaddr.undefined()) {
+        LOG_ERROR_RETURN(0, nullptr, "DNS resolve failed, name = `", host)
+    }
+
     EndPoint ep(ipaddr, port);
     LOG_DEBUG("Connecting ` ssl: `", ep, secure);
     ISocketStream *sock = nullptr;
@@ -101,7 +105,8 @@ enum RoundtripStatus {
     ROUNDTRIP_FAILED,
     ROUNDTRIP_REDIRECT,
     ROUNDTRIP_NEED_RETRY,
-    ROUNDTRIP_FORCE_RETRY
+    ROUNDTRIP_FORCE_RETRY,
+    ROUNDTRIP_FAST_RETRY,
 };
 
 class ClientImpl : public Client {
@@ -109,8 +114,8 @@ class ClientImpl : public Client {
     PooledDialer m_dialer;
     CommonHeaders<> m_common_headers;
     ICookieJar *m_cookie_jar;
-    ClientImpl(ICookieJar *cookie_jar, bool ipv6) :
-            m_dialer(ipv6), m_cookie_jar(cookie_jar) {}
+    ClientImpl(ICookieJar *cookie_jar) :
+        m_cookie_jar(cookie_jar) {}
 
     using SocketStream_ptr = std::unique_ptr<ISocketStream>;
     int redirect(Operation* op) {
@@ -154,7 +159,12 @@ class ClientImpl : public Client {
         auto s = (m_proxy && !m_proxy_url.empty())
                      ? m_dialer.dial(m_proxy_url, tmo.timeout())
                      : m_dialer.dial(req, tmo.timeout());
-        if (!s) LOG_ERROR_RETURN(0, ROUNDTRIP_NEED_RETRY, "connection failed");
+        if (!s) {
+            if (errno == ECONNREFUSED) {
+                LOG_ERROR_RETURN(0, ROUNDTRIP_FAST_RETRY, "connection refused")
+            }
+            LOG_ERROR_RETURN(0, ROUNDTRIP_NEED_RETRY, "connection failed");
+        }
 
         SocketStream_ptr sock(s);
         LOG_DEBUG("Sending request ` `", req.verb(), req.target());
@@ -235,6 +245,9 @@ class ClientImpl : public Client {
                     sleep_interval = (sleep_interval + 500) * 2;
                     ++retry;
                     break;
+                case ROUNDTRIP_FAST_RETRY:
+                    ++retry;
+                    break;
                 case ROUNDTRIP_REDIRECT:
                     retry = 0;
                     ++followed;
@@ -245,7 +258,7 @@ class ClientImpl : public Client {
             if (tmo.timeout() == 0)
                 LOG_ERROR_RETURN(ETIMEDOUT, -1, "connection timedout");
             if (followed > op->follow || retry > op->retry)
-                LOG_ERROR_RETURN(ENOENT, -1,  "connection failed");
+                LOG_ERRNO_RETURN(0, -1,  "connection failed");
         }
         if (ret != ROUNDTRIP_SUCCESS) LOG_ERROR_RETURN(0, -1,"too many retry, roundtrip failed");
         return 0;
@@ -260,7 +273,7 @@ class ClientImpl : public Client {
     }
 };
 
-Client* new_http_client(ICookieJar *cookie_jar, bool ipv6) { return new ClientImpl(cookie_jar, ipv6); }
+Client* new_http_client(ICookieJar *cookie_jar) { return new ClientImpl(cookie_jar); }
 
 } // namespace http
 } // namespace net
diff --git a/net/http/client.h b/net/http/client.h
index 2d68b27c..7a767786 100644
--- a/net/http/client.h
+++ b/net/http/client.h
@@ -133,7 +133,7 @@ class Client : public Object {
 };
 
 //A Client without cookie_jar would ignore all response-header "Set-Cookies"
-Client* new_http_client(ICookieJar *cookie_jar = nullptr, bool ipv6 = false);
+Client* new_http_client(ICookieJar *cookie_jar = nullptr);
 
 ICookieJar* new_simple_cookie_jar();
 
diff --git a/net/http/test/CMakeLists.txt b/net/http/test/CMakeLists.txt
index 23ef62af..8ac5c878 100644
--- a/net/http/test/CMakeLists.txt
+++ b/net/http/test/CMakeLists.txt
@@ -1,28 +1,26 @@
-include_directories(${CMAKE_SOURCE_DIR}/include)
-link_directories(${CMAKE_LIBRARY_OUTPUT_DIRECTORY})
 add_definitions(-w)
 
 add_executable(client_perf client_perf.cpp)
-target_link_libraries(client_perf PRIVATE photon_shared ${testing_libs})
+target_link_libraries(client_perf PRIVATE photon_static)
 
 add_executable(server_perf server_perf.cpp)
-target_link_libraries(server_perf PRIVATE photon_shared ${testing_libs})
+target_link_libraries(server_perf PRIVATE photon_shared)
 
 add_executable(pure_libcurl pure_libcurl.cpp)
-target_link_libraries(pure_libcurl PRIVATE photon_shared ${testing_libs})
+target_link_libraries(pure_libcurl PRIVATE photon_static)
 
 add_executable(client_function_test client_function_test.cpp)
-target_link_libraries(client_function_test PRIVATE photon_shared ${testing_libs})
+target_link_libraries(client_function_test PRIVATE photon_shared)
 add_test(NAME client_function_test COMMAND $<TARGET_FILE:client_function_test>)
 
 add_executable(server_function_test server_function_test.cpp)
-target_link_libraries(server_function_test PRIVATE photon_shared ${testing_libs})
+target_link_libraries(server_function_test PRIVATE photon_shared)
 add_test(NAME server_function_test COMMAND $<TARGET_FILE:server_function_test>)
 
 add_executable(cookie_jar_test cookie_jar_test.cpp)
-target_link_libraries(cookie_jar_test PRIVATE photon_shared ${testing_libs})
+target_link_libraries(cookie_jar_test PRIVATE photon_shared)
 add_test(NAME cookie_jar_test COMMAND $<TARGET_FILE:cookie_jar_test>)
 
 add_executable(headers_test headers_test.cpp)
-target_link_libraries(headers_test PRIVATE photon_shared ${testing_libs})
+target_link_libraries(headers_test PRIVATE photon_shared)
 add_test(NAME headers_test COMMAND $<TARGET_FILE:headers_test>)
diff --git a/net/kernel_socket.cpp b/net/kernel_socket.cpp
index 323cddf1..59749dd8 100644
--- a/net/kernel_socket.cpp
+++ b/net/kernel_socket.cpp
@@ -37,6 +37,7 @@ limitations under the License.
 #include <photon/thread/thread11.h>
 #include <photon/thread/list.h>
 #include <photon/thread/timer.h>
+#include <photon/common/estring.h>
 #include <photon/common/utility.h>
 #include <photon/common/event-loop.h>
 #include <photon/net/basic_socket.h>
@@ -59,22 +60,6 @@ limitations under the License.
 #define AF_SMC 43
 #endif
 
-LogBuffer& operator<<(LogBuffer& log, const in_addr& iaddr) {
-    return log << photon::net::IPAddr(iaddr);
-}
-LogBuffer& operator<<(LogBuffer& log, const in6_addr& iaddr) {
-    return log << photon::net::IPAddr(iaddr);
-}
-LogBuffer& operator<<(LogBuffer& log, const sockaddr_in& addr) {
-    return log << photon::net::sockaddr_storage(addr).to_endpoint();
-}
-LogBuffer& operator<<(LogBuffer& log, const sockaddr_in6& addr) {
-    return log << photon::net::sockaddr_storage(addr).to_endpoint();
-}
-LogBuffer& operator<<(LogBuffer& log, const sockaddr& addr) {
-    return log << photon::net::sockaddr_storage(addr).to_endpoint();
-}
-
 namespace photon {
 namespace net {
 
@@ -240,7 +225,12 @@ class KernelSocketClient : public SocketClientBase {
     ISocketStream* do_connect(const sockaddr* remote, socklen_t len_remote,
                               const sockaddr* local = nullptr, socklen_t len_local = 0) {
         auto stream = create_stream();
-        std::unique_ptr<KernelSocketStream> ptr(stream);
+        auto deleter = [&](KernelSocketStream*) {
+            auto errno_backup = errno;
+            delete stream;
+            errno = errno_backup;
+        };
+        std::unique_ptr<KernelSocketStream, decltype(deleter)> ptr(stream, deleter);
         if (!ptr || ptr->fd < 0) {
             LOG_ERROR_RETURN(0, nullptr, "Failed to create socket fd");
         }
@@ -983,25 +973,6 @@ class ETKernelSocketServer : public KernelSocketServer, public NotifyContext {
 
 /* ET Socket - End */
 
-LogBuffer& operator<<(LogBuffer& log, const IPAddr addr) {
-    if (addr.is_ipv4())
-        return log.printf(addr.a, '.', addr.b, '.', addr.c, '.', addr.d);
-    else {
-        if (log.size < INET6_ADDRSTRLEN)
-            return log;
-        inet_ntop(AF_INET6, &addr.addr, log.ptr, INET6_ADDRSTRLEN);
-        log.consume(strlen(log.ptr));
-        return log;
-    }
-}
-
-LogBuffer& operator<<(LogBuffer& log, const EndPoint ep) {
-    if (ep.is_ipv4())
-        return log << ep.addr << ':' << ep.port;
-    else
-        return log << '[' << ep.addr << "]:" << ep.port;
-}
-
 extern "C" ISocketClient* new_tcp_socket_client() {
     return new KernelSocketClient(AF_INET, true);
 }
@@ -1063,5 +1034,61 @@ extern "C" ISocketServer* new_fstack_dpdk_socket_server() {
 #endif // ENABLE_FSTACK_DPDK
 #endif // __linux__
 
+////////////////////////////////////////////////////////////////////////////////
+
+/* Implementations in socket.h */
+
+EndPoint::EndPoint(const char* s) {
+    estring_view ep(s);
+    auto pos = ep.find_last_of(':');
+    if (pos == estring::npos)
+        return;
+    // Detect IPv6 or IPv4
+    estring ip_str = ep[pos - 1] == ']' ? ep.substr(1, pos - 2) : ep.substr(0, pos);
+    auto ip = IPAddr(ip_str.c_str());
+    if (ip.undefined())
+        return;
+    auto port_str = ep.substr(pos + 1);
+    if (!port_str.all_digits())
+        return;
+    addr = ip;
+    port = std::stoul(port_str);
 }
+
+LogBuffer& operator<<(LogBuffer& log, const IPAddr addr) {
+    if (addr.is_ipv4())
+        return log.printf(addr.a, '.', addr.b, '.', addr.c, '.', addr.d);
+    else {
+        if (log.size < INET6_ADDRSTRLEN)
+            return log;
+        inet_ntop(AF_INET6, &addr.addr, log.ptr, INET6_ADDRSTRLEN);
+        log.consume(strlen(log.ptr));
+        return log;
+    }
 }
+
+LogBuffer& operator<<(LogBuffer& log, const EndPoint ep) {
+    if (ep.is_ipv4())
+        return log << ep.addr << ':' << ep.port;
+    else
+        return log << '[' << ep.addr << "]:" << ep.port;
+}
+
+}
+}
+
+LogBuffer& operator<<(LogBuffer& log, const in_addr& iaddr) {
+    return log << photon::net::IPAddr(iaddr);
+}
+LogBuffer& operator<<(LogBuffer& log, const in6_addr& iaddr) {
+    return log << photon::net::IPAddr(iaddr);
+}
+LogBuffer& operator<<(LogBuffer& log, const sockaddr_in& addr) {
+    return log << photon::net::sockaddr_storage(addr).to_endpoint();
+}
+LogBuffer& operator<<(LogBuffer& log, const sockaddr_in6& addr) {
+    return log << photon::net::sockaddr_storage(addr).to_endpoint();
+}
+LogBuffer& operator<<(LogBuffer& log, const sockaddr& addr) {
+    return log << photon::net::sockaddr_storage(addr).to_endpoint();
+}
\ No newline at end of file
diff --git a/net/security-context/test/CMakeLists.txt b/net/security-context/test/CMakeLists.txt
index b775204a..4fe464f1 100644
--- a/net/security-context/test/CMakeLists.txt
+++ b/net/security-context/test/CMakeLists.txt
@@ -1,14 +1,12 @@
-include_directories(${CMAKE_SOURCE_DIR}/include)
-link_directories(${CMAKE_LIBRARY_OUTPUT_DIRECTORY})
 add_definitions(-w)
 
 add_executable(test-tls test.cpp)
-target_link_libraries(test-tls PRIVATE photon_shared ${testing_libs})
+target_link_libraries(test-tls PRIVATE photon_shared)
 add_test(NAME test-tls COMMAND $<TARGET_FILE:test-tls>)
 
 if (ENABLE_SASL AND (NOT (APPLE AND (${ARCH} STREQUAL arm64))))
     add_executable(test-sasl test-sasl.cpp)
-    target_link_libraries(test-sasl PRIVATE photon_shared ${testing_libs})
+    target_link_libraries(test-sasl PRIVATE photon_shared)
     add_test(NAME test-sasl COMMAND $<TARGET_FILE:test-sasl>)
 endif ()
 
diff --git a/net/socket.h b/net/socket.h
index 719e2436..75592a66 100644
--- a/net/socket.h
+++ b/net/socket.h
@@ -150,6 +150,7 @@ namespace net {
         uint16_t port = 0;
         EndPoint() = default;
         EndPoint(IPAddr ip, uint16_t port) : addr(ip), port(port) {}
+        explicit EndPoint(const char* s);
         bool is_ipv4() const {
             return addr.is_ipv4();
         };
diff --git a/net/test/CMakeLists.txt b/net/test/CMakeLists.txt
index 93862a4f..d8860fae 100644
--- a/net/test/CMakeLists.txt
+++ b/net/test/CMakeLists.txt
@@ -1,28 +1,26 @@
-include_directories(${CMAKE_SOURCE_DIR}/include)
-link_directories(${CMAKE_LIBRARY_OUTPUT_DIRECTORY})
 add_definitions(-w)
 
 add_executable(test-socket test.cpp)
-target_link_libraries(test-socket PRIVATE photon_shared ${testing_libs})
+target_link_libraries(test-socket PRIVATE photon_shared)
 add_test(NAME test-socket COMMAND $<TARGET_FILE:test-socket>)
 
 add_executable(test-dg-socket test-udp.cpp)
-target_link_libraries(test-dg-socket PRIVATE photon_shared ${testing_libs})
+target_link_libraries(test-dg-socket PRIVATE photon_shared)
 add_test(NAME test-dg-socket COMMAND $<TARGET_FILE:test-dg-socket>)
 
 add_executable(test-sockpool test_sockpool.cpp)
-target_link_libraries(test-sockpool PRIVATE photon_shared ${testing_libs})
+target_link_libraries(test-sockpool PRIVATE photon_shared)
 add_test(NAME test-sockpool COMMAND $<TARGET_FILE:test-sockpool>)
 
 add_executable(test-curl test_curl.cpp)
-target_link_libraries(test-curl PRIVATE photon_shared ${testing_libs})
+target_link_libraries(test-curl PRIVATE photon_static)
 add_test(NAME test-curl COMMAND $<TARGET_FILE:test-curl>)
 
 add_executable(test-server test-server.cpp)
-target_link_libraries(test-server PRIVATE photon_shared ${testing_libs})
+target_link_libraries(test-server PRIVATE photon_shared)
 
 add_executable(test-client test-client.cpp)
-target_link_libraries(test-client PRIVATE photon_shared ${testing_libs})
+target_link_libraries(test-client PRIVATE photon_shared)
 
 add_executable(test-ipv6 test-ipv6.cpp)
-target_link_libraries(test-ipv6 PRIVATE photon_shared ${testing_libs})
\ No newline at end of file
+target_link_libraries(test-ipv6 PRIVATE photon_shared)
\ No newline at end of file
diff --git a/net/test/test-ipv6.cpp b/net/test/test-ipv6.cpp
index 9400b5b8..4e3cc809 100644
--- a/net/test/test-ipv6.cpp
+++ b/net/test/test-ipv6.cpp
@@ -7,14 +7,28 @@
 #include <photon/net/utils.h>
 #include <photon/common/alog.h>
 
-TEST(ipv6, addr) {
-    EXPECT_NO_THROW(photon::net::IPAddr a("::1"));
-    EXPECT_NO_THROW(photon::net::IPAddr a("1.2.3.4"));
-    EXPECT_NO_THROW(photon::net::IPAddr a("fdbd:dc01:ff:312:9641:f71:10c4:2378"));
-    EXPECT_NO_THROW(photon::net::IPAddr a("fdbd:dc01:ff:312:9641:f71::2378"));
-    EXPECT_NO_THROW(photon::net::IPAddr a("fdbd:dc01:ff:312:9641::2378"));
+TEST(ipv6, endpoint) {
+    auto c = photon::net::EndPoint("127.0.0.1");
+    EXPECT_TRUE(c.undefined());
+    c = photon::net::EndPoint("127.0.0.1:8888");
+    EXPECT_FALSE(c.undefined());
+    c = photon::net::EndPoint("[::1]:8888");
+    EXPECT_FALSE(c.undefined());
+}
 
-    auto c = photon::net::IPAddr("zfdbd:dq01:8:165::158");
+TEST(ipv6, addr) {
+    auto c = photon::net::IPAddr("::1");
+    EXPECT_FALSE(c.undefined());
+    c = photon::net::IPAddr("::1");
+    EXPECT_FALSE(c.undefined());
+    c = photon::net::IPAddr("fdbd:dc01:ff:312:9641:f71:10c4:2378");
+    EXPECT_FALSE(c.undefined());
+    c = photon::net::IPAddr("fdbd:dc01:ff:312:9641:f71::2378");
+    EXPECT_FALSE(c.undefined());
+    c = photon::net::IPAddr("fdbd:dc01:ff:312:9641::2378");
+    EXPECT_FALSE(c.undefined());
+
+    c = photon::net::IPAddr("zfdbd:dq01:8:165::158");
     EXPECT_TRUE(c.undefined());
     c = photon::net::IPAddr("fdbd::ff:312:9641:f71::2378");
     EXPECT_TRUE(c.undefined());
diff --git a/net/utils.cpp b/net/utils.cpp
index 2137b47e..e1de9661 100644
--- a/net/utils.cpp
+++ b/net/utils.cpp
@@ -256,8 +256,8 @@ bool Base64Decode(std::string_view in, std::string &out) {
 
 class DefaultResolver : public Resolver {
 public:
-    DefaultResolver(uint64_t cache_ttl, uint64_t resolve_timeout, bool ipv6)
-        : dnscache_(cache_ttl), resolve_timeout_(resolve_timeout), ipv6_(ipv6) {}
+    DefaultResolver(uint64_t cache_ttl, uint64_t resolve_timeout)
+        : dnscache_(cache_ttl), resolve_timeout_(resolve_timeout) {}
     ~DefaultResolver() { dnscache_.clear(); }
 
     IPAddr resolve(const char *host) override {
@@ -279,9 +279,10 @@ class DefaultResolver : public Resolver {
                 LOG_WARN("Domain resolution for ` failed", host);
                 return new IPAddr;  // undefined addr
             }
-            for (auto& each : addrs) {
-                if ((each.is_ipv4() ^ !ipv6_) == 0)
-                    return new IPAddr(each);
+            // TODO: support ipv6
+            for (auto& ip : addrs) {
+                if (ip.is_ipv4())
+                    return new IPAddr(ip);
             }
             return new IPAddr;      // undefined addr
         };
@@ -298,11 +299,10 @@ class DefaultResolver : public Resolver {
 private:
     ObjectCache<std::string, IPAddr *> dnscache_;
     uint64_t resolve_timeout_;
-    bool ipv6_;
 };
 
-Resolver* new_default_resolver(uint64_t cache_ttl, uint64_t resolve_timeout, bool ipv6) {
-    return new DefaultResolver(cache_ttl, resolve_timeout, ipv6);
+Resolver* new_default_resolver(uint64_t cache_ttl, uint64_t resolve_timeout) {
+    return new DefaultResolver(cache_ttl, resolve_timeout);
 }
 
 }  // namespace net
diff --git a/net/utils.h b/net/utils.h
index b5457c3a..3f63f27c 100644
--- a/net/utils.h
+++ b/net/utils.h
@@ -165,10 +165,9 @@ class Resolver : public Object {
  *
  * @param cache_ttl cache's lifetime in microseconds.
  * @param resolve_timeout timeout in microseconds for domain resolution.
- * @param ipv6 specify v4 or v6 domain name
  * @return Resolver*
  */
-Resolver* new_default_resolver(uint64_t cache_ttl = 3600UL * 1000000, uint64_t resolve_timeout = -1, bool ipv6 = false);
+Resolver* new_default_resolver(uint64_t cache_ttl = 3600UL * 1000000, uint64_t resolve_timeout = -1);
 
 }  // namespace net
 }
diff --git a/rpc/rpc.cpp b/rpc/rpc.cpp
index 2ded983a..8082674f 100644
--- a/rpc/rpc.cpp
+++ b/rpc/rpc.cpp
@@ -427,15 +427,17 @@ namespace rpc {
 
     class StubPoolImpl : public StubPool {
     public:
-        explicit StubPoolImpl(uint64_t expiration, uint64_t connect_timeout, uint64_t rpc_timeout, bool ipv6) {
+        explicit StubPoolImpl(uint64_t expiration, uint64_t connect_timeout, uint64_t rpc_timeout) {
             tls_ctx = net::new_tls_context(nullptr, nullptr, nullptr);
-            tcpclient = ipv6 ? net::new_tcp_socket_client_ipv6() : net::new_tcp_socket_client();
+            tcpclient = net::new_tcp_socket_client();
+            tcpclientv6 = net::new_tcp_socket_client_ipv6();
             tcpclient->timeout(connect_timeout);
             m_pool = new ObjectCache<net::EndPoint, rpc::Stub*>(expiration);
             m_rpc_timeout = rpc_timeout;
         }
 
         ~StubPoolImpl() {
+            delete tcpclientv6;
             delete tcpclient;
             delete m_pool;
             delete tls_ctx;
@@ -468,7 +470,7 @@ namespace rpc {
     protected:
         net::ISocketStream* get_socket(const net::EndPoint& ep, bool tls) const {
             LOG_INFO("Connect to ", ep);
-            auto sock = tcpclient->connect(ep);
+            auto sock = ep.is_ipv4() ? tcpclient->connect(ep) : tcpclientv6->connect(ep);
             if (!sock) return nullptr;
             sock->timeout(m_rpc_timeout);
             if (tls) {
@@ -479,6 +481,7 @@ namespace rpc {
 
         ObjectCache<net::EndPoint, rpc::Stub*>* m_pool;
         net::ISocketClient *tcpclient;
+        net::ISocketClient *tcpclientv6;
         net::TLSContext* tls_ctx = nullptr;
         uint64_t m_rpc_timeout;
     };
@@ -489,7 +492,7 @@ namespace rpc {
     public:
         explicit UDSStubPoolImpl(const char* path, uint64_t expiration,
                                  uint64_t connect_timeout, uint64_t rpc_timeout)
-            : StubPoolImpl(expiration, connect_timeout, rpc_timeout, false),
+            : StubPoolImpl(expiration, connect_timeout, rpc_timeout),
               m_path(path), m_client(net::new_uds_client()) {
                   m_client->timeout(connect_timeout);
               }
@@ -515,8 +518,8 @@ namespace rpc {
         net::ISocketClient * m_client;
     };
 
-    StubPool* new_stub_pool(uint64_t expiration, uint64_t connect_timeout, uint64_t rpc_timeout, bool ipv6) {
-        return new StubPoolImpl(expiration, connect_timeout, rpc_timeout, ipv6);
+    StubPool* new_stub_pool(uint64_t expiration, uint64_t connect_timeout, uint64_t rpc_timeout) {
+        return new StubPoolImpl(expiration, connect_timeout, rpc_timeout);
     }
 
     StubPool* new_uds_stub_pool(const char* path, uint64_t expiration,
diff --git a/rpc/rpc.h b/rpc/rpc.h
index b5e6b083..0b2b19fa 100644
--- a/rpc/rpc.h
+++ b/rpc/rpc.h
@@ -234,8 +234,7 @@ namespace rpc
     };
 
     extern "C" Stub* new_rpc_stub(IStream* stream, bool ownership = false);
-    extern "C" StubPool* new_stub_pool(uint64_t expiration, uint64_t connect_timeout, uint64_t rpc_timeout,
-                                       bool ipv6 = false);
+    extern "C" StubPool* new_stub_pool(uint64_t expiration, uint64_t connect_timeout, uint64_t rpc_timeout);
     extern "C" StubPool* new_uds_stub_pool(const char* path, uint64_t expiration,
                                 uint64_t connect_timeout,
                                 uint64_t rpc_timeout);
diff --git a/rpc/test/CMakeLists.txt b/rpc/test/CMakeLists.txt
index 584852b1..efcf7f44 100644
--- a/rpc/test/CMakeLists.txt
+++ b/rpc/test/CMakeLists.txt
@@ -1,15 +1,13 @@
-include_directories(${CMAKE_SOURCE_DIR}/include)
-link_directories(${CMAKE_LIBRARY_OUTPUT_DIRECTORY})
 add_definitions(-w)
 
 add_executable(test-rpc test.cpp)
-target_link_libraries(test-rpc PRIVATE photon_shared ${testing_libs})
+target_link_libraries(test-rpc PRIVATE photon_shared)
 add_test(NAME test-rpc COMMAND $<TARGET_FILE:test-rpc>)
 
 add_executable(test-ooo test-ooo.cpp)
-target_link_libraries(test-ooo PRIVATE photon_shared ${testing_libs})
+target_link_libraries(test-ooo PRIVATE photon_shared)
 add_test(NAME test-ooo COMMAND $<TARGET_FILE:test-ooo>)
 
 add_executable(test-rpc-message test-rpc-message.cpp)
-target_link_libraries(test-rpc-message PRIVATE photon_shared ${testing_libs})
+target_link_libraries(test-rpc-message PRIVATE photon_shared)
 add_test(NAME test-rpc-message COMMAND $<TARGET_FILE:test-rpc-message>)
\ No newline at end of file
diff --git a/thread/test/CMakeLists.txt b/thread/test/CMakeLists.txt
index 9a2d0218..a1847f54 100644
--- a/thread/test/CMakeLists.txt
+++ b/thread/test/CMakeLists.txt
@@ -1,35 +1,33 @@
-include_directories(${CMAKE_SOURCE_DIR}/include)
-link_directories(${CMAKE_LIBRARY_OUTPUT_DIRECTORY})
 add_definitions(-w)
 
 add_executable(perf_usleepdefer_semaphore perf_usleepdefer_semaphore.cpp)
-target_link_libraries(perf_usleepdefer_semaphore PRIVATE photon_shared ${testing_libs})
+target_link_libraries(perf_usleepdefer_semaphore PRIVATE photon_shared)
 add_test(NAME perf_usleepdefer_semaphore COMMAND $<TARGET_FILE:perf_usleepdefer_semaphore>)
 
 add_executable(test-thread test.cpp x.cpp)
-target_link_libraries(test-thread PRIVATE photon_static ${testing_libs})
+target_link_libraries(test-thread PRIVATE photon_static)
 add_test(NAME test-thread COMMAND $<TARGET_FILE:test-thread>)
 
 add_executable(test-std-compat test-std-compat.cpp)
-target_link_libraries(test-std-compat PRIVATE photon_shared ${testing_libs})
+target_link_libraries(test-std-compat PRIVATE photon_shared)
 add_test(NAME test-std-compat COMMAND $<TARGET_FILE:test-std-compat>)
 
 add_executable(test-specific-key test-specific-key.cpp)
-target_link_libraries(test-specific-key PRIVATE photon_shared ${testing_libs})
+target_link_libraries(test-specific-key PRIVATE photon_shared)
 add_test(NAME test-specific-key COMMAND $<TARGET_FILE:test-specific-key>)
 
 add_executable(test-thread-local test-thread-local.cpp)
-target_link_libraries(test-thread-local PRIVATE photon_shared ${testing_libs})
+target_link_libraries(test-thread-local PRIVATE photon_shared)
 add_test(NAME test-thread-local COMMAND $<TARGET_FILE:test-thread-local>)
 
 add_executable(test-tls-order-native test-tls-order-native.cpp)
-target_link_libraries(test-tls-order-native PRIVATE photon_shared ${testing_libs})
+target_link_libraries(test-tls-order-native PRIVATE photon_shared)
 add_test(NAME test-tls-order-native COMMAND $<TARGET_FILE:test-tls-order-native>)
 
 add_executable(test-tls-order-photon test-tls-order-photon.cpp)
-target_link_libraries(test-tls-order-photon PRIVATE photon_shared ${testing_libs})
+target_link_libraries(test-tls-order-photon PRIVATE photon_shared)
 add_test(NAME test-tls-order-photon COMMAND $<TARGET_FILE:test-tls-order-photon>)
 
 add_executable(test-lib-data test-lib-data.cpp)
-target_link_libraries(test-lib-data PRIVATE photon_shared ${testing_libs})
+target_link_libraries(test-lib-data PRIVATE photon_shared)
 add_test(NAME test-lib-data COMMAND $<TARGET_FILE:test-lib-data>)
\ No newline at end of file
diff --git a/tools/libphoton.mri b/tools/libphoton.mri
new file mode 100644
index 00000000..51aef26c
--- /dev/null
+++ b/tools/libphoton.mri
@@ -0,0 +1,4 @@
+create libphoton.a
+addlib libphoton.a
+save
+end
\ No newline at end of file

From 3b0d9a8fb57dd673b2ddee09078107d32a5611f8 Mon Sep 17 00:00:00 2001
From: Lanzheng Liu <lanzheng.liulz@alibaba-inc.com>
Date: Thu, 16 Nov 2023 17:12:35 +0800
Subject: [PATCH 16/76] marking inode 1 inuse for extfs/mkfs (#247)

Signed-off-by: liulanzheng <lanzheng.liulz@alibaba-inc.com>
---
 fs/extfs/mkfs.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/fs/extfs/mkfs.cpp b/fs/extfs/mkfs.cpp
index addd93f9..42cc9bd1 100644
--- a/fs/extfs/mkfs.cpp
+++ b/fs/extfs/mkfs.cpp
@@ -116,6 +116,7 @@ int do_mkfs(io_manager manager, size_t size, char *uuid) {
         return ret;
     }
     // reserve inodes
+    ext2fs_inode_alloc_stats2(fs, EXT2_BAD_INO, +1, 0);
     for (ext2_ino_t i = EXT2_ROOT_INO + 1; i < EXT2_FIRST_INODE(fs->super); i++)
         ext2fs_inode_alloc_stats2(fs, i, +1, 0);
     ext2fs_mark_ib_dirty(fs);

From 046443c6382a5ecd01bf7b05068e45a35adee66a Mon Sep 17 00:00:00 2001
From: Bob Chen <beef9999@qq.com>
Date: Thu, 16 Nov 2023 22:30:11 +0800
Subject: [PATCH 17/76] Update CI

---
 .github/workflows/ci.linux.arm.yml |  8 +++-----
 .github/workflows/ci.linux.x86.yml | 12 +++++-------
 .github/workflows/ci.macos.arm.yml |  2 +-
 .github/workflows/ci.macos.yml     |  2 +-
 4 files changed, 10 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/ci.linux.arm.yml b/.github/workflows/ci.linux.arm.yml
index c7766b4b..cc6b00ab 100644
--- a/.github/workflows/ci.linux.arm.yml
+++ b/.github/workflows/ci.linux.arm.yml
@@ -4,11 +4,11 @@ on:
   push:
     branches: [ "main", "release/*" ]
   pull_request:
-    branches: [ "main", "release/*" ]
+    branches: [ "release/*" ]
 
 jobs:
   centos8-gcc921-epoll-release:
-    runs-on: [self-hosted, Linux, ARM64]
+    runs-on: ubuntu-latest
 
     container:
       image: dokken/centos-stream-8:sha-40294ce
@@ -43,12 +43,11 @@ jobs:
 
       - name: Test
         run: |
-          ulimit -l unlimited
           cd build
           ctest --timeout 3600 -V
 
   centos8-gcc921-epoll-debug:
-    runs-on: [self-hosted, Linux, ARM64]
+    runs-on: ubuntu-latest
 
     container:
       image: dokken/centos-stream-8:sha-40294ce
@@ -83,6 +82,5 @@ jobs:
 
       - name: Test
         run: |
-          ulimit -l unlimited
           cd build
           ctest --timeout 3600 -V
\ No newline at end of file
diff --git a/.github/workflows/ci.linux.x86.yml b/.github/workflows/ci.linux.x86.yml
index 74fb7995..974cf2ff 100644
--- a/.github/workflows/ci.linux.x86.yml
+++ b/.github/workflows/ci.linux.x86.yml
@@ -4,11 +4,11 @@ on:
   push:
     branches: [ "main", "release/*" ]
   pull_request:
-    branches: [ "main", "release/*" ]
+    branches: [ "release/*" ]
 
 jobs:
   centos8-gcc921-epoll-release:
-    runs-on: [self-hosted, Linux, X64]
+    runs-on: ubuntu-latest
 
     container:
       image: dokken/centos-stream-8:sha-40294ce
@@ -43,12 +43,11 @@ jobs:
 
       - name: Test
         run: |
-          ulimit -l unlimited
           cd build
           ctest --timeout 3600 -V
 
   centos8-gcc921-epoll-debug:
-    runs-on: [self-hosted, Linux, X64]
+    runs-on: ubuntu-latest
 
     container:
       image: dokken/centos-stream-8:sha-40294ce
@@ -83,12 +82,12 @@ jobs:
 
       - name: Test
         run: |
-          ulimit -l unlimited
           cd build
           ctest --timeout 3600 -V
 
   centos8-gcc921-iouring-release:
-    runs-on: [self-hosted, Linux, X64]
+#    runs-on: [self-hosted, Linux, X64]
+    runs-on: ubuntu-latest
 
     container:
       image: dokken/centos-stream-8:sha-40294ce
@@ -121,6 +120,5 @@ jobs:
 
       - name: Test
         run: |
-          ulimit -l unlimited
           cd build
           ctest --timeout 3600 -V
diff --git a/.github/workflows/ci.macos.arm.yml b/.github/workflows/ci.macos.arm.yml
index c8e17718..271ad498 100644
--- a/.github/workflows/ci.macos.arm.yml
+++ b/.github/workflows/ci.macos.arm.yml
@@ -4,7 +4,7 @@ on:
   push:
     branches: [ "main", "release/*" ]
   pull_request:
-    branches: [ "main", "release/*" ]
+    branches: [ "release/*" ]
 
 jobs:
   macOS-clang-debug:
diff --git a/.github/workflows/ci.macos.yml b/.github/workflows/ci.macos.yml
index 5e75d552..33ce1e58 100644
--- a/.github/workflows/ci.macos.yml
+++ b/.github/workflows/ci.macos.yml
@@ -4,7 +4,7 @@ on:
   push:
     branches: [ "main", "release/*" ]
   pull_request:
-    branches: [ "main", "release/*" ]
+    branches: [ "release/*" ]
 
 jobs:
   macOS-12-Monterey-debug:

From 9245e93f780b464e844c98f18194a977cdf64df2 Mon Sep 17 00:00:00 2001
From: Huiba Li <lihuiba@gmail.com>
Date: Fri, 17 Nov 2023 16:04:12 +0800
Subject: [PATCH 18/76] Fix http message (#250)

fix operator priority bug in http/status.cpp;
enhance estring;
---
 common/estring.cpp                     | 47 +++++++++++++++-----------
 common/estring.h                       | 43 +++++++++++++++++++----
 common/test/test.cpp                   |  9 +++++
 net/http/message.cpp                   |  4 +--
 net/http/test/client_function_test.cpp | 10 +++---
 net/utils.cpp                          | 40 ++++++++++------------
 6 files changed, 98 insertions(+), 55 deletions(-)

diff --git a/common/estring.cpp b/common/estring.cpp
index 6822f891..54166380 100644
--- a/common/estring.cpp
+++ b/common/estring.cpp
@@ -55,30 +55,39 @@ size_t estring_view::find_last_not_of(const charset& set) const
 
 bool estring_view::to_uint64_check(uint64_t* v) const
 {
-    v ? (*v = 0) : 0;
-    for (unsigned char c : *this) {
-        if (c > '9' || c < '0')
-            return false;
-        v ? (*v = *v * 10 + (c - '0')) : 0;
+    if (this->empty()) return false;
+    uint64_t val = (*this)[0] - '0';
+    if (val > 9) return false;
+    for (unsigned char c : this->substr(1)) {
+        c -= '0';
+        if (c > 9) break;
+        val = val * 10 + c;
     }
+    if (v) *v = val;
     return true;
 }
 
-uint64_t estring_view::hex_to_uint64() const
-{
-    uint64_t ret = 0;
-    for (unsigned char c : *this) {
-        if (c >= '0' && c <= '9') {
-            ret = ret * 16 + (c - '0');
-        } else if (c >= 'A' && c <= 'F') {
-            ret = ret * 16 + (c - 'A' + 10);
-        } else if (c >= 'a' && c <= 'f') {
-            ret = ret * 16 + (c - 'a' + 10);
-        } else {
-            return ret;
-        }
+inline char hex_char_to_digit(char c) {
+    unsigned char cc = c - '0';
+    if (cc < 10) return cc;
+    const unsigned char mask = 'a' - 'A';
+    static_assert(mask == 32, "..."); // single digit
+    c |= mask; // unified to 'a'..'f'
+    cc = c - 'a';
+    return (cc < 6) ? (cc + 10) : -1;
+}
+
+bool estring_view::hex_to_uint64_check(uint64_t* v) const {
+    if (this->empty()) return false;
+    uint64_t val = hex_char_to_digit((*this)[0]);
+    if (val == -1ul) return false;
+    for (unsigned char c : this->substr(1)) {
+        auto d = hex_char_to_digit(c);
+        if (d == -1) break;
+        val = val * 16 + d;
     }
-    return ret;
+    if (v) *v = val;
+    return true;
 }
 
 estring& estring::append(uint64_t x)
diff --git a/common/estring.h b/common/estring.h
index 9e57214f..cdd90af7 100644
--- a/common/estring.h
+++ b/common/estring.h
@@ -18,6 +18,7 @@ limitations under the License.
 #include <cstdio>
 #include <cassert>
 #include <cstring>
+#include <math.h>
 #include <limits>
 #include <string>
 #include <bitset>
@@ -289,11 +290,41 @@ class estring_view : public std::string_view
     uint64_t to_uint64(uint64_t default_val = 0) const
     {
         uint64_t val;
-        to_uint64_check(&val);
-        return val;
+        return to_uint64_check(&val) ? val : default_val;
+    }
+    bool     to_int64_check(int64_t* v = nullptr) const
+    {
+        if (this->empty()) return false;
+        if (this->front() != '-') return to_uint64_check((uint64_t*)v);
+        bool ret = this->substr(1).to_uint64_check((uint64_t*)v);
+        if (ret) *v = -*v;
+        return ret;
+    }
+    int64_t to_int64(int64_t default_val = 0) const
+    {
+        int64_t val;
+        return to_int64_check(&val) ? val : default_val;
+    }
+    bool to_double_check(double* v = nullptr)
+    {
+        char buf[32];
+        auto len = std::max(this->size(), sizeof(buf) - 1 );
+        memcpy(buf, data(), len);
+        buf[len] = '0';
+        return sscanf(buf, "%lf", v) == 1;
+    }
+    double to_double(double default_val = NAN)
+    {
+        double val;
+        return to_double_check(&val) ? val : default_val;
+    }
+    // not including 0x/0X prefix
+    bool hex_to_uint64_check(uint64_t* v = nullptr) const;
+    uint64_t hex_to_uint64(uint64_t default_val = 0) const
+    {
+        uint64_t val;
+        return hex_to_uint64_check(&val) ? val : default_val;
     }
-    // do not support 0x/0X prefix
-    uint64_t hex_to_uint64() const;
 };
 
 inline bool operator == (const std::string_view& sv, const std::string& s)
@@ -312,8 +343,8 @@ class rstring_view  // relative string_view, that stores values relative
 protected:
     static_assert(std::is_integral<OffsetType>::value, "...");
     static_assert(std::is_integral<LengthType>::value, "...");
-    OffsetType _offset;
-    LengthType _length;
+    OffsetType _offset = 0;
+    LengthType _length = 0;
 
     estring_view to_abs(const char* s) const
     {
diff --git a/common/test/test.cpp b/common/test/test.cpp
index 7c87049b..e9b2b27e 100644
--- a/common/test/test.cpp
+++ b/common/test/test.cpp
@@ -867,6 +867,15 @@ TEST(estring, test)
     estring as = "   \tasdf  \t\r\n";
     auto trimmed = as.trim();
     EXPECT_EQ(trimmed, "asdf");
+
+    EXPECT_EQ(estring_view("234423").to_uint64(), 234423);
+    EXPECT_EQ(estring_view("-234423").to_int64(), -234423);
+    EXPECT_EQ(estring_view("asfdsf").to_uint64(32), 32);
+    EXPECT_EQ(estring_view("-3.14").to_double(), -3.14);
+    EXPECT_EQ(estring_view("1e10").to_double(), 1e10);
+
+    EXPECT_EQ(estring_view("1").hex_to_uint64(), 0x1);
+    EXPECT_EQ(estring_view("1a2b3d4e5f").hex_to_uint64(), 0x1a2b3d4e5f);
 }
 
 TEST(generator, example)
diff --git a/net/http/message.cpp b/net/http/message.cpp
index ab8e3179..481c059f 100644
--- a/net/http/message.cpp
+++ b/net/http/message.cpp
@@ -331,7 +331,7 @@ int Request::redirect(Verb v, estring_view location, bool enable_proxy) {
     }
     StoredURL u(location);
     auto new_request_line_size = verbstr[v].size() + sizeof(" HTTP/1.1\r\n") +
-        enable_proxy ? full_url_size(u) : u.target().size();
+        (enable_proxy ? full_url_size(u) : u.target().size());
 
     auto delta = new_request_line_size - m_buf_size;
     LOG_DEBUG(VALUE(delta));
@@ -368,7 +368,7 @@ int Response::parse_status_line(Parser &p) {
     p.skip_chars(' ');
     auto code = p.extract_integer();
     if (code <= 0 || code >= 1000)
-        LOG_ERROR_RETURN(0, -1, "invalid status code");
+        LOG_ERROR_RETURN(0, -1, "invalid status code ", code);
     m_status_code = (uint16_t)code;
     p.skip_chars(' ');
     m_status_message = p.extract_until_char('\r');
diff --git a/net/http/test/client_function_test.cpp b/net/http/test/client_function_test.cpp
index 7af2d2df..062f34c0 100644
--- a/net/http/test/client_function_test.cpp
+++ b/net/http/test/client_function_test.cpp
@@ -413,9 +413,9 @@ TEST(http_client, debug) {
     server->setsockopt(SOL_SOCKET, SO_REUSEPORT, 1);
     server->set_handler({nullptr, &chunked_handler_debug});
     auto ret = server->bind(ep.port, ep.addr);
-    if (ret < 0) LOG_ERROR(VALUE(errno));
+    if (ret < 0) LOG_ERROR(ERRNO());
     ret |= server->listen(100);
-    if (ret < 0) LOG_ERROR(VALUE(errno));
+    if (ret < 0) LOG_ERROR(ERRNO());
     EXPECT_EQ(0, ret);
     LOG_INFO("Ready to accept");
     server->start_loop();
@@ -438,13 +438,13 @@ TEST(http_client, debug) {
     memset((void*)buf.data(), '0', std_data_size);
     ret = op_test->resp.read((void*)buf.data(), std_data_size);
     EXPECT_EQ(std_data_size, ret);
-    EXPECT_EQ(true, buf == std_data);
+    EXPECT_TRUE(buf == std_data);
     for (int i = 0; i < buf.size(); i++) {
         if (buf[i] != std_data[i]) {
-            std::cout << i << std::endl;
+            LOG_ERROR("first occurrence of difference at: ", i);
+            break;
         }
     }
-    std::cout << "new" << std::endl;
 }
 int sleep_handler(void*, ISocketStream* sock) {
     photon::thread_sleep(3);
diff --git a/net/utils.cpp b/net/utils.cpp
index e1de9661..82985ffb 100644
--- a/net/utils.cpp
+++ b/net/utils.cpp
@@ -101,22 +101,25 @@ int _gethostbyname(const char* name, Delegate<int, IPAddr> append_op) {
     return idx;
 }
 
-inline __attribute__((always_inline)) void base64_translate_3to4(const char *in, char *out)  {
-    struct xlator {
-        unsigned char _;
-        unsigned char a : 6;
-        unsigned char b : 6;
-        unsigned char c : 6;
-        unsigned char d : 6;
-    } __attribute__((packed));
-    static_assert(sizeof(xlator) == 4, "...");
+struct xlator {
+    unsigned char _;
+    unsigned char a : 6;
+    unsigned char b : 6;
+    unsigned char c : 6;
+    unsigned char d : 6;
+} __attribute__((packed));
+static_assert(sizeof(xlator) == 4, "...");
+
+inline __attribute__((always_inline))
+void base64_translate_3to4(const char *in, char *out)  {
     static const unsigned char tbl[] =
         "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
     auto v = htonl(*(uint32_t *)in);
-    auto x = *(xlator *)(&v);
-    *(uint32_t *)out = ((tbl[x.a] << 24) + (tbl[x.b] << 16) +
-                        (tbl[x.c] << 8) + (tbl[x.d] << 0));
+    auto x = (xlator*) &v;
+    *(uint32_t *)out = ((tbl[x->a] << 24) + (tbl[x->b] << 16) +
+                        (tbl[x->c] << 8) + (tbl[x->d] << 0));
 }
+
 void Base64Encode(std::string_view in, std::string &out) {
     auto main = in.size() / 3;
     auto remain = in.size() % 3;
@@ -137,7 +140,6 @@ void Base64Encode(std::string_view in, std::string &out) {
         base64_translate_3to4(_in + 9, _out + 12);
     }
 
-
     for (; _in < end; _in += 3, _out += 4) {
         base64_translate_3to4(_in, _out);
     }
@@ -191,16 +193,8 @@ static unsigned char get_index_of(char val, bool &ok) {
 }
  #undef EI
 
+inline
 bool base64_translate_4to3(const char *in, char *out)  {
-    struct xlator {
-        unsigned char _;
-        unsigned char a : 6;
-        unsigned char b : 6;
-        unsigned char c : 6;
-        unsigned char d : 6;
-    } __attribute__((packed));
-    static_assert(sizeof(xlator) == 4, "...");
-
     xlator v;
     bool f1, f2, f3, f4;
     v.a = get_index_of(*(in+3), f1);
@@ -208,10 +202,10 @@ bool base64_translate_4to3(const char *in, char *out)  {
     v.c = get_index_of(*(in+1), f3);
     v.d = get_index_of(*(in),   f4);
 
-
     *(uint32_t *)out = ntohl(*(uint32_t *)&v);
     return (f1 && f2 && f3 && f4);
 }
+
 bool Base64Decode(std::string_view in, std::string &out) {
 #define GSIZE 4 //Size of each group
     auto in_size = in.size();

From 0a012f6bb33fab96b825f67eadd85b042fb6c302 Mon Sep 17 00:00:00 2001
From: Bob Chen <beef9999@qq.com>
Date: Sun, 19 Nov 2023 22:14:15 +0800
Subject: [PATCH 19/76] fix io_uring cancel_wait

---
 .github/workflows/ci.linux.arm.yml      |   2 +-
 .github/workflows/ci.linux.x86.yml      |   2 +-
 .github/workflows/ci.macos.arm.yml      |   2 +-
 .github/workflows/ci.macos.yml          |   2 +-
 CMake/build-from-src.cmake              |   2 +-
 io/iouring-wrapper.cpp                  | 102 ++++++++++--------------
 thread/test/CMakeLists.txt              |   6 +-
 thread/test/test-multi-vcpu-locking.cpp |  93 +++++++++++++++++++++
 8 files changed, 146 insertions(+), 65 deletions(-)
 create mode 100644 thread/test/test-multi-vcpu-locking.cpp

diff --git a/.github/workflows/ci.linux.arm.yml b/.github/workflows/ci.linux.arm.yml
index 68b3656c..4b903cf8 100644
--- a/.github/workflows/ci.linux.arm.yml
+++ b/.github/workflows/ci.linux.arm.yml
@@ -4,7 +4,7 @@ on:
   push:
     branches: [ "main", "release/*" ]
   pull_request:
-    branches: [ "release/*" ]
+    branches: [ "main", "release/*" ]
 
 jobs:
   centos8-gcc921-epoll-release:
diff --git a/.github/workflows/ci.linux.x86.yml b/.github/workflows/ci.linux.x86.yml
index 974cf2ff..cf083d05 100644
--- a/.github/workflows/ci.linux.x86.yml
+++ b/.github/workflows/ci.linux.x86.yml
@@ -4,7 +4,7 @@ on:
   push:
     branches: [ "main", "release/*" ]
   pull_request:
-    branches: [ "release/*" ]
+    branches: [ "main", "release/*" ]
 
 jobs:
   centos8-gcc921-epoll-release:
diff --git a/.github/workflows/ci.macos.arm.yml b/.github/workflows/ci.macos.arm.yml
index 271ad498..c8e17718 100644
--- a/.github/workflows/ci.macos.arm.yml
+++ b/.github/workflows/ci.macos.arm.yml
@@ -4,7 +4,7 @@ on:
   push:
     branches: [ "main", "release/*" ]
   pull_request:
-    branches: [ "release/*" ]
+    branches: [ "main", "release/*" ]
 
 jobs:
   macOS-clang-debug:
diff --git a/.github/workflows/ci.macos.yml b/.github/workflows/ci.macos.yml
index 33ce1e58..5e75d552 100644
--- a/.github/workflows/ci.macos.yml
+++ b/.github/workflows/ci.macos.yml
@@ -4,7 +4,7 @@ on:
   push:
     branches: [ "main", "release/*" ]
   pull_request:
-    branches: [ "release/*" ]
+    branches: [ "main", "release/*" ]
 
 jobs:
   macOS-12-Monterey-debug:
diff --git a/CMake/build-from-src.cmake b/CMake/build-from-src.cmake
index ee9fcca7..16f455ea 100644
--- a/CMake/build-from-src.cmake
+++ b/CMake/build-from-src.cmake
@@ -83,7 +83,7 @@ function(build_from_src [dep])
                 URL_MD5 bad68bb6bd9908da75e2c8dedc536b29
                 BUILD_IN_SOURCE ON
                 CONFIGURE_COMMAND ./config -fPIC no-unit-test no-shared --openssldir=${BINARY_DIR} --prefix=${BINARY_DIR}
-                BUILD_COMMAND make depend -j ${NumCPU} && make -j ${NumCPU}
+                BUILD_COMMAND make depend -j 4 && make -j 4 #  Not using ${NumCPU}. Too may parallel might fail
                 INSTALL_COMMAND make install
         )
         ExternalProject_Get_Property(openssl SOURCE_DIR)
diff --git a/io/iouring-wrapper.cpp b/io/iouring-wrapper.cpp
index 2dfa4274..d2e179ca 100644
--- a/io/iouring-wrapper.cpp
+++ b/io/iouring-wrapper.cpp
@@ -40,8 +40,7 @@ limitations under the License.
 
 namespace photon {
 
-constexpr static EventsMap<EVUnderlay<POLLIN | POLLRDHUP, POLLOUT, POLLERR>>
-    evmap;
+constexpr static EventsMap<EVUnderlay<POLLIN | POLLRDHUP, POLLOUT, POLLERR>> evmap;
 
 class iouringEngine : public MasterEventEngine, public CascadingEventEngine {
 public:
@@ -49,19 +48,13 @@ class iouringEngine : public MasterEventEngine, public CascadingEventEngine {
 
     ~iouringEngine() {
         LOG_INFO("Finish event engine: iouring ", VALUE(m_master));
-        if (m_cancel_poller != nullptr) {
-            m_cancel_poller_running = false;
-            thread_interrupt(m_cancel_poller);
-            thread_join((join_handle*) m_cancel_poller);
-        }
-        if (m_cancel_fd >= 0) {
-            close(m_cancel_fd);
-        }
-        if (m_cascading_event_fd >= 0) {
+        if (m_eventfd >= 0 && !m_master) {
             if (io_uring_unregister_eventfd(m_ring) != 0) {
                 LOG_ERROR("iouring: failed to unregister cascading event fd");
             }
-            close(m_cascading_event_fd);
+        }
+        if (m_eventfd >= 0) {
+            close(m_eventfd);
         }
         if (m_ring != nullptr) {
             io_uring_queue_exit(m_ring);
@@ -128,22 +121,25 @@ class iouringEngine : public MasterEventEngine, public CascadingEventEngine {
             }
         }
 
+        m_eventfd = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK);
+        if (m_eventfd < 0) {
+            LOG_ERRNO_RETURN(0, -1, "iouring: failed to create eventfd");
+        }
+
         if (m_master) {
-            // Setup a cancel poller to watch on master engine
-            m_cancel_fd = eventfd(0, EFD_CLOEXEC);
-            if (m_cancel_fd < 0) {
-                LOG_ERRNO_RETURN(0, -1, "iouring: failed to create eventfd");
+            // Setup a multishot poll on master engine to watch the cancel_wait
+            uint32_t poll_mask = evmap.translate_bitwisely(EVENT_READ);
+            auto sqe = _get_sqe();
+            if (!sqe) return -1;
+            io_uring_prep_poll_multishot(sqe, m_eventfd, poll_mask);
+            io_uring_sqe_set_data(sqe, this);
+            ret = io_uring_submit(m_ring);
+            if (ret <= 0) {
+                LOG_ERROR_RETURN(0, -1, "iouring: fail to submit multishot poll, ", ERRNO(-ret));
             }
-            m_cancel_poller = thread_create11(64 * 1024, &iouringEngine::run_cancel_poller, this);
-            thread_enable_join(m_cancel_poller);
-
         } else {
-            // Register an event fd for cascading engine
-            m_cascading_event_fd = eventfd(0, EFD_CLOEXEC);
-            if (m_cascading_event_fd < 0) {
-                LOG_ERRNO_RETURN(0, -1, "iouring: failed to create cascading event fd");
-            }
-            if (io_uring_register_eventfd(m_ring, m_cascading_event_fd) != 0) {
+            // Register cascading engine to eventfd
+            if (io_uring_register_eventfd(m_ring, m_eventfd) != 0) {
                 LOG_ERRNO_RETURN(0, -1, "iouring: failed to register cascading event fd");
             }
         }
@@ -155,7 +151,7 @@ class iouringEngine : public MasterEventEngine, public CascadingEventEngine {
      *     later in the `wait_and_fire_events`.
      * @param timeout Timeout in usec. It could be 0 (immediate cancel), and -1 (most efficient way, no linked SQE).
      *     Note that the cancelling has no guarantee to succeed, it's just an attempt.
-     * @param ring_flags The lowest 8 bits is for sqe.flags, and the rest is reserved.
+     * @param ring_flags The lowest 8 bits is for sqe.flags. The rest is reserved.
      * @retval Non negative integers for success, -1 for failure. If failed with timeout, errno will
      *     be set to ETIMEDOUT. If failed because of external interruption, errno will also be set accordingly.
      */
@@ -165,15 +161,15 @@ class iouringEngine : public MasterEventEngine, public CascadingEventEngine {
         if (sqe == nullptr)
             return -1;
         prep(sqe, args...);
-        sqe->flags |= (uint8_t) (ring_flags & 0xff);
-        return _async_io(sqe, timeout);
+        return _async_io(sqe, timeout, ring_flags);
     }
 
-    int32_t _async_io(io_uring_sqe* sqe, uint64_t timeout) {
-        ioCtx io_ctx{photon::CURRENT, -1, false, false};
+    int32_t _async_io(io_uring_sqe* sqe, uint64_t timeout, uint32_t ring_flags) {
+        sqe->flags |= (uint8_t) (ring_flags & 0xff);
+        ioCtx io_ctx(false, false);
         io_uring_sqe_set_data(sqe, &io_ctx);
 
-        ioCtx timer_ctx{photon::CURRENT, -1, true, false};
+        ioCtx timer_ctx(true, false);
         __kernel_timespec ts{};
         if (timeout < std::numeric_limits<int64_t>::max()) {
             sqe->flags |= IOSQE_IO_LINK;
@@ -202,7 +198,7 @@ class iouringEngine : public MasterEventEngine, public CascadingEventEngine {
             sqe = _get_sqe();
             if (sqe == nullptr)
                 return -1;
-            ioCtx cancel_ctx{CURRENT, -1, true, false};
+            ioCtx cancel_ctx(true, false);
             io_uring_prep_cancel(sqe, &io_ctx, 0);
             io_uring_sqe_set_data(sqe, &cancel_ctx);
             photon::thread_sleep(-1);
@@ -231,7 +227,7 @@ class iouringEngine : public MasterEventEngine, public CascadingEventEngine {
 
         bool one_shot = e.interests & ONE_SHOT;
         fdInterest fd_interest{e.fd, (uint32_t)evmap.translate_bitwisely(e.interests)};
-        ioCtx io_ctx{CURRENT, -1, false, true};
+        ioCtx io_ctx(false, true);
         eventCtx event_ctx{e, one_shot, io_ctx};
         auto pair = m_event_contexts.insert({fd_interest, event_ctx});
         if (!pair.second) {
@@ -265,12 +261,12 @@ class iouringEngine : public MasterEventEngine, public CascadingEventEngine {
 
     ssize_t wait_for_events(void** data, size_t count, uint64_t timeout = -1) override {
         // Use master engine to wait for self event fd
-        int ret = get_vcpu()->master_event_engine->wait_for_fd_readable(m_cascading_event_fd, timeout);
+        int ret = get_vcpu()->master_event_engine->wait_for_fd_readable(m_eventfd, timeout);
         if (ret < 0) {
             return errno == ETIMEDOUT ? 0 : -1;
         }
         uint64_t value = 0;
-        if (eventfd_read(m_cascading_event_fd, &value)) {
+        if (eventfd_read(m_eventfd, &value)) {
             LOG_ERROR("iouring: error reading cascading event fd, `", ERRNO());
         }
 
@@ -334,6 +330,13 @@ class iouringEngine : public MasterEventEngine, public CascadingEventEngine {
                 continue;
             }
 
+            if (ctx == (ioCtx*) this) {
+                // Triggered by cancel_wait
+                eventfd_t val;
+                eventfd_read(m_eventfd, &val);
+                continue;
+            }
+
             if (cqe->flags & IORING_CQE_F_NOTIF) {
                 // The cqe for notify, corresponding to IORING_CQE_F_MORE
                 if (unlikely(cqe->res != 0))
@@ -367,7 +370,7 @@ class iouringEngine : public MasterEventEngine, public CascadingEventEngine {
     }
 
     int cancel_wait() override {
-        if (eventfd_write(m_cancel_fd, 1) != 0) {
+        if (eventfd_write(m_eventfd, 1) != 0) {
             LOG_ERRNO_RETURN(0, -1, "iouring: write eventfd failed");
         }
         return 0;
@@ -399,8 +402,9 @@ class iouringEngine : public MasterEventEngine, public CascadingEventEngine {
 
 private:
     struct ioCtx {
-        photon::thread* th_id;
-        int32_t res;
+        ioCtx(bool canceller, bool event) : is_canceller(canceller), is_event(event) {}
+        photon::thread* th_id = photon::CURRENT;
+        int32_t res = -1;
         bool is_canceller;
         bool is_event;
     };
@@ -501,23 +505,6 @@ class iouringEngine : public MasterEventEngine, public CascadingEventEngine {
         }
     }
 
-    void run_cancel_poller() {
-        while (m_cancel_poller_running) {
-            uint32_t poll_mask = evmap.translate_bitwisely(EVENT_READ);
-            int ret = async_io(&io_uring_prep_poll_add, -1, 0, m_cancel_fd, poll_mask);
-            if (ret < 0) {
-                if (errno == EINTR) {
-                    break;
-                }
-                LOG_ERROR("iouring: poll eventfd failed, `", ERRNO());
-            }
-            eventfd_t val;
-            if (eventfd_read(m_cancel_fd, &val) != 0) {
-                LOG_ERROR("iouring: read eventfd failed, `", ERRNO());
-            }
-        }
-    }
-
     static void usec_to_timespec(int64_t usec, __kernel_timespec* ts) {
         int64_t usec_rounded_to_sec = usec / 1000000L * 1000000L;
         long long nsec = (usec - usec_rounded_to_sec) * 1000L;
@@ -533,11 +520,8 @@ class iouringEngine : public MasterEventEngine, public CascadingEventEngine {
     static const int REGISTER_FILES_SPARSE_FD = -1;
     static const int REGISTER_FILES_MAX_NUM = 10000;
     bool m_master;
-    int m_cascading_event_fd = -1;
     io_uring* m_ring = nullptr;
-    int m_cancel_fd = -1;
-    thread* m_cancel_poller = nullptr;
-    bool m_cancel_poller_running = true;
+    int m_eventfd = -1;
     std::unordered_map<fdInterest, eventCtx, fdInterestHasher> m_event_contexts;
     static int m_register_files_flag;
     static int m_cooperative_task_flag;
diff --git a/thread/test/CMakeLists.txt b/thread/test/CMakeLists.txt
index a1847f54..44634a93 100644
--- a/thread/test/CMakeLists.txt
+++ b/thread/test/CMakeLists.txt
@@ -30,4 +30,8 @@ add_test(NAME test-tls-order-photon COMMAND $<TARGET_FILE:test-tls-order-photon>
 
 add_executable(test-lib-data test-lib-data.cpp)
 target_link_libraries(test-lib-data PRIVATE photon_shared)
-add_test(NAME test-lib-data COMMAND $<TARGET_FILE:test-lib-data>)
\ No newline at end of file
+add_test(NAME test-lib-data COMMAND $<TARGET_FILE:test-lib-data>)
+
+add_executable(test-multi-vcpu-locking test-multi-vcpu-locking.cpp)
+target_link_libraries(test-multi-vcpu-locking PRIVATE photon_static)
+add_test(NAME test-multi-vcpu-locking COMMAND $<TARGET_FILE:test-multi-vcpu-locking>)
\ No newline at end of file
diff --git a/thread/test/test-multi-vcpu-locking.cpp b/thread/test/test-multi-vcpu-locking.cpp
new file mode 100644
index 00000000..86456812
--- /dev/null
+++ b/thread/test/test-multi-vcpu-locking.cpp
@@ -0,0 +1,93 @@
+/*
+Copyright 2022 The Photon Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#include <atomic>
+#include <chrono>
+#include <vector>
+#include <gtest/gtest.h>
+#include <photon/photon.h>
+#include <photon/thread/std-compat.h>
+#include <photon/common/alog.h>
+#include <photon/common/utility.h>
+
+static constexpr int nMutexes = 4;
+static constexpr int nWorkers = 4;
+static constexpr int nThreads = 32;    // Increase if necessary
+static constexpr int testTimeSeconds = 60;
+
+photon_std::mutex mutexes[nMutexes];
+std::atomic<int64_t> acquisitionCounters[nMutexes];
+std::atomic<int64_t> acquisitionDurations[nMutexes];
+bool running = true;
+
+static void timedLock(photon_std::mutex& mutex, int index) {
+    long long durationMicros;
+    auto start = std::chrono::steady_clock::now();
+    {
+        photon_std::lock_guard<photon_std::mutex> lock(mutex);
+        acquisitionCounters[index].fetch_add(1);
+        auto end = std::chrono::steady_clock::now();
+        durationMicros = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();
+        acquisitionDurations[index].fetch_add(durationMicros);
+    }
+    if (durationMicros > 1'000'000) {
+        LOG_ERROR("long acquisition. mutex `, duration: `ms", index, durationMicros / 1000);
+        std::abort();
+    }
+}
+
+static void myThread(int tid) {
+    LOG_INFO("thread ` starting", tid);
+    while (running) {
+        for (int i = 0; i < nMutexes; i++) {
+            timedLock(mutexes[i], i);
+        }
+    }
+}
+
+TEST(multi_vcpu_locking, long_time_acquisition_should_abort) {
+    int ret = photon::init(photon::INIT_EVENT_DEFAULT, photon::INIT_IO_NONE);
+    GTEST_ASSERT_EQ(0, ret);
+    DEFER(photon::fini());
+
+    ret = photon_std::work_pool_init(nWorkers, photon::INIT_EVENT_DEFAULT, photon::INIT_IO_NONE);
+    GTEST_ASSERT_EQ(0, ret);
+    DEFER(photon_std::work_pool_fini());
+
+    std::vector<photon_std::thread> threads;
+    for (int i = 0; i < nThreads; i++) {
+        threads.emplace_back(myThread, i);
+    }
+
+    for (int i = 0; i < testTimeSeconds; ++i) {
+        for (int j = 0; j < nMutexes; j++) {
+            auto count = acquisitionCounters[j].load();
+            auto durationMs = acquisitionDurations[j].load() / 1000;
+            LOG_INFO("mutex `: total acquisitions: `, total wait time: `ms", j, count, durationMs);
+        }
+        photon_std::this_thread::sleep_for(std::chrono::seconds(1));
+    }
+
+    running = false;
+    for (auto& th: threads) {
+        th.join();
+    }
+}
+
+int main(int argc, char** argv) {
+    ::testing::InitGoogleTest(&argc, argv);
+    return RUN_ALL_TESTS();
+}
\ No newline at end of file

From 31d1ad6639a0dc6ba72c4829954c6946d0578520 Mon Sep 17 00:00:00 2001
From: Louis Williams <louiswilliams@users.noreply.github.com>
Date: Sun, 19 Nov 2023 23:31:40 -0800
Subject: [PATCH 20/76] GDB pretty printer for ARM (#262)

---
 tools/photongdb.py | 49 ++++++++++++++++++++++++++++++++++------------
 1 file changed, 37 insertions(+), 12 deletions(-)

diff --git a/tools/photongdb.py b/tools/photongdb.py
index fcb6cb63..d4d3e76b 100644
--- a/tools/photongdb.py
+++ b/tools/photongdb.py
@@ -28,6 +28,23 @@ class bcolors:
 enabling = False
 photon = []
 
+def get_arch():
+    frame = gdb.selected_frame()
+    arch = frame.architecture()
+    return arch.name()
+
+def get_regs(arch):
+    regs = {}
+    if arch == 'aarch64':
+        regs['sp'] = '$sp'
+        regs['bp'] = '$x29'
+        regs['ip'] = '$pc'
+    else:
+        regs['sp'] = '$rsp'
+        regs['bp'] = '$rbp'
+        regs['ip'] = '$rip'
+
+    return regs
 
 def cprint(stat, *args):
     print('{}{}{} {}'.format(CMAP[stat], stat, bcolors.ENDC,
@@ -55,11 +72,11 @@ def in_sleep(q):
     return [(q['_M_impl']['_M_start'][i]) for i in range(size)]
 
 
-def switch_to_ph(rsp, rbp, rip):
+def switch_to_ph(regs, rsp, rbp, rip):
     cprint('SWITCH', "to {} {} {}".format(hex(rsp), hex(rbp), hex(rip)))
-    gdb.parse_and_eval("$rsp={}".format(rsp))
-    gdb.parse_and_eval("$rbp={}".format(rbp))
-    gdb.parse_and_eval("$rip={}".format(rip))
+    gdb.parse_and_eval("{}={}".format(regs['sp'], rsp))
+    gdb.parse_and_eval("{}={}".format(regs['bp'], rbp))
+    gdb.parse_and_eval("{}={}".format(regs['ip'], rip))
 
 
 def get_u64_ptr(p):
@@ -143,14 +160,20 @@ def invoke(self, arg, tty):
         if i < 0 or i > len(photon):
             print("No such photon thread")
             return
-        switch_to_ph(photon[i][2], photon[i][3], photon[i][4])
+
+        arch = get_arch()
+        regs = get_regs(arch)
+        switch_to_ph(regs, photon[i][2], photon[i][3], photon[i][4])
 
 
 def photon_init():
     global photon
-    set_u64_reg('$saved_rsp', '$rsp')
-    set_u64_reg('$saved_rbp', '$rbp')
-    set_u64_reg('$saved_rip', '$rip')
+
+    arch = get_arch()
+    regs = get_regs(arch)
+    set_u64_reg('$saved_rsp', regs['sp'])
+    set_u64_reg('$saved_rbp', regs['bp'])
+    set_u64_reg('$saved_rip', regs['ip'])
     load_photon_threads()
     if len(photon) == 0:
         return
@@ -171,9 +194,11 @@ def invoke(self, arg, tty):
 def photon_restore():
     if not enabling:
         return
-    set_u64_reg('$rsp', '$saved_rsp')
-    set_u64_reg('$rbp', '$saved_rbp')
-    set_u64_reg('$rip', '$saved_rip')
+    arch = get_arch()
+    regs = get_regs(arch)
+    set_u64_reg(regs['sp'], '$saved_rsp')
+    set_u64_reg(regs['bp'], '$saved_rbp')
+    set_u64_reg(regs['ip'], '$saved_rip')
 
 
 class PhotonRestore(gdb.Command):
@@ -208,4 +233,4 @@ def invoke(self, arg, tty):
 PhotonLs()
 PhotonFr()
 
-cprint('INFO', 'Photon-GDB-extension loaded')
\ No newline at end of file
+cprint('INFO', 'Photon-GDB-extension loaded')

From 2348d41c95bbb8f6336a7975055363ffa660f1dd Mon Sep 17 00:00:00 2001
From: Bob Chen <beef9999@qq.com>
Date: Mon, 20 Nov 2023 16:27:04 +0800
Subject: [PATCH 21/76] update CI script

---
 .github/workflows/ci.linux.arm.yml | 16 ++++++----------
 .github/workflows/ci.linux.x86.yml | 18 +++++++-----------
 2 files changed, 13 insertions(+), 21 deletions(-)

diff --git a/.github/workflows/ci.linux.arm.yml b/.github/workflows/ci.linux.arm.yml
index 4b903cf8..5418a3e3 100644
--- a/.github/workflows/ci.linux.arm.yml
+++ b/.github/workflows/ci.linux.arm.yml
@@ -8,7 +8,7 @@ on:
 
 jobs:
   centos8-gcc921-epoll-release:
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, Linux, ARM64]
 
     container:
       image: dokken/centos-stream-8:sha-40294ce
@@ -35,10 +35,8 @@ jobs:
       - name: Build
         run: |
           source /opt/rh/gcc-toolset-9/enable
-          cmake -B build -D CMAKE_BUILD_TYPE=MinSizeRel \
-            -D PHOTON_BUILD_DEPENDENCIES=OFF -D PHOTON_BUILD_TESTING=ON \
-            -D PHOTON_ENABLE_SASL=ON -D PHOTON_ENABLE_FUSE=ON \
-            -D PHOTON_ENABLE_EXTFS=ON
+          cmake -B build -D CMAKE_BUILD_TYPE=MinSizeRel -D PHOTON_BUILD_TESTING=ON \
+            -D PHOTON_ENABLE_SASL=ON -D PHOTON_ENABLE_FUSE=ON -D PHOTON_ENABLE_EXTFS=ON
           cmake --build build -j -- VERBOSE=1
 
       - name: Test
@@ -47,7 +45,7 @@ jobs:
           ctest --timeout 3600 -V
 
   centos8-gcc921-epoll-debug:
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, Linux, ARM64]
 
     container:
       image: dokken/centos-stream-8:sha-40294ce
@@ -74,10 +72,8 @@ jobs:
       - name: Build
         run: |
           source /opt/rh/gcc-toolset-9/enable
-          cmake -B build -D CMAKE_BUILD_TYPE=Debug \
-            -D PHOTON_BUILD_DEPENDENCIES=OFF -D PHOTON_BUILD_TESTING=ON \
-            -D PHOTON_ENABLE_SASL=ON -D PHOTON_ENABLE_FUSE=ON \
-            -D PHOTON_ENABLE_EXTFS=ON
+          cmake -B build -D CMAKE_BUILD_TYPE=Debug -D PHOTON_BUILD_TESTING=ON \
+            -D PHOTON_ENABLE_SASL=ON -D PHOTON_ENABLE_FUSE=ON -D PHOTON_ENABLE_EXTFS=ON
           cmake --build build -j -- VERBOSE=1
 
       - name: Test
diff --git a/.github/workflows/ci.linux.x86.yml b/.github/workflows/ci.linux.x86.yml
index cf083d05..fce7d236 100644
--- a/.github/workflows/ci.linux.x86.yml
+++ b/.github/workflows/ci.linux.x86.yml
@@ -35,10 +35,8 @@ jobs:
       - name: Build
         run: |
           source /opt/rh/gcc-toolset-9/enable
-          cmake -B build -D CMAKE_BUILD_TYPE=MinSizeRel \
-            -D PHOTON_BUILD_DEPENDENCIES=OFF -D PHOTON_BUILD_TESTING=ON \
-            -D PHOTON_ENABLE_SASL=ON -D PHOTON_ENABLE_FUSE=ON \
-            -D PHOTON_ENABLE_EXTFS=ON
+          cmake -B build -D CMAKE_BUILD_TYPE=MinSizeRel -D PHOTON_BUILD_TESTING=ON \
+            -D PHOTON_ENABLE_SASL=ON -D PHOTON_ENABLE_FUSE=ON -D PHOTON_ENABLE_EXTFS=ON
           cmake --build build -j -- VERBOSE=1
 
       - name: Test
@@ -74,10 +72,8 @@ jobs:
       - name: Build
         run: |
           source /opt/rh/gcc-toolset-9/enable
-          cmake -B build -D CMAKE_BUILD_TYPE=Debug \
-            -D PHOTON_BUILD_DEPENDENCIES=OFF -D PHOTON_BUILD_TESTING=ON \
-            -D PHOTON_ENABLE_SASL=ON -D PHOTON_ENABLE_FUSE=ON \
-            -D PHOTON_ENABLE_EXTFS=ON
+          cmake -B build -D CMAKE_BUILD_TYPE=Debug -D PHOTON_BUILD_TESTING=ON \
+            -D PHOTON_ENABLE_SASL=ON -D PHOTON_ENABLE_FUSE=ON -D PHOTON_ENABLE_EXTFS=ON
           cmake --build build -j -- VERBOSE=1
 
       - name: Test
@@ -86,8 +82,7 @@ jobs:
           ctest --timeout 3600 -V
 
   centos8-gcc921-iouring-release:
-#    runs-on: [self-hosted, Linux, X64]
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, Linux, X64, io_uring]
 
     container:
       image: dokken/centos-stream-8:sha-40294ce
@@ -116,9 +111,10 @@ jobs:
             -D PHOTON_BUILD_DEPENDENCIES=ON \
             -D PHOTON_BUILD_TESTING=ON \
             -D PHOTON_ENABLE_URING=ON
-          cmake --build build -j -- VERBOSE=1
+          cmake --build build -j
 
       - name: Test
         run: |
           cd build
+          ulimit -l unlimited
           ctest --timeout 3600 -V

From 9c6405001eaf22c8534b79e703077dcc10e89339 Mon Sep 17 00:00:00 2001
From: Louis Williams <louiswilliams@users.noreply.github.com>
Date: Wed, 22 Nov 2023 23:08:52 -0800
Subject: [PATCH 22/76] Use acquire-release memory order for mutex::try_lock
 (#274)

(cherry picked from commit 201fb0233d0b058dc72c540f73db2af0fcba3a78)
---
 thread/thread.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/thread/thread.cpp b/thread/thread.cpp
index d2a85426..1a6d3fe6 100644
--- a/thread/thread.cpp
+++ b/thread/thread.cpp
@@ -1572,7 +1572,7 @@ R"(
     {
         thread* ptr = nullptr;
         bool ret = owner.compare_exchange_strong(ptr, CURRENT,
-            std::memory_order_release, std::memory_order_relaxed);
+            std::memory_order_acq_rel, std::memory_order_relaxed);
         return (int)ret - 1;
     }
     inline void do_mutex_unlock(mutex* m)

From a43fca99dd732187c3288389034810f8311cf12c Mon Sep 17 00:00:00 2001
From: Bob Chen <beef9999@qq.com>
Date: Fri, 24 Nov 2023 09:55:03 +0800
Subject: [PATCH 23/76] Fix cascading engine submit; Support running specified
 event engine in CI test (#273)

---
 .github/workflows/ci.linux.x86.yml      |   1 +
 CMakeLists.txt                          |   4 +-
 io/fd-events.h                          |   1 +
 io/iouring-wrapper.cpp                  |  16 +-
 io/test/CMakeLists.txt                  |   2 +-
 io/test/test-iouring.cpp                | 201 +++++++++++++++---------
 test/ci-tools.cpp                       |  26 +++
 test/ci-tools.h                         |   7 +
 thread/test/test-multi-vcpu-locking.cpp |   4 +-
 9 files changed, 178 insertions(+), 84 deletions(-)
 create mode 100644 test/ci-tools.cpp
 create mode 100644 test/ci-tools.h

diff --git a/.github/workflows/ci.linux.x86.yml b/.github/workflows/ci.linux.x86.yml
index fce7d236..d19e377b 100644
--- a/.github/workflows/ci.linux.x86.yml
+++ b/.github/workflows/ci.linux.x86.yml
@@ -117,4 +117,5 @@ jobs:
         run: |
           cd build
           ulimit -l unlimited
+          export PHOTON_CI_EV_ENGINE=io_uring
           ctest --timeout 3600 -V
diff --git a/CMakeLists.txt b/CMakeLists.txt
index ed48b296..3d73dc91 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -329,8 +329,10 @@ endif ()
 if (PHOTON_BUILD_TESTING)
     include(CTest)
 
+    add_library(ci-tools STATIC test/ci-tools.cpp)
+
     include_directories(photon_static ${GFLAGS_INCLUDE_DIRS} ${GOOGLETEST_INCLUDE_DIRS})
-    link_libraries(${GFLAGS_LIBRARIES} ${GOOGLETEST_LIBRARIES})
+    link_libraries(${GFLAGS_LIBRARIES} ${GOOGLETEST_LIBRARIES} ci-tools)
 
     add_subdirectory(examples)
     add_subdirectory(common/checksum/test)
diff --git a/io/fd-events.h b/io/fd-events.h
index fb65f8a1..0bcd9986 100644
--- a/io/fd-events.h
+++ b/io/fd-events.h
@@ -111,6 +111,7 @@ class CascadingEventEngine {
 
     /**
      * @brief Wait for events, returns number of the arrived events, and their associated `data`
+     * @note This call will not return until timeout, if there had been no events.
      * @param[out] data
      * @return -1 for error, positive integer for the number of events, 0 for no events and should run it again
      * @warning Do NOT block vcpu
diff --git a/io/iouring-wrapper.cpp b/io/iouring-wrapper.cpp
index d2e179ca..7137788c 100644
--- a/io/iouring-wrapper.cpp
+++ b/io/iouring-wrapper.cpp
@@ -117,11 +117,11 @@ class iouringEngine : public MasterEventEngine, public CascadingEventEngine {
             }
             ret = io_uring_register_files(m_ring, entries, REGISTER_FILES_MAX_NUM);
             if (ret != 0) {
-                LOG_ERROR_RETURN(EPERM, -1, "iouring: unable to register files, ", ERRNO(-ret));
+                LOG_ERROR_RETURN(-ret, -1, "iouring: unable to register files, ", ERRNO(-ret));
             }
         }
 
-        m_eventfd = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK);
+        m_eventfd = eventfd(0, EFD_CLOEXEC);
         if (m_eventfd < 0) {
             LOG_ERRNO_RETURN(0, -1, "iouring: failed to create eventfd");
         }
@@ -135,7 +135,7 @@ class iouringEngine : public MasterEventEngine, public CascadingEventEngine {
             io_uring_sqe_set_data(sqe, this);
             ret = io_uring_submit(m_ring);
             if (ret <= 0) {
-                LOG_ERROR_RETURN(0, -1, "iouring: fail to submit multishot poll, ", ERRNO(-ret));
+                LOG_ERROR_RETURN(-ret, -1, "iouring: fail to submit multishot poll, ", ERRNO(-ret));
             }
         } else {
             // Register cascading engine to eventfd
@@ -240,6 +240,10 @@ class iouringEngine : public MasterEventEngine, public CascadingEventEngine {
             io_uring_prep_poll_multishot(sqe, fd_interest.fd, fd_interest.interest);
         }
         io_uring_sqe_set_data(sqe, &pair.first->second.io_ctx);
+        int ret = io_uring_submit(m_ring);
+        if (ret < 0) {
+            LOG_ERROR_RETURN(-ret, -1, "iouring: fail to submit when adding interest, ", ERRNO(-ret));
+        }
         return 0;
     }
 
@@ -256,6 +260,10 @@ class iouringEngine : public MasterEventEngine, public CascadingEventEngine {
 
         io_uring_prep_poll_remove(sqe, (__u64) &iter->second.io_ctx);
         io_uring_sqe_set_data(sqe, nullptr);
+        int ret = io_uring_submit(m_ring);
+        if (ret < 0) {
+            LOG_ERROR_RETURN(-ret, -1, "iouring: fail to submit when removing interest, ", ERRNO(-ret));
+        }
         return 0;
     }
 
@@ -287,7 +295,7 @@ class iouringEngine : public MasterEventEngine, public CascadingEventEngine {
                 LOG_ERROR_RETURN(0, -1, "iouring: multi-shot poll got POLLERR");
             }
             if (!ctx->is_event) {
-                LOG_ERROR_RETURN(0, -1, "iouring: only cascading engine need to handle event. Must be a bug...")
+                LOG_ERROR_RETURN(0, -1, "iouring: cascading engine only needs to handle event. Must be a bug...")
             }
             eventCtx* event_ctx = container_of(ctx, eventCtx, io_ctx);
             fdInterest fd_interest{event_ctx->event.fd, (uint32_t)evmap.translate_bitwisely(event_ctx->event.interests)};
diff --git a/io/test/CMakeLists.txt b/io/test/CMakeLists.txt
index eea05c43..5a3ff5d6 100644
--- a/io/test/CMakeLists.txt
+++ b/io/test/CMakeLists.txt
@@ -14,6 +14,6 @@ target_link_libraries(test-syncio PRIVATE photon_shared)
 add_test(NAME test-syncio COMMAND $<TARGET_FILE:test-syncio>)
 
 add_executable(test-iouring test-iouring.cpp)
-target_link_libraries(test-iouring PRIVATE photon_shared)
+target_link_libraries(test-iouring PRIVATE photon_static)
 add_test(NAME test-iouring COMMAND $<TARGET_FILE:test-iouring>)
 endif ()
\ No newline at end of file
diff --git a/io/test/test-iouring.cpp b/io/test/test-iouring.cpp
index 72f181ea..464bb088 100644
--- a/io/test/test-iouring.cpp
+++ b/io/test/test-iouring.cpp
@@ -33,7 +33,7 @@ limitations under the License.
 #include <photon/io/iouring-wrapper.h>
 #include <photon/common/alog.h>
 #include <photon/photon.h>
-
+#include "../../test/ci-tools.h"
 
 using namespace photon;
 
@@ -339,183 +339,238 @@ TEST(perf, DISABLED_read) {
 
 /* Event Engine Tests */
 
-photon::CascadingEventEngine* new_cascading_engine(bool iouring = false) {
-    // return photon::new_iouring_cascading_engine();
-    return photon::new_epoll_cascading_engine();
-}
+class event_engine : public testing::Test {
+protected:
+    void SetUp() override {
+        GTEST_ASSERT_EQ(0, photon::init(ci_ev_engine, photon::INIT_IO_NONE));
+#ifdef PHOTON_URING
+        engine = (ci_ev_engine == photon::INIT_EVENT_EPOLL) ? photon::new_epoll_cascading_engine()
+                                                          : photon::new_iouring_cascading_engine();
+#else
+        engine = photon::new_default_cascading_engine();
+#endif
+    }
+    void TearDown() override {
+        delete engine;
+        photon::fini();
+    };
 
-TEST(event_engine, master) {
+    photon::CascadingEventEngine* engine = nullptr;
+};
+
+TEST_F(event_engine, master) {
     int fd[2];
     pipe(fd);
     char buf[1];
+    photon::semaphore sem;
     auto f = [&] {
-        LOG_INFO("sleep 2s");
-        photon::thread_sleep(2);
-        LOG_INFO("start write");
+        sem.wait(1);
         write(fd[1], buf, 1);
     };
-    photon::thread* sub = photon::thread_create11(&decltype(f)::operator(), &f);
+    photon::thread* sub = photon::thread_create11(f);
     photon::thread_enable_join(sub);
-    LOG_INFO("wait 3s at most");
-    ASSERT_EQ(photon::wait_for_fd_readable(fd[0], 3000000), 0);
+    sem.signal(1);
+    LOG_INFO("wait 1s at most");
+    ASSERT_EQ(0, photon::wait_for_fd_readable(fd[0], 1000000));
     photon::thread_join((photon::join_handle*) sub);
 }
 
-TEST(event_engine, master_timeout) {
+TEST_F(event_engine, master_timeout) {
     int fd[2];
     pipe(fd);
     char buf[1];
     auto f = [&] {
         LOG_INFO("sleep 2s");
         photon::thread_sleep(2);
-        LOG_INFO("start write");
         write(fd[1], buf, 1);
     };
-    photon::thread* sub = photon::thread_create11(&decltype(f)::operator(), &f);
+    photon::thread* sub = photon::thread_create11(f);
     photon::thread_enable_join(sub);
     LOG_INFO("wait 1s at most");
-    ASSERT_EQ(photon::wait_for_fd_readable(fd[0], 1000000), -1);
-    ASSERT_EQ(errno, ETIMEDOUT);
+    ASSERT_EQ(-1, photon::wait_for_fd_readable(fd[0], 1000000));
+    ASSERT_EQ(ETIMEDOUT, errno);
     photon::thread_join((photon::join_handle*) sub);
 }
 
-TEST(event_engine, master_interrupted) {
+TEST_F(event_engine, master_interrupted) {
     int fd[2];
     pipe(fd);
+    photon::semaphore sem;
     photon::thread* main = photon::CURRENT;
     auto f = [&] {
-        LOG_INFO("sleep 2s");
-        photon::thread_sleep(2);
+        sem.wait(1);
         LOG_INFO("start interrupt main");
-        photon::thread_interrupt(main, EPERM);
+        photon::thread_interrupt(main);
 
     };
-    photon::thread* sub = photon::thread_create11(&decltype(f)::operator(), &f);
+    photon::thread* sub = photon::thread_create11(f);
     photon::thread_enable_join(sub);
-    LOG_INFO("wait 3s at most");
-    ASSERT_EQ(photon::wait_for_fd_readable(fd[0], 3000000), -1);
-    ASSERT_EQ(errno, EPERM);
+    LOG_INFO("wait 1s at most");
+    sem.signal(1);
+    ASSERT_EQ(-1, photon::wait_for_fd_readable(fd[0], 1000000));
+    ASSERT_EQ(EINTR, errno);
     photon::thread_join((photon::join_handle*) sub);
 }
 
-TEST(event_engine, master_interrupted_after_io) {
+TEST_F(event_engine, master_interrupted_after_io) {
     int fd[2];
     pipe(fd);
     char buf[1];
+    photon::semaphore sem;
     photon::thread* main = photon::CURRENT;
     auto f = [&] {
-        LOG_INFO("sleep 2s");
-        photon::thread_sleep(2);
-        LOG_INFO("start write");
+        sem.wait(1);
         write(fd[1], buf, 1);
         LOG_INFO("start interrupt main");
-        photon::thread_interrupt(main, EPERM);
+        photon::thread_interrupt(main);
 
     };
-    photon::thread* sub = photon::thread_create11(&decltype(f)::operator(), &f);
+    photon::thread* sub = photon::thread_create11(f);
     photon::thread_enable_join(sub);
-    LOG_INFO("wait 3s at most");
-    ASSERT_EQ(photon::wait_for_fd_readable(fd[0], 3000000), -1);
-    ASSERT_EQ(errno, EPERM);
+    LOG_INFO("wait 1s at most");
+    sem.signal(1);
+    ASSERT_EQ(-1, photon::wait_for_fd_readable(fd[0], 1000000));
+    ASSERT_EQ(EINTR, errno);
     photon::thread_join((photon::join_handle*) sub);
 }
 
-TEST(event_engine, cascading) {
+TEST_F(event_engine, cascading_add) {
     int fd1[2];
     int fd2[2];
     pipe(fd1);
     pipe(fd2);
     char buf[1];
+    photon::semaphore sem;
     auto f = [&] {
-        photon::thread_sleep(2);
-        LOG_INFO("write pipe");
+        sem.wait(1);
         write(fd1[1], buf, 1);
         write(fd2[1], buf, 1);
     };
-    photon::thread* sub = photon::thread_create11(&decltype(f)::operator(), &f);
+    photon::thread* sub = photon::thread_create11(f);
     photon::thread_enable_join(sub);
 
-    auto engine = new_cascading_engine();
-    DEFER(delete engine);
     engine->add_interest({fd1[0], photon::EVENT_READ, (void*) 0x1111});
     engine->add_interest({fd2[0], photon::EVENT_READ, (void*) 0x2222});
 
+    sem.signal(1);
+
     void* data[5] = {};
     ssize_t num_events = engine->wait_for_events(data, 5, -1UL);
-    ASSERT_EQ(num_events, 2);
+    ASSERT_EQ(2, num_events);
+
+    // data order is not ensured
     bool b1 = data[0] == (void*) 0x1111 && data[1] == (void*) 0x2222;
     bool b2 = data[0] == (void*) 0x2222 && data[1] == (void*) 0x1111;
     ASSERT_EQ(b1 || b2, true);
 
-    engine->rm_interest({fd1[0], photon::EVENT_READ, (void*) 0x1111});
-    engine->rm_interest({fd2[0], photon::EVENT_READ, (void*) 0x2222});
-
     photon::thread_join((photon::join_handle*) sub);
 }
 
-TEST(event_engine, cascading_timeout) {
+TEST_F(event_engine, cascading_timeout) {
     int fd1[2];
-    int fd2[2];
     pipe(fd1);
-    pipe(fd2);
     char buf[1];
     auto f = [&] {
         photon::thread_sleep(2);
         write(fd1[1], buf, 1);
-        write(fd2[1], buf, 1);
     };
-    photon::thread* sub = photon::thread_create11(&decltype(f)::operator(), &f);
+    photon::thread* sub = photon::thread_create11(f);
     photon::thread_enable_join(sub);
 
-    auto engine = new_cascading_engine();
-    DEFER(delete engine);
     engine->add_interest({fd1[0], photon::EVENT_READ, (void*) 0x1111});
-    engine->add_interest({fd2[0], photon::EVENT_READ, (void*) 0x2222});
 
     void* data[5] = {};
     ssize_t num_events = engine->wait_for_events(data, 5, 1000000);
     ASSERT_EQ(0, num_events);
 
-    engine->rm_interest({fd1[0], photon::EVENT_READ, (void*) 0x1111});
-    engine->rm_interest({fd2[0], photon::EVENT_READ, (void*) 0x2222});
-
     photon::thread_join((photon::join_handle*) sub);
 }
 
-TEST(event_engine, cascading_remove) {
+TEST_F(event_engine, cascading_remove) {
     int fd1[2];
     int fd2[2];
     pipe(fd1);
     pipe(fd2);
     char buf[1];
-    auto engine = new_cascading_engine();
-    DEFER(delete engine);
-    auto f = [&] {
-        photon::thread_sleep(1);
+    photon::semaphore sem;
+
+    auto sub = photon::thread_create11([&] {
+        sem.wait(1);
+        // 3. Remove one
         engine->rm_interest({fd1[0], photon::EVENT_READ, (void*) 0x1111});
-        photon::thread_sleep(1);
-        LOG_INFO("start write fd");
+        // 4. Write both
         write(fd1[1], buf, 1);
         write(fd2[1], buf, 1);
-    };
-    photon::thread* sub = photon::thread_create11(&decltype(f)::operator(), &f);
+    });
     photon::thread_enable_join(sub);
 
+    // 1. Add both
     engine->add_interest({fd1[0], photon::EVENT_READ, (void*) 0x1111});
     engine->add_interest({fd2[0], photon::EVENT_READ, (void*) 0x2222});
 
+    sem.signal(1);
+
+    // 2. Wait both
     void* data[5] = {};
     ssize_t num_events = engine->wait_for_events(data, 5, -1UL);
-    ASSERT_EQ(num_events, 1);
+
+    // 5. Should get only one
+    ASSERT_EQ(1, num_events);
     ASSERT_EQ(data[0], (void*) 0x2222);
+    photon::thread_join((photon::join_handle*) sub);
 
+    sub = photon::thread_create11([&] {
+        // 7. Remove one
+        engine->rm_interest({fd2[0], photon::EVENT_READ, (void*) 0x2222});
+    });
+    photon::thread_enable_join(sub);
 
-    engine->rm_interest({fd2[0], photon::EVENT_READ, (void*) 0x2222});
+    // 6. Wait one
+    num_events = engine->wait_for_events(data, 5, 1000000);
+    ASSERT_EQ(0, num_events);
 
     photon::thread_join((photon::join_handle*) sub);
 }
 
-TEST(event_engine, cascading_one_shot) {
+TEST_F(event_engine, cascading_remove_inplace) {
+    int fd1[2];
+    pipe(fd1);
+    char buf[1];
+
+    engine->add_interest({fd1[0], photon::EVENT_READ, (void*) 0x1111});
+    engine->rm_interest({fd1[0], photon::EVENT_READ, (void*) 0x2222});
+
+    write(fd1[1], buf, 1);
+
+    void* data[5] = {};
+    ssize_t num_events = engine->wait_for_events(data, 5, 1000000);
+    ASSERT_EQ(0, num_events);
+}
+
+TEST_F(event_engine, cascading_interrupt) {
+    int fd1[2];
+    pipe(fd1);
+    photon::thread* main = photon::CURRENT;
+    photon::semaphore sem;
+
+    engine->add_interest({fd1[0], photon::EVENT_READ, (void*) 0x1111});
+
+    auto th = photon::thread_create11([&] {
+        sem.wait(1);
+        photon::thread_interrupt(main);
+    });
+    photon::thread_enable_join(th);
+
+    sem.signal(1);
+
+    void* data[5] = {};
+    ssize_t num_events = engine->wait_for_events(data, 5, 1000000);
+    ASSERT_EQ(-1, num_events);
+    ASSERT_EQ(EINTR, errno);
+    photon::thread_join((photon::join_handle*) th);
+}
+
+TEST_F(event_engine, cascading_one_shot) {
     int fd1[2];
     int fd2[2];
     pipe(fd1);
@@ -532,11 +587,9 @@ TEST(event_engine, cascading_one_shot) {
         write(fd2[1], buf, 1);
 
     };
-    photon::thread* sub = photon::thread_create11(&decltype(f)::operator(), &f);
+    photon::thread* sub = photon::thread_create11(f);
     photon::thread_enable_join(sub);
 
-    auto engine = new_cascading_engine();
-    DEFER(delete engine);
     engine->add_interest({fd1[0], photon::EVENT_READ | photon::ONE_SHOT, (void*) 0x1111});
     engine->add_interest({fd2[0], photon::EVENT_READ, (void*) 0x2222});
 
@@ -558,7 +611,6 @@ TEST(event_engine, cascading_one_shot) {
     LOG_INFO("wait non events");
     num_events = engine->wait_for_events(data, 5, 2000000);
     ASSERT_EQ(num_events, 0);
-    ASSERT_EQ(errno, ETIMEDOUT);
 
     photon::thread_join((photon::join_handle*) sub);
 }
@@ -566,14 +618,9 @@ TEST(event_engine, cascading_one_shot) {
 int main(int argc, char** arg) {
     srand(time(nullptr));
     set_log_output_level(ALOG_INFO);
-
     testing::InitGoogleTest(&argc, arg);
     testing::FLAGS_gtest_break_on_failure = true;
     gflags::ParseCommandLineFlags(&argc, &arg, true);
-
-    if (photon::init())
-        return -1;
-    DEFER(photon::fini());
-
+    ci_parse_env();
     return RUN_ALL_TESTS();
 }
diff --git a/test/ci-tools.cpp b/test/ci-tools.cpp
new file mode 100644
index 00000000..265b4ac3
--- /dev/null
+++ b/test/ci-tools.cpp
@@ -0,0 +1,26 @@
+#include "ci-tools.h"
+#include <cstdlib>
+#include <cstring>
+#include "../photon.h"
+
+#if __linux__
+uint64_t ci_ev_engine = photon::INIT_EVENT_EPOLL;
+#else
+uint64_t ci_ev_engine = photon::INIT_EVENT_KQUEUE;
+#endif
+
+uint64_t ci_ev_engine_with_signal = ci_ev_engine | photon::INIT_EVENT_SIGNAL;
+
+void ci_parse_env() {
+    const char* ev_engine = getenv("PHOTON_CI_EV_ENGINE");
+    if (!ev_engine)
+        return;
+    if (strcmp(ev_engine, "epoll") == 0) {
+        ci_ev_engine = photon::INIT_EVENT_EPOLL;
+    } else if (strcmp(ev_engine, "io_uring") == 0) {
+        ci_ev_engine = photon::INIT_EVENT_IOURING;
+    } else if (strcmp(ev_engine, "kqueue") == 0) {
+        ci_ev_engine = photon::INIT_EVENT_KQUEUE;
+    }
+    ci_ev_engine_with_signal = ci_ev_engine | photon::INIT_EVENT_SIGNAL;
+}
\ No newline at end of file
diff --git a/test/ci-tools.h b/test/ci-tools.h
new file mode 100644
index 00000000..f0cbedd8
--- /dev/null
+++ b/test/ci-tools.h
@@ -0,0 +1,7 @@
+#pragma once
+#include <cstdint>
+
+extern uint64_t ci_ev_engine;
+extern uint64_t ci_ev_engine_with_signal;
+
+void ci_parse_env();
\ No newline at end of file
diff --git a/thread/test/test-multi-vcpu-locking.cpp b/thread/test/test-multi-vcpu-locking.cpp
index 86456812..aefafe0c 100644
--- a/thread/test/test-multi-vcpu-locking.cpp
+++ b/thread/test/test-multi-vcpu-locking.cpp
@@ -22,6 +22,7 @@ limitations under the License.
 #include <photon/thread/std-compat.h>
 #include <photon/common/alog.h>
 #include <photon/common/utility.h>
+#include "../../test/ci-tools.h"
 
 static constexpr int nMutexes = 4;
 static constexpr int nWorkers = 4;
@@ -59,7 +60,7 @@ static void myThread(int tid) {
 }
 
 TEST(multi_vcpu_locking, long_time_acquisition_should_abort) {
-    int ret = photon::init(photon::INIT_EVENT_DEFAULT, photon::INIT_IO_NONE);
+    int ret = photon::init(ci_ev_engine, photon::INIT_IO_NONE);
     GTEST_ASSERT_EQ(0, ret);
     DEFER(photon::fini());
 
@@ -89,5 +90,6 @@ TEST(multi_vcpu_locking, long_time_acquisition_should_abort) {
 
 int main(int argc, char** argv) {
     ::testing::InitGoogleTest(&argc, argv);
+    ci_parse_env();
     return RUN_ALL_TESTS();
 }
\ No newline at end of file

From 19099b4cc3e8c3d5b9c8baaf029cb5f23702bc2b Mon Sep 17 00:00:00 2001
From: Bob Chen <beef9999@qq.com>
Date: Sun, 3 Dec 2023 17:38:02 +0800
Subject: [PATCH 24/76] fix ssl build script in cmake

---
 CMake/build-from-src.cmake | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/CMake/build-from-src.cmake b/CMake/build-from-src.cmake
index 16f455ea..646dc9f3 100644
--- a/CMake/build-from-src.cmake
+++ b/CMake/build-from-src.cmake
@@ -82,12 +82,12 @@ function(build_from_src [dep])
                 URL ${PHOTON_OPENSSL_SOURCE}
                 URL_MD5 bad68bb6bd9908da75e2c8dedc536b29
                 BUILD_IN_SOURCE ON
-                CONFIGURE_COMMAND ./config -fPIC no-unit-test no-shared --openssldir=${BINARY_DIR} --prefix=${BINARY_DIR}
-                BUILD_COMMAND make depend -j 4 && make -j 4 #  Not using ${NumCPU}. Too may parallel might fail
+                CONFIGURE_COMMAND ./config -fPIC --prefix=${BINARY_DIR} --openssldir=${BINARY_DIR} shared
+                BUILD_COMMAND make -j ${NumCPU}
                 INSTALL_COMMAND make install
         )
         ExternalProject_Get_Property(openssl SOURCE_DIR)
-        set(OPENSSL_ROOT_DIR ${SOURCE_DIR} PARENT_SCOPE)
+        set(OPENSSL_ROOT_DIR ${BINARY_DIR} PARENT_SCOPE)
         set(OPENSSL_INCLUDE_DIRS ${BINARY_DIR}/include PARENT_SCOPE)
         set(OPENSSL_LIBRARIES ${BINARY_DIR}/lib/libssl.a ${BINARY_DIR}/lib/libcrypto.a PARENT_SCOPE)
 
@@ -101,15 +101,13 @@ function(build_from_src [dep])
                 URL ${PHOTON_CURL_SOURCE}
                 URL_MD5 a66270f11e3fbfad709600bbd1686704
                 BUILD_IN_SOURCE ON
-                CONFIGURE_COMMAND export CC=${CMAKE_C_COMPILER} && export CXX=${CMAKE_CXX_COMPILER} &&
-                    export LD=${CMAKE_LINKER} && export CFLAGS=-fPIC &&
-                    autoreconf -i && ./configure --with-ssl=${OPENSSL_ROOT_DIR}
+                CONFIGURE_COMMAND autoreconf -i && ./configure --with-ssl=${OPENSSL_ROOT_DIR}
                     --without-libssh2 --enable-static --enable-shared=no --enable-optimize
                     --disable-manual --without-libidn
                     --disable-ftp --disable-file --disable-ldap --disable-ldaps
                     --disable-rtsp --disable-dict --disable-telnet --disable-tftp
                     --disable-pop3 --disable-imap --disable-smb --disable-smtp
-                    --disable-gopher --without-nghttp2 --enable-http
+                    --disable-gopher --without-nghttp2 --enable-http --disable-verbose
                     --with-pic=PIC --prefix=${BINARY_DIR}
                 BUILD_COMMAND make -j ${NumCPU}
                 INSTALL_COMMAND make install

From f97ac8f06a2ab4734e2a50d443a7b3f9bfcf9c39 Mon Sep 17 00:00:00 2001
From: Bob Chen <beef9999@qq.com>
Date: Mon, 4 Dec 2023 10:03:22 +0800
Subject: [PATCH 25/76] Fix the va_args order of http header (#286)

---
 fs/httpfs/httpfs.cpp    | 4 +++-
 fs/httpfs/httpfs_v2.cpp | 4 +++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/fs/httpfs/httpfs.cpp b/fs/httpfs/httpfs.cpp
index 4ef3e207..511332d3 100644
--- a/fs/httpfs/httpfs.cpp
+++ b/fs/httpfs/httpfs.cpp
@@ -251,7 +251,9 @@ class HttpFile : public fs::VirtualReadOnlyFile {
     }
 
     void add_header(va_list args) {
-        common_header[va_arg(args, const char*)] = va_arg(args, const char*);
+        auto k = va_arg(args, const char*);
+        auto v = va_arg(args, const char*);
+        common_header[k] = v;
     }
 
     void add_url_param(va_list args) { url_param = va_arg(args, const char*); }
diff --git a/fs/httpfs/httpfs_v2.cpp b/fs/httpfs/httpfs_v2.cpp
index 5aaff947..3aa3d881 100644
--- a/fs/httpfs/httpfs_v2.cpp
+++ b/fs/httpfs/httpfs_v2.cpp
@@ -230,7 +230,9 @@ class HttpFile_v2 : public fs::VirtualReadOnlyFile {
 
     //TODO: 这里是否需要考虑m_common_header被打爆的问题？
     void add_header(va_list args) {
-        m_common_header.insert(va_arg(args, const char*), va_arg(args, const char*));
+        auto k = va_arg(args, const char*);
+        auto v = va_arg(args, const char*);
+        m_common_header.insert(k, v);
     }
 
     void add_url_param(va_list args) { m_url_param = va_arg(args, const char*); }

From 3ae9e95461ea2e025330df4bda372fac9615b3ca Mon Sep 17 00:00:00 2001
From: Bob Chen <beef9999@qq.com>
Date: Tue, 5 Dec 2023 09:59:44 +0800
Subject: [PATCH 26/76] Mute an UBSan check error for SCOPED_LOCK macro (#283)

---
 thread/thread.cpp | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/thread/thread.cpp b/thread/thread.cpp
index 1a6d3fe6..bde8ab21 100644
--- a/thread/thread.cpp
+++ b/thread/thread.cpp
@@ -72,8 +72,6 @@ inline int posix_memalign(void** memptr, size_t alignment, size_t size) {
    by target vcpu in resume_thread(), when its runq becomes empty;
 */
 
-#define SCOPED_MEMBER_LOCK(x) SCOPED_LOCK(&(x)->lock, ((bool)x) * 2)
-
 // Define assembly section header for clang and gcc
 #if defined(__APPLE__)
 #define DEF_ASM_FUNC(name) ".text\n" \
@@ -1199,7 +1197,8 @@ R"(
     __attribute__((always_inline)) inline
     Switch prepare_usleep(uint64_t useconds, thread_list* waitq, RunQ rq = {})
     {
-        SCOPED_MEMBER_LOCK(waitq);
+        spinlock* waitq_lock = waitq ? &waitq->lock : nullptr;
+        SCOPED_LOCK(waitq_lock, ((bool) waitq) * 2);
         SCOPED_LOCK(rq.current->lock);
         assert(!AtomicRunQ(rq).single());
         auto sw = AtomicRunQ(rq).remove_current(states::SLEEPING);

From 8f8e96cdcc8c7e6c1686855adb15a6763a247b70 Mon Sep 17 00:00:00 2001
From: yuchen0cc <91253032+yuchen0cc@users.noreply.github.com>
Date: Wed, 6 Dec 2023 14:31:48 +0800
Subject: [PATCH 27/76] io: support reset photon at fork (#282)

In photon context, event engine based module need reset after
fork, if exec will not be called after fork.
This is implicitly done by pthread_atfork hook.

Signed-off-by: yuchen.cc <yuchen.cc@alibaba-inc.com>
---
 CMakeLists.txt         |   1 +
 io/aio-wrapper.cpp     |  40 ++++-
 io/epoll.cpp           |  10 +-
 io/fstack-dpdk.cpp     |   7 +-
 io/iouring-wrapper.cpp |  14 +-
 io/kqueue.cpp          |  12 +-
 io/reset_handle.cpp    |  52 +++++++
 io/reset_handle.h      |  32 ++++
 io/signal.cpp          |  45 ++++--
 io/test/CMakeLists.txt |   7 +
 io/test/test-fork.cpp  | 325 +++++++++++++++++++++++++++++++++++++++++
 net/curl.cpp           |  20 +++
 photon.cpp             |   9 +-
 13 files changed, 552 insertions(+), 22 deletions(-)
 create mode 100644 io/reset_handle.cpp
 create mode 100644 io/reset_handle.h
 create mode 100644 io/test/test-fork.cpp

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 3d73dc91..9f86eb6e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -166,6 +166,7 @@ file(GLOB PHOTON_SRC
         fs/xfile.cpp
         fs/httpfs/*.cpp
         io/signal.cpp
+        io/reset_handle.cpp
         net/*.cpp
         net/http/*.cpp
         net/security-context/tls-stream.cpp
diff --git a/io/aio-wrapper.cpp b/io/aio-wrapper.cpp
index 2b7b9b0d..2ea2f2c2 100644
--- a/io/aio-wrapper.cpp
+++ b/io/aio-wrapper.cpp
@@ -34,6 +34,7 @@ limitations under the License.
 #include "fd-events.h"
 #include "../common/utility.h"
 #include "../common/alog.h"
+#include "reset_handle.h"
 
 namespace photon
 {
@@ -186,7 +187,6 @@ namespace photon
 
     static void* libaio_polling(void*)
     {
-        libaio_ctx->running = 1;
         DEFER(libaio_ctx->running = 0);
         while (libaio_ctx->running == 1)
         {
@@ -342,6 +342,29 @@ namespace photon
         return rst;
     }
 
+    class AioResetHandle : public ResetHandle {
+        int reset() override {
+            if (!libaio_ctx)
+                return 0;
+            LOG_INFO("reset libaio by reset handle");
+            close(libaio_ctx->evfd);
+            libaio_ctx->evfd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
+            if (libaio_ctx->evfd < 0) {
+                LOG_ERROR("failed to create eventfd ", ERRNO());
+                exit(-1);
+            }
+            io_destroy(libaio_ctx->aio_ctx);
+            libaio_ctx->aio_ctx = {0};
+            int ret = io_setup(IODEPTH, &libaio_ctx->aio_ctx);
+            if (ret < 0) {
+                LOG_ERROR("failed to create aio context by io_setup() ", ERRNO(), VALUE(ret));
+                exit(-1);
+            }
+            thread_interrupt(libaio_ctx->polling_thread, ECANCELED);
+            return 0;
+        }
+    };
+    static thread_local AioResetHandle *reset_handler = nullptr;
 
     int libaio_wrapper_init()
     {
@@ -356,7 +379,7 @@ namespace photon
         int ret = io_setup(IODEPTH, &ctx->aio_ctx);
         if (ret < 0)
         {
-            LOG_ERROR("failed to create aio context by io_setup() ", ERRNO());
+            LOG_ERROR("failed to create aio context by io_setup() ", ERRNO(), VALUE(ret));
             close(ctx->evfd);
             return ret;
         }
@@ -364,7 +387,11 @@ namespace photon
         ctx->polling_thread = thread_create(&libaio_polling, nullptr);
         assert(ctx->polling_thread);
         libaio_ctx = ctx.release();
-        thread_yield_to(libaio_ctx->polling_thread);
+        libaio_ctx->running = 1;
+        if (reset_handler == nullptr) {
+            reset_handler = new AioResetHandle();
+        }
+        LOG_DEBUG("libaio initialized");
         return 0;
     }
 
@@ -384,10 +411,9 @@ namespace photon
         io_destroy(libaio_ctx->aio_ctx);
         close(libaio_ctx->evfd);
         libaio_ctx->evfd = -1;
-        delete libaio_ctx;
-        libaio_ctx = nullptr;
+        safe_delete(libaio_ctx);
+        safe_delete(reset_handler);
+        LOG_DEBUG("libaio finished");
         return 0;
     }
 }
-
-
diff --git a/io/epoll.cpp b/io/epoll.cpp
index d6340831..4100bd69 100644
--- a/io/epoll.cpp
+++ b/io/epoll.cpp
@@ -26,6 +26,7 @@ limitations under the License.
 #include <photon/thread/thread.h>
 #include <photon/io/fd-events.h>
 #include "events_map.h"
+#include "reset_handle.h"
 
 namespace photon {
 #ifndef EPOLLRDHUP
@@ -48,7 +49,7 @@ struct InFlightEvent {
     void* error_data;
 };
 
-class EventEngineEPoll : public MasterEventEngine, public CascadingEventEngine {
+class EventEngineEPoll : public MasterEventEngine, public CascadingEventEngine, public ResetHandle {
 public:
     int _evfd = -1;
     int _engine_fd = -1;
@@ -71,6 +72,13 @@ class EventEngineEPoll : public MasterEventEngine, public CascadingEventEngine {
         epfd = evfd = -1;
         return 0;
     }
+    int reset() override {
+        if_close_fd(_engine_fd);    // close original fd
+        if_close_fd(_evfd);
+        _inflight_events.clear();   // reset members
+        _events_remain = 0;
+        return init();              // re-init
+    }
     virtual ~EventEngineEPoll() override {
         LOG_INFO("Finish event engine: epoll");
         if_close_fd(_engine_fd);
diff --git a/io/fstack-dpdk.cpp b/io/fstack-dpdk.cpp
index 8308b4db..96d53ee0 100644
--- a/io/fstack-dpdk.cpp
+++ b/io/fstack-dpdk.cpp
@@ -27,6 +27,7 @@ limitations under the License.
 #include "../thread/thread11.h"
 #include "../common/alog.h"
 #include "../net/basic_socket.h"
+#include "reset_handle.h"
 
 #ifndef EVFILT_EXCEPT
 #define EVFILT_EXCEPT (-15)
@@ -37,7 +38,7 @@ namespace photon {
 constexpr static EventsMap<EVUnderlay<EVFILT_READ, EVFILT_WRITE, EVFILT_EXCEPT>>
     evmap;
 
-class FstackDpdkEngine : public MasterEventEngine, public CascadingEventEngine {
+class FstackDpdkEngine : public MasterEventEngine, public CascadingEventEngine, public ResetHandle {
 public:
     struct InFlightEvent {
         uint32_t interests = 0;
@@ -76,6 +77,10 @@ class FstackDpdkEngine : public MasterEventEngine, public CascadingEventEngine {
         return 0;
     }
 
+    int reset() override {
+        assert(false);
+    }
+
     ~FstackDpdkEngine() override {
         LOG_INFO("Finish f-stack dpdk engine");
         // if (_n > 0) LOG_INFO(VALUE(_events[0].ident), VALUE(_events[0].filter), VALUE(_events[0].flags));
diff --git a/io/iouring-wrapper.cpp b/io/iouring-wrapper.cpp
index 7137788c..d0e29059 100644
--- a/io/iouring-wrapper.cpp
+++ b/io/iouring-wrapper.cpp
@@ -30,6 +30,7 @@ limitations under the License.
 #include <photon/thread/thread11.h>
 #include <photon/io/fd-events.h>
 #include "events_map.h"
+#include "reset_handle.h"
 
 #ifndef _GNU_SOURCE
 #define _GNU_SOURCE
@@ -42,12 +43,22 @@ namespace photon {
 
 constexpr static EventsMap<EVUnderlay<POLLIN | POLLRDHUP, POLLOUT, POLLERR>> evmap;
 
-class iouringEngine : public MasterEventEngine, public CascadingEventEngine {
+class iouringEngine : public MasterEventEngine, public CascadingEventEngine, public ResetHandle {
 public:
     explicit iouringEngine(bool master) : m_master(master) {}
 
     ~iouringEngine() {
         LOG_INFO("Finish event engine: iouring ", VALUE(m_master));
+        fini();
+    }
+
+    int reset() override {
+        fini();
+        m_event_contexts.clear();
+        return init();
+    }
+
+    int fini() {
         if (m_eventfd >= 0 && !m_master) {
             if (io_uring_unregister_eventfd(m_ring) != 0) {
                 LOG_ERROR("iouring: failed to unregister cascading event fd");
@@ -61,6 +72,7 @@ class iouringEngine : public MasterEventEngine, public CascadingEventEngine {
         }
         delete m_ring;
         m_ring = nullptr;
+        return 0;
     }
 
     int init() {
diff --git a/io/kqueue.cpp b/io/kqueue.cpp
index c2b8121e..4e314585 100644
--- a/io/kqueue.cpp
+++ b/io/kqueue.cpp
@@ -21,13 +21,14 @@ limitations under the License.
 #include <sys/event.h>
 #include <photon/common/alog.h>
 #include "events_map.h"
+#include "reset_handle.h"
 
 namespace photon {
 
 constexpr static EventsMap<EVUnderlay<EVFILT_READ, EVFILT_WRITE, EVFILT_EXCEPT>>
     evmap;
 
-class KQueue : public MasterEventEngine, public CascadingEventEngine {
+class KQueue : public MasterEventEngine, public CascadingEventEngine, public ResetHandle {
 public:
     struct InFlightEvent {
         uint32_t interests = 0;
@@ -55,6 +56,15 @@ class KQueue : public MasterEventEngine, public CascadingEventEngine {
         return 0;
     }
 
+    int reset() override {
+        LOG_INFO("Reset event engine: kqueue");
+        _kq = -1;                   // kqueue fd is not inherited from the parent process
+        _inflight_events.clear();   // reset members
+        _n = 0;
+        _tm = {0, 0};
+        return init();              // re-init
+    }
+
     ~KQueue() override {
         LOG_INFO("Finish event engine: kqueue");
         // if (_n > 0) LOG_INFO(VALUE(_events[0].ident), VALUE(_events[0].filter), VALUE(_events[0].flags));
diff --git a/io/reset_handle.cpp b/io/reset_handle.cpp
new file mode 100644
index 00000000..c560a611
--- /dev/null
+++ b/io/reset_handle.cpp
@@ -0,0 +1,52 @@
+/*
+Copyright 2022 The Photon Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#include "reset_handle.h"
+#include <photon/common/alog.h>
+#include <photon/thread/std-compat.h>
+#include <pthread.h>
+
+namespace photon {
+
+static thread_local ResetHandle *list = nullptr;
+
+ResetHandle::ResetHandle() {
+    LOG_DEBUG("push ", VALUE(this));
+    if (list == nullptr) {
+        list = this;
+    } else {
+        list->insert_tail(this);
+    }
+}
+
+ResetHandle::~ResetHandle() {
+    LOG_DEBUG("erase ", VALUE(this));
+    auto nx = this->remove_from_list();
+    if (this == list)
+        list = nx;
+}
+
+void reset_all_handle() {
+    if (list == nullptr)
+        return;
+    LOG_INFO("reset all handle called");
+    for (auto handler : *list) {
+        LOG_DEBUG("reset ", VALUE(handler));
+        handler->reset();
+    }
+}
+
+} // namespace photon
diff --git a/io/reset_handle.h b/io/reset_handle.h
new file mode 100644
index 00000000..cbc92a6c
--- /dev/null
+++ b/io/reset_handle.h
@@ -0,0 +1,32 @@
+/*
+Copyright 2022 The Photon Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#pragma once
+#include <photon/thread/list.h>
+
+namespace photon {
+
+class ResetHandle : public intrusive_list_node<ResetHandle> {
+public:
+    ResetHandle();
+    virtual ~ResetHandle();
+
+    virtual int reset() = 0;
+};
+
+void reset_all_handle();
+
+} // namespace photon
diff --git a/io/signal.cpp b/io/signal.cpp
index 77bf2cd8..63b1a751 100644
--- a/io/signal.cpp
+++ b/io/signal.cpp
@@ -26,6 +26,7 @@ limitations under the License.
 #include "fd-events.h"
 #include "../common/event-loop.h"
 #include "../common/alog.h"
+#include "reset_handle.h"
 
 namespace photon
 {
@@ -227,14 +228,35 @@ namespace photon
 #endif
     }
 
-    // should be invoked in child process after forked, to clear signal mask
-    static void fork_hook_child(void)
-    {
-        LOG_DEBUG("Fork hook");
-        sigset_t sigset0;       // can NOT use photon::clear_signal_mask(),
-        sigemptyset(&sigset0);  // as memory may be shared with parent, when vfork()ed
-        sigprocmask(SIG_SETMASK, &sigset0, nullptr);
-    }
+    class SignalResetHandle : public ResetHandle {
+        int reset() override {
+            if (sgfd < 0)
+                return 0;
+            LOG_INFO("reset signalfd by reset handle");
+            sigset_t sigset0;       // can NOT use photon::clear_signal_mask(),
+            sigemptyset(&sigset0);  // as memory may be shared with parent, when vfork()ed
+            sigprocmask(SIG_SETMASK, &sigset0, nullptr);
+            // reset sgfd
+            memset(sighandlers, 0, sizeof(sighandlers));
+#ifdef __APPLE__
+            sgfd = kqueue();        // kqueue fd is not inherited from the parent process
+#else
+            close(sgfd);
+            sigfillset(&sigset);
+            sgfd = signalfd(-1, &sigset, SFD_CLOEXEC | SFD_NONBLOCK);
+#endif
+            if (sgfd == -1) {
+                LOG_ERROR("failed to create signalfd() or kqueue()");
+                exit(1);
+            }
+            // interrupt event loop by ETIMEDOUT to replace sgfd
+            if (eloop)
+                thread_interrupt(eloop->loop_thread(), ETIMEDOUT);
+
+            return 0;
+        }
+    };
+    static SignalResetHandle *reset_handler = nullptr;
 
     int sync_signal_init()
     {
@@ -262,9 +284,11 @@ namespace photon
             LOG_ERROR_RETURN(EFAULT, -1, "failed to thread_create() for signal handling");
         }
         eloop->async_run();
-        LOG_INFO("signalfd initialized");
         thread_yield(); // give a chance let eloop to execute do_wait
-        pthread_atfork(nullptr, nullptr, &fork_hook_child);
+        if (reset_handler == nullptr) {
+            reset_handler = new SignalResetHandle();
+        }
+        LOG_INFO("signalfd initialized");
         return clear_signal_mask();
     }
 
@@ -298,6 +322,7 @@ namespace photon
             }
         }
 #endif
+        safe_delete(reset_handler);
         LOG_INFO("signalfd finished");
         return clear_signal_mask();
     }
diff --git a/io/test/CMakeLists.txt b/io/test/CMakeLists.txt
index 5a3ff5d6..de102709 100644
--- a/io/test/CMakeLists.txt
+++ b/io/test/CMakeLists.txt
@@ -8,6 +8,13 @@ add_executable(signalfdboom signalfdboom.cpp)
 target_link_libraries(signalfdboom PRIVATE photon_shared)
 add_test(NAME signalfdboom COMMAND $<TARGET_FILE:signalfdboom>)
 
+add_executable(test-fork test-fork.cpp)
+target_link_libraries(test-fork PRIVATE photon_shared)
+if (PHOTON_ENABLE_URING)
+    target_compile_definitions(test-fork PRIVATE PHOTON_URING=on)
+endif()
+add_test(NAME test-fork COMMAND $<TARGET_FILE:test-fork>)
+
 if (NOT APPLE)
 add_executable(test-syncio test-syncio.cpp)
 target_link_libraries(test-syncio PRIVATE photon_shared)
diff --git a/io/test/test-fork.cpp b/io/test/test-fork.cpp
new file mode 100644
index 00000000..1f8f8b8a
--- /dev/null
+++ b/io/test/test-fork.cpp
@@ -0,0 +1,325 @@
+/*
+Copyright 2022 The Photon Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#include <unistd.h>
+#include <sys/wait.h>
+#include <gtest/gtest.h>
+#include <thread>
+#include <fcntl.h>
+
+#include <photon/common/alog.h>
+#include <photon/photon.h>
+#include <photon/thread/thread.h>
+#include <photon/io/fd-events.h>
+#include <photon/io/signal.h>
+#include <photon/net/curl.h>
+#include <photon/fs/localfs.h>
+
+bool exit_flag = false;
+bool exit_normal = false;
+
+void sigint_handler(int signal = SIGINT) {
+    LOG_INFO("signal ` received, pid `", signal, getpid());
+    exit_flag = true;
+}
+
+inline int check_process_exit_stat(int &statVal, int &pid) {
+    if (WIFEXITED(statVal)) { // child exit normally
+        LOG_INFO("process with pid ` finished with code `.", pid, WEXITSTATUS(statVal));
+        return WEXITSTATUS(statVal);
+    } else {
+        if (WIFSIGNALED(statVal)) { // child terminated due to uncaptured signal
+            LOG_INFO("process with pid ` terminated due to uncaptured signal `.", pid,
+                     WTERMSIG(statVal));
+        } else if (WIFSTOPPED(statVal)) { // child terminated unexpectedly
+            LOG_INFO("process with pid ` terminated unexpectedly with signal `.", pid,
+                     WSTOPSIG(statVal));
+        } else {
+            LOG_INFO("process with pid ` terminated abnormally.", pid);
+        }
+        return -1;
+    }
+}
+
+void wait_process_end(pid_t pid) {
+    if (pid > 0) {
+        int statVal;
+        if (waitpid(pid, &statVal, 0) > 0) {
+            check_process_exit_stat(statVal, pid);
+        } else {
+            /// EINTR
+            if (EINTR == errno) {
+                LOG_INFO("process with pid ` waitpid is interrupted.", pid);
+            } else {
+                LOG_INFO("process with pid ` waitpid exception, strerror: `.", pid,
+                         strerror(errno));
+            }
+        }
+    }
+}
+
+void wait_process_end_no_hang(pid_t pid) {
+    if (pid > 0) {
+        int statVal;
+        int retry = 100;
+    again:
+        if (waitpid(pid, &statVal, WNOHANG) <= 0) {
+            if (retry--) {
+                photon::thread_usleep(50 * 1000);
+                goto again;
+            } else {
+                if (kill(pid, SIGKILL) == 0) {
+                    LOG_WARN("force kill child process with pid `", pid);
+                } else {
+                    LOG_ERROR("force kill child process with pid ` error, errno:`:`", pid, errno,
+                              strerror(errno));
+                }
+                wait_process_end(pid);
+            }
+        } else {
+            if (check_process_exit_stat(statVal, pid) == 0) {
+                exit_normal = true;
+            }
+        }
+    }
+}
+
+int fork_child_process() {
+    pid_t pid = fork();
+    if (pid < 0) {
+        LOG_ERRNO_RETURN(0, -1, "fork error");
+        return -1;
+    }
+
+    if (pid == 0) {
+        photon::block_all_signal();
+        photon::sync_signal(SIGTERM, &sigint_handler);
+
+        LOG_INFO("child hello, pid `", getpid());
+        while (!exit_flag) {
+            photon::thread_usleep(200 * 1000);
+        }
+        photon::fini();
+        LOG_INFO("child exited, pid `", getpid());
+        exit(0);
+    } else {
+        LOG_INFO("parent hello, pid `", getpid());
+        return pid;
+    }
+}
+
+int fork_parent_process(uint64_t event_engine) {
+    pid_t m_pid = fork();
+    if (m_pid < 0) {
+        LOG_ERRNO_RETURN(0, -1, "fork error");
+        return -1;
+    }
+
+    if (m_pid > 0) {
+        LOG_INFO("main hello, pid `", getpid());
+        return m_pid;
+    }
+    photon::fini();
+    photon::init(event_engine, photon::INIT_IO_LIBCURL);
+
+    photon::block_all_signal();
+    photon::sync_signal(SIGINT, &sigint_handler);
+
+    LOG_INFO("parent hello, pid `", getpid());
+    photon::thread_sleep(1);
+    auto pid = fork_child_process();
+    photon::thread_sleep(1);
+
+    int statVal;
+    if (waitpid(pid, &statVal, WNOHANG) == 0) {
+        if (kill(pid, SIGTERM) == 0) {
+            LOG_INFO("kill child process with pid `", pid);
+        } else {
+            ERRNO eno;
+            LOG_ERROR("kill child process with pid ` error, `", pid, eno);
+        }
+        wait_process_end_no_hang(pid);
+    } else {
+        check_process_exit_stat(statVal, pid);
+        LOG_ERROR("child process exit unexpected");
+    }
+
+    LOG_INFO("child process exit status `", exit_normal);
+    EXPECT_EQ(true, exit_normal);
+
+    while (!exit_flag) {
+        photon::thread_usleep(200 * 1000);
+    }
+    LOG_INFO("parent exited, pid `", getpid());
+    photon::fini();
+    exit(exit_normal ? 0 : -1);
+}
+
+TEST(ForkTest, Fork) {
+    photon::init(photon::INIT_EVENT_NONE, photon::INIT_IO_NONE);
+    DEFER(photon::fini());
+    exit_flag = false;
+    exit_normal = false;
+#if defined(__linux__)
+    auto pid = fork_parent_process(photon::INIT_EVENT_EPOLL | photon::INIT_EVENT_SIGNAL);
+#else  // macOS, FreeBSD ...
+    auto pid = fork_parent_process(photon::INIT_EVENT_DEFAULT);
+#endif
+    photon::thread_sleep(5);
+
+    int statVal;
+    if (waitpid(pid, &statVal, WNOHANG) == 0) {
+        if (kill(pid, SIGINT) == 0) {
+            LOG_INFO("kill parent process with pid `", pid);
+        } else {
+            ERRNO eno;
+            LOG_ERROR("kill parent process with pid ` error, `", pid, eno);
+        }
+        wait_process_end_no_hang(pid);
+    } else {
+        check_process_exit_stat(statVal, pid);
+        LOG_ERROR("parent process exit unexpected");
+    }
+
+    LOG_INFO("parent process exit status `", exit_normal);
+    EXPECT_EQ(true, exit_normal);
+}
+
+TEST(ForkTest, ForkInThread) {
+    photon::init(photon::INIT_EVENT_DEFAULT, photon::INIT_IO_LIBCURL);
+    DEFER(photon::fini());
+
+    int ret = -1;
+    std::thread th([&]() {
+        pid_t pid = fork();
+        ASSERT_GE(pid, 0);
+
+        if (pid == 0) {
+            LOG_INFO("child hello, pid `", getpid());
+            exit(0);
+        } else {
+            LOG_INFO("parent hello, pid `", getpid());
+            int statVal;
+            waitpid(pid, &statVal, 0);
+            ret = check_process_exit_stat(statVal, pid);
+        }
+    });
+    th.join();
+    EXPECT_EQ(0, ret);
+}
+
+TEST(ForkTest, PopenInThread) {
+    photon::init(photon::INIT_EVENT_DEFAULT, photon::INIT_IO_LIBCURL);
+    DEFER(photon::fini());
+
+    photon::semaphore sem(0);
+    auto cmd = "du -s \"/tmp\"";
+    ssize_t size = -1;
+    std::thread([&] {
+        auto f = popen(cmd, "r");
+        EXPECT_NE(nullptr, f);
+        DEFER(fclose(f));
+        fscanf(f, "%lu", &size);
+        sem.signal(1);
+        LOG_INFO("popen done");
+    }).detach();
+    sem.wait(1);
+    EXPECT_NE(-1, size);
+    LOG_INFO(VALUE(size));
+}
+
+#if defined(__linux__) && defined(PHOTON_URING)
+TEST(ForkTest, Iouring) {
+    photon::init(photon::INIT_EVENT_NONE, photon::INIT_IO_NONE);
+    DEFER(photon::fini());
+    exit_flag = false;
+    exit_normal = false;
+    auto pid = fork_parent_process(photon::INIT_EVENT_IOURING | photon::INIT_EVENT_SIGNAL);
+
+    photon::thread_sleep(5);
+
+    int statVal;
+    if (waitpid(pid, &statVal, WNOHANG) == 0) {
+        if (kill(pid, SIGINT) == 0) {
+            LOG_INFO("kill parent process with pid `", pid);
+        } else {
+            ERRNO eno;
+            LOG_ERROR("kill parent process with pid ` error, `", pid, eno);
+        }
+        wait_process_end_no_hang(pid);
+    } else {
+        check_process_exit_stat(statVal, pid);
+        LOG_ERROR("parent process exit unexpected");
+    }
+
+    LOG_INFO("parent process exit status `", exit_normal);
+    EXPECT_EQ(true, exit_normal);
+}
+#endif
+
+#if defined(__linux__)
+TEST(ForkTest, LIBAIO) {
+    photon::init(photon::INIT_EVENT_EPOLL, photon::INIT_IO_LIBAIO);
+    DEFER(photon::fini());
+
+    std::unique_ptr<photon::fs::IFileSystem> fs(
+        photon::fs::new_localfs_adaptor("/tmp/", photon::fs::ioengine_libaio));
+    std::unique_ptr<photon::fs::IFile> lf(
+        fs->open("test_local_fs_fork_parent", O_RDWR | O_CREAT, 0755));
+    void* buf = nullptr;
+    ::posix_memalign(&buf, 4096, 4096);
+    DEFER(free(buf));
+    int ret = lf->pwrite(buf, 4096, 0);
+    EXPECT_EQ(ret, 4096);
+
+    ret = -1;
+    pid_t pid = fork();
+    ASSERT_GE(pid, 0);
+
+    if (pid == 0) {
+        std::unique_ptr<photon::fs::IFileSystem> fs(
+            photon::fs::new_localfs_adaptor("/tmp/", photon::fs::ioengine_libaio));
+        std::unique_ptr<photon::fs::IFile> lf(
+            fs->open("test_local_fs_fork", O_RDWR | O_CREAT, 0755));
+        void* buf = nullptr;
+        ::posix_memalign(&buf, 4096, 4096);
+        DEFER(free(buf));
+        auto ret = lf->pwrite(buf, 4096, 0);
+        EXPECT_EQ(ret, 4096);
+        ret = lf->close();
+        photon::fini();
+        exit(ret);
+    } else {
+        int statVal;
+        waitpid(pid, &statVal, 0);
+        ret = check_process_exit_stat(statVal, pid);
+    }
+    EXPECT_EQ(0, ret);
+
+    ret = lf->pwrite(buf, 4096, 0);
+    EXPECT_EQ(ret, 4096);
+    ret = lf->close();
+    EXPECT_EQ(0, ret);
+}
+#endif
+
+int main(int argc, char **argv) {
+    set_log_output_level(0);
+
+    ::testing::InitGoogleTest(&argc, argv);
+    auto ret = RUN_ALL_TESTS();
+    if (ret) LOG_ERROR_RETURN(0, ret, VALUE(ret));
+}
diff --git a/net/curl.cpp b/net/curl.cpp
index f2eeb33f..6906f43d 100644
--- a/net/curl.cpp
+++ b/net/curl.cpp
@@ -28,6 +28,7 @@ limitations under the License.
 #include <photon/common/timeout.h>
 #include <photon/common/utility.h>
 #include <photon/net/security-context/tls-stream.h>
+#include "../io/reset_handle.h"
 
 namespace photon {
 namespace net {
@@ -159,6 +160,8 @@ class cURLLoop : public Object {
 
     void stop() { loop->stop(); }
 
+    photon::thread* loop_thread() { return loop->loop_thread(); }
+
 protected:
     EventLoop* loop;
     int cnt;
@@ -210,6 +213,17 @@ void __OpenSSLGlobalInit();
 //     curl_global_cleanup();
 // }
 
+class CurlResetHandle : public ResetHandle {
+     int reset() override {
+        LOG_INFO("reset libcurl by reset handle");
+        // interrupt g_loop by ETIMEDOUT to replace g_poller
+        if (cctx.g_loop)
+            thread_interrupt(cctx.g_loop->loop_thread(), ETIMEDOUT);
+        return 0;
+     }
+};
+static thread_local CurlResetHandle *reset_handler = nullptr;
+
 int libcurl_init(long flags, long pipelining, long maxconn) {
     if (cctx.g_loop == nullptr) {
         __OpenSSLGlobalInit();
@@ -240,6 +254,10 @@ int libcurl_init(long flags, long pipelining, long maxconn) {
 
         libcurl_set_pipelining(pipelining);
         libcurl_set_maxconnects(maxconn);
+        if (reset_handler == nullptr) {
+            reset_handler = new CurlResetHandle();
+        }
+        LOG_INFO("libcurl initialized");
     }
 
     return 0;
@@ -256,6 +274,8 @@ void libcurl_fini() {
     if (ret != CURLM_OK)
         LOG_ERROR("libcurl-multi cleanup error: ", curl_multi_strerror(ret));
     cctx.g_libcurl_multi = nullptr;
+    safe_delete(reset_handler);
+    LOG_INFO("libcurl finished");
 }
 
 std::string url_escape(const char* str) {
diff --git a/photon.cpp b/photon.cpp
index cafc770c..8780969b 100644
--- a/photon.cpp
+++ b/photon.cpp
@@ -22,6 +22,7 @@ limitations under the License.
 #ifdef ENABLE_FSTACK_DPDK
 #include "io/fstack-dpdk.h"
 #endif
+#include "io/reset_handle.h"
 #include "net/curl.h"
 #include "net/socket.h"
 #include "fs/exportfs.h"
@@ -31,6 +32,7 @@ namespace photon {
 using namespace fs;
 using namespace net;
 
+static bool reset_handle_registed = false;
 static thread_local uint64_t g_event_engine = 0, g_io_engine = 0;
 
 #define INIT_IO(name, prefix)    if (INIT_IO_##name & io_engine) { if (prefix##_init() < 0) return -1; }
@@ -74,6 +76,11 @@ int init(uint64_t event_engine, uint64_t io_engine) {
 #endif
     g_event_engine = event_engine;
     g_io_engine = io_engine;
+    if (!reset_handle_registed) {
+        pthread_atfork(nullptr, nullptr, &reset_all_handle);
+        LOG_DEBUG("reset_all_handle registed ", VALUE(getpid()));
+        reset_handle_registed = true;
+    }
     return 0;
 }
 
@@ -96,4 +103,4 @@ int fini() {
     return 0;
 }
 
-}
\ No newline at end of file
+}

From a8a55aaeb8668bf3aafa3949c9a4e8acea1e5d19 Mon Sep 17 00:00:00 2001
From: Bob Chen <beef9999@qq.com>
Date: Tue, 5 Dec 2023 15:03:59 +0800
Subject: [PATCH 28/76] RPC support graceful shutdown

---
 examples/rpc/server.h         |   2 +-
 net/basic_socket.cpp          |   2 +-
 net/socket.h                  |   1 +
 rpc/rpc.cpp                   |  94 +++++++----------
 rpc/rpc.h                     |  24 ++++-
 rpc/test/test-rpc-message.cpp |   2 +-
 rpc/test/test.cpp             | 189 +++++++++++++++++++++++++++++++---
 7 files changed, 237 insertions(+), 77 deletions(-)

diff --git a/examples/rpc/server.h b/examples/rpc/server.h
index ff6ce94e..a7348957 100644
--- a/examples/rpc/server.h
+++ b/examples/rpc/server.h
@@ -59,7 +59,7 @@ struct ExampleServer {
 
     // Serve provides handler for socket server
     int serve(photon::net::ISocketStream* stream) {
-        return skeleton->serve(stream, false);
+        return skeleton->serve(stream);
     }
 
     void term() {
diff --git a/net/basic_socket.cpp b/net/basic_socket.cpp
index bd54f8fd..8446b8dd 100644
--- a/net/basic_socket.cpp
+++ b/net/basic_socket.cpp
@@ -97,7 +97,7 @@ int connect(int fd, const struct sockaddr *addr, socklen_t addrlen,
                 if (ret < 0) return -1;
                 socklen_t n = sizeof(err);
                 ret = getsockopt(fd, SOL_SOCKET, SO_ERROR, &err, &n);
-                if (ret < 0) return -1;
+                if (unlikely(ret < 0)) return -1;
                 if (err) {
                     errno = err;
                     return -1;
diff --git a/net/socket.h b/net/socket.h
index 75592a66..c0914ef8 100644
--- a/net/socket.h
+++ b/net/socket.h
@@ -255,6 +255,7 @@ namespace net {
         using Handler = Callback<ISocketStream*>;
         virtual ISocketServer* set_handler(Handler handler) = 0;
         virtual int start_loop(bool block = false) = 0;
+        // Close the listening fd. It's the user's responsibility to close the active connections.
         virtual void terminate() = 0;
     };
 
diff --git a/rpc/rpc.cpp b/rpc/rpc.cpp
index 8082674f..d55a778e 100644
--- a/rpc/rpc.cpp
+++ b/rpc/rpc.cpp
@@ -222,7 +222,7 @@ namespace rpc {
             bool got_it;
             int* stream_serv_count;
             photon::condition_variable *stream_cv;
-            std::shared_ptr<photon::mutex> w_lock;
+            photon::mutex* w_lock;
 
             Context(SkeletonImpl* sk, IStream* s) :
                 request(sk->m_allocator), stream(s), sk(sk) { }
@@ -289,6 +289,7 @@ namespace rpc {
             }
             int response_sender(iovector* resp)
             {
+                assert(w_lock);
                 Header h;
                 h.size = (uint32_t)resp->sum();
                 h.function = header.function;
@@ -297,13 +298,11 @@ namespace rpc {
                 resp->push_front(&h, sizeof(h));
                 if (stream == nullptr)
                     LOG_ERRNO_RETURN(0, -1, "socket closed ");
-                if (w_lock) {
-                    w_lock->lock();
-                }
+
+                w_lock->lock();
                 ssize_t ret = stream->writev(resp->iovec(), resp->iovcnt());
-                if (w_lock) {
-                    w_lock->unlock();
-                }
+                w_lock->unlock();
+
                 if (ret < (ssize_t)(sizeof(h) + h.size)) {
                     stream->shutdown(ShutdownHow::ReadWrite);
                     LOG_ERRNO_RETURN(0, -1, "failed to send rpc response to stream ", stream);
@@ -314,43 +313,38 @@ namespace rpc {
         condition_variable m_cond_served;
         struct ThreadLink : public intrusive_list_node<ThreadLink>
         {
-            photon::thread* thread = nullptr;
+            photon::thread* thread = photon::CURRENT;
         };
-        intrusive_list<ThreadLink> m_list;
+        intrusive_list<ThreadLink> m_list;  // Stores the thread ID of every stream
         uint64_t m_serving_count = 0;
-        bool m_concurrent;
         bool m_running = true;
         photon::ThreadPoolBase *m_thread_pool;
-        virtual int serve(IStream* stream, bool ownership) override
+        virtual int serve(IStream* stream) override
         {
-            if (!m_running)
-                LOG_ERROR_RETURN(ENOTSUP, -1, "the skeleton has closed");
+            if (unlikely(!m_running))
+                return -1;
 
             ThreadLink node;
             m_list.push_back(&node);
             DEFER(m_list.erase(&node));
-            DEFER(if (ownership) delete stream;);
             // stream serve refcount
             int stream_serv_count = 0;
+            photon::mutex w_lock;
             photon::condition_variable stream_cv;
-            // once serve goint to exit, stream may destruct
+            // once serve exit, stream will destruct
             // make sure all requests relies on this stream are finished
             DEFER({
                 while (stream_serv_count > 0) stream_cv.wait_no_lock();
             });
             if (stream_accept_notify) stream_accept_notify(stream);
             DEFER(if (stream_close_notify) stream_close_notify(stream));
-            auto w_lock = m_concurrent ? std::make_shared<photon::mutex>() : nullptr;
-            while(m_running)
-            {
+
+            while(likely(m_running)) {
                 Context context(this, stream);
                 context.stream_serv_count = &stream_serv_count;
                 context.stream_cv = &stream_cv;
-                context.w_lock = w_lock;
-                node.thread = CURRENT;
+                context.w_lock = &w_lock;
                 int ret = context.read_request();
-                ERRNO err;
-                node.thread = nullptr;
                 if (ret < 0) {
                     // should only shutdown read, for other threads
                     // might still writing
@@ -363,16 +357,11 @@ namespace rpc {
                     }
                 }
 
-                if (!m_concurrent) {
-                    context.serve_request();
-                } else {
-                    context.got_it = false;
-                    m_thread_pool->thread_create(&async_serve, &context);
-                    // async_serve will be start, add refcount here
-                    stream_serv_count ++;
-                    while(!context.got_it)
-                        thread_yield_to(nullptr);
-                }
+                context.got_it = false;
+                m_thread_pool->thread_create(&async_serve, &context);
+                stream_serv_count ++;
+                while(!context.got_it)
+                    thread_yield();
             }
             return 0;
         }
@@ -381,48 +370,45 @@ namespace rpc {
             auto ctx = (Context*)args_;
             Context context(std::move(*ctx));
             ctx->got_it = true;
-            thread_yield_to(nullptr);
+            thread_yield();
             context.serve_request();
             // serve done, here reduce refcount
             (*ctx->stream_serv_count) --;
             ctx->stream_cv->notify_all();
             return nullptr;
         }
-        virtual int shutdown_no_wait() override {
-            photon::thread_create11(&SkeletonImpl::shutdown, this);
+        virtual int shutdown(bool no_more_requests) override {
+            m_running = !no_more_requests;
+            while (m_list) {
+                auto th = m_list.front()->thread;
+                thread_enable_join(th);
+                if (no_more_requests) {
+                    thread_interrupt(th);
+                }
+                // Wait all streams destructed. Their attached RPC requests are finished as well.
+                thread_join((join_handle*) th);
+            }
             return 0;
         }
-        virtual int shutdown() override
-        {
+        int shutdown_no_wait() override {
             m_running = false;
-            for (const auto& x: m_list)
-                if (x->thread)
-                    thread_interrupt(x->thread);
-            // it should confirm that all threads are finished
-            // or m_list may not destruct correctly
-            while (m_serving_count > 0) {
-                // means shutdown called by rpc serve, should return to give chance to shutdown
-                if ((m_serving_count == 1) && (m_list.front()->thread == nullptr))
-                    return 0;
-                m_cond_served.wait_no_lock();
+            for (auto* each: m_list) {
+                thread_interrupt(each->thread);
             }
-            while (!m_list.empty())
-                thread_usleep(1000);
             return 0;
         }
         virtual ~SkeletonImpl() {
-            shutdown();
+            shutdown(true);
             photon::delete_thread_pool(m_thread_pool);
         }
-        explicit SkeletonImpl(bool concurrent = true, uint32_t pool_size = 128)
-            : m_concurrent(concurrent),
+        explicit SkeletonImpl(uint32_t pool_size = 128) :
               m_thread_pool(photon::new_thread_pool(pool_size)) {
             m_thread_pool->enable_autoscale();
         }
     };
-    Skeleton* new_skeleton(bool concurrent, uint32_t pool_size)
+    Skeleton* new_skeleton(uint32_t pool_size)
     {
-        return new SkeletonImpl(concurrent, pool_size);
+        return new SkeletonImpl(pool_size);
     }
 
     class StubPoolImpl : public StubPool {
diff --git a/rpc/rpc.h b/rpc/rpc.h
index 0b2b19fa..eb195342 100644
--- a/rpc/rpc.h
+++ b/rpc/rpc.h
@@ -161,16 +161,25 @@ namespace rpc
         virtual int set_close_notify(Notifier notifier) = 0;
 
         // can be invoked concurrently by multiple threads
-        // if have the ownership of stream, `serve` will delete it before exit
-        virtual int serve(IStream* stream, bool ownership_stream = false) = 0;
+        virtual int serve(IStream* stream) = 0;
+
+        __attribute__((deprecated))
+        int serve(IStream* stream, bool /*ownership_stream*/) {
+            return serve(stream);
+        }
 
         // set the allocator to allocate memory for recving responses
         // the default allocator is defined in iovector.h/cpp
         virtual void set_allocator(IOAlloc allocation) = 0;
 
-        virtual int shutdown_no_wait() = 0;
+        /**
+         * @brief Shutdown the rpc server from outside.
+         * @warning DO NOT invoke this function within the RPC request.
+         *          You should create a thread to invoke it, or just use shutdown_no_wait.
+         */
+        virtual int shutdown(bool no_more_requests = true) = 0;
 
-        virtual int shutdown() = 0;
+        virtual int shutdown_no_wait() = 0;
 
         template <class ServerClass>
         int register_service(ServerClass* obj)
@@ -238,7 +247,12 @@ namespace rpc
     extern "C" StubPool* new_uds_stub_pool(const char* path, uint64_t expiration,
                                 uint64_t connect_timeout,
                                 uint64_t rpc_timeout);
-    extern "C" Skeleton* new_skeleton(bool concurrent = true, uint32_t pool_size = 128);
+    extern "C" Skeleton* new_skeleton(uint32_t pool_size = 128);
+
+    __attribute__((deprecated))
+    inline Skeleton* new_skeleton(bool /*concurrent*/, uint32_t pool_size = 128) {
+        return new_skeleton(pool_size);
+    }
 
     struct __example__operation1__   // defination of operator
     {
diff --git a/rpc/test/test-rpc-message.cpp b/rpc/test/test-rpc-message.cpp
index 6597e0cd..ef188830 100644
--- a/rpc/test/test-rpc-message.cpp
+++ b/rpc/test/test-rpc-message.cpp
@@ -156,7 +156,7 @@ class TestRPCServer {
     }
 
     int serve(photon::net::ISocketStream* stream) {
-        return skeleton->serve(stream, false);
+        return skeleton->serve(stream);
     }
 
     int run() {
diff --git a/rpc/test/test.cpp b/rpc/test/test.cpp
index 3b06f18f..29f85bda 100644
--- a/rpc/test/test.cpp
+++ b/rpc/test/test.cpp
@@ -16,16 +16,30 @@ limitations under the License.
 
 #include "../../rpc/rpc.cpp"
 #include <memory>
+#include <chrono>
 #include <gtest/gtest.h>
 #include <photon/thread/thread.h>
 #include <photon/common/memory-stream/memory-stream.h>
 #include <photon/common/utility.h>
 #include <photon/common/alog-stdstring.h>
 #include <photon/net/socket.h>
+#include <photon/photon.h>
+#include "../../test/ci-tools.h"
+
 using namespace std;
 using namespace photon;
 using namespace rpc;
 
+class RpcTest : public testing::Test {
+public:
+    void SetUp() override {
+        GTEST_ASSERT_EQ(0, photon::init(ci_ev_engine, photon::INIT_IO_NONE));
+    }
+    void TearDown() override {
+        photon::fini();
+    }
+};
+
 std::string S = "1234567890";
 struct Args
 {
@@ -38,15 +52,10 @@ struct Args
     }
     void verify()
     {
-        LOG_DEBUG(VALUE(a));
         EXPECT_EQ(a, 123);
-        LOG_DEBUG(VALUE(b));
         EXPECT_EQ(b, 123);
-        LOG_DEBUG(VALUE(c));
         EXPECT_EQ(c, 123);
-        LOG_DEBUG(VALUE(d));
         EXPECT_EQ(d, 123);
-        LOG_DEBUG(VALUE(s));
         EXPECT_EQ(s, S);
     }
     uint64_t serialize(iovector& iov)
@@ -142,13 +151,13 @@ int server_exit_function(void* instance, iovector* request, rpc::Skeleton::Respo
 
 bool skeleton_exited;
 photon::condition_variable skeleton_exit;
-rpc::Skeleton* g_sk;
+
 void* rpc_skeleton(void* args)
 {
-    skeleton_exited = false;
     auto s = (IStream*)args;
     auto sk = new_skeleton();
-    g_sk = sk;
+    DEFER(delete sk);
+
     sk->add_function(FID, rpc::Skeleton::Function((void*)123, &server_function));
     sk->add_function(-1,  rpc::Skeleton::Function(sk, &server_exit_function));
     sk->serve(s);
@@ -171,7 +180,7 @@ void do_call(StubImpl& stub, uint64_t function)
     EXPECT_EQ(memcmp(STR, resp_iov.iov.back().iov_base, LEN(STR)), 0);
 }
 
-TEST(rpc, call)
+TEST_F(RpcTest, call)
 {
     unique_ptr<DuplexMemoryStream> ds( new_duplex_memory_stream(16) );
     thread_create(&rpc_skeleton, ds->endpoint_a);
@@ -207,13 +216,14 @@ void* do_concurrent_call_shut(void* arg)
     return nullptr;
 }
 
-TEST(rpc, concurrent)
+TEST_F(RpcTest, concurrent)
 {
 //    log_output_level = 1;
     LOG_INFO("Creating 1,000 threads, each doing 1,000 RPC calls");
     // ds will be destruct just after function returned
     // but server will not
     // therefore, it will cause assert when destruction
+    skeleton_exited = false;
     unique_ptr<DuplexMemoryStream> ds( new_duplex_memory_stream(16) );
     thread_create(&rpc_skeleton, ds->endpoint_a);
 
@@ -232,7 +242,6 @@ TEST(rpc, concurrent)
     ds->close();
     if (!skeleton_exited)
         skeleton_exit.wait_no_lock();
-    log_output_level = 0;
 }
 
 void do_call_timeout(StubImpl& stub, uint64_t function)
@@ -280,10 +289,9 @@ int server_function_timeout(void* instance, iovector* request, rpc::Skeleton::Re
 
 void* rpc_skeleton_timeout(void* args)
 {
-    skeleton_exited = false;
     auto s = (IStream*)args;
     auto sk = new_skeleton();
-    g_sk = sk;
+    DEFER(delete sk);
     sk->add_function(FID, rpc::Skeleton::Function((void*)123, &server_function_timeout));
     sk->add_function(-1,  rpc::Skeleton::Function(sk, &server_exit_function));
     sk->serve(s);
@@ -293,13 +301,14 @@ void* rpc_skeleton_timeout(void* args)
     return nullptr;
 }
 
-TEST(rpc, timeout) {
+TEST_F(RpcTest, timeout) {
     LOG_INFO("Creating 1,000 threads, each doing 1,000 RPC calls");
     // ds will be destruct just after function returned
     // but server will not
     // therefore, it will cause assert when destruction
     unique_ptr<DuplexMemoryStream> ds( new_duplex_memory_stream(655360) );
 
+    skeleton_exited = false;
     thread_create(&rpc_skeleton_timeout, ds->endpoint_a);
 
     LOG_DEBUG("asdf1");
@@ -320,9 +329,159 @@ TEST(rpc, timeout) {
     log_output_level = 0;
 }
 
+class RpcServer {
+public:
+    RpcServer(Skeleton* skeleton, net::ISocketServer* socket) : m_socket(socket), m_skeleton(skeleton) {
+        m_skeleton->register_service<Operation>(this);
+        m_socket->set_handler({this, &RpcServer::serve});
+    }
+    struct Operation {
+        const static uint32_t IID = 0x1;
+        const static uint32_t FID = 0x2;
+        struct Request : public photon::rpc::Message {
+            int code = 0;
+            PROCESS_FIELDS(code);
+        };
+        struct Response : public photon::rpc::Message {
+            int code = 0;
+            PROCESS_FIELDS(code);
+        };
+    };
+    int do_rpc_service(Operation::Request* req, Operation::Response* resp, IOVector* iov, IStream* stream) {
+        resp->code = req->code;
+        return 0;
+    }
+    int serve(photon::net::ISocketStream* stream) {
+        return m_skeleton->serve(stream);
+    }
+    int run() {
+        m_socket->setsockopt(SOL_SOCKET, SO_REUSEPORT, 1);
+        if (m_socket->bind(9527, net::IPAddr::V6Any()) != 0)
+            LOG_ERRNO_RETURN(0, -1, "bind failed");
+        if (m_socket->listen() != 0)
+            LOG_ERRNO_RETURN(0, -1, "listen failed");
+        return m_socket->start_loop(false);
+    }
+    net::ISocketServer* m_socket;
+    Skeleton* m_skeleton;
+};
+
+static int do_call_2(Stub* stub) {
+    RpcServer::Operation::Request req;
+    RpcServer::Operation::Response resp;
+    return stub->call<RpcServer::Operation>(req, resp);
+}
+
+TEST_F(RpcTest, shutdown) {
+    auto socket_server = photon::net::new_tcp_socket_server_ipv6();
+    GTEST_ASSERT_NE(nullptr, socket_server);
+    DEFER(delete socket_server);
+    auto sk = photon::rpc::new_skeleton();
+    GTEST_ASSERT_NE(nullptr, sk);
+    DEFER(delete sk);
+
+    RpcServer rpc_server(sk, socket_server);
+    GTEST_ASSERT_EQ(0, rpc_server.run());
+
+    auto pool = photon::rpc::new_stub_pool(-1, -1, -1);
+    DEFER(delete pool);
+
+    photon::net::EndPoint ep(net::IPAddr::V4Loopback(), 9527);
+    auto stub = pool->get_stub(ep, false);
+    ASSERT_NE(nullptr, stub);
+    DEFER(pool->put_stub(ep, true));
+
+    photon::thread_create11([&]{
+        photon::thread_sleep(1);
+        sk->shutdown();
+        delete sk;
+        sk = nullptr;
+    });
+
+    auto start = std::chrono::steady_clock::now();
+    while (true) {
+        int ret = do_call_2(stub);
+        if (ret < 0) {
+            GTEST_ASSERT_EQ(ECONNRESET, errno);
+            break;
+        }
+    }
+    auto end = std::chrono::steady_clock::now();
+    auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
+    GTEST_ASSERT_GT(duration, 900);
+    GTEST_ASSERT_LT(duration, 1100);
+}
+
+TEST_F(RpcTest, passive_shutdown) {
+    auto socket_server = photon::net::new_tcp_socket_server_ipv6();
+    GTEST_ASSERT_NE(nullptr, socket_server);
+    DEFER(delete socket_server);
+    auto sk = photon::rpc::new_skeleton();
+    GTEST_ASSERT_NE(nullptr, sk);
+    DEFER(delete sk);
+
+    RpcServer rpc_server(sk, socket_server);
+    GTEST_ASSERT_EQ(0, rpc_server.run());
+
+    photon::net::EndPoint ep(net::IPAddr::V4Loopback(), 9527);
+
+    photon::thread_create11([&]{
+        // Should always succeed in 3 seconds
+        auto pool = photon::rpc::new_stub_pool(-1, -1, -1);
+        DEFER(delete pool);
+        auto stub = pool->get_stub(ep, false);
+        if (!stub) abort();
+        DEFER(pool->put_stub(ep, true));
+        for (int i = 0 ; i < 30; ++i) {
+            int ret = do_call_2(stub);
+            if (ret < 0) {
+                LOG_ERROR(VALUE(ret));
+                abort();
+            }
+            photon::thread_usleep(100'000);
+        }
+    });
+
+    photon::thread_create11([&]{
+        photon::thread_sleep(2);
+        // Should get connection refused after 2 seconds. Because socket closed listen fd at 1 second.
+        auto pool = photon::rpc::new_stub_pool(-1, -1, -1);
+        DEFER(delete pool);
+        auto stub = pool->get_stub(ep, false);
+        if (stub) {
+            LOG_ERROR("should not get stub");
+            abort();
+        }
+        if (errno != ECONNREFUSED) {
+            LOG_ERROR(ERRNO());
+            abort();
+        }
+    });
+
+    auto start = std::chrono::steady_clock::now();
+
+    photon::thread_sleep(1);
+    socket_server->terminate();
+    delete socket_server;
+    socket_server = nullptr;
+
+    LOG_INFO("begin passive shutdown");
+    sk->shutdown(false);
+    LOG_INFO("end passive shutdown");
+    delete sk;
+    sk = nullptr;
+
+    auto end = std::chrono::steady_clock::now();
+    auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
+
+    // The passive shutdown took 3 seconds, until client closed the connection
+    GTEST_ASSERT_GT(duration, 2900);
+    GTEST_ASSERT_LT(duration, 3200);
+}
+
 int main(int argc, char** arg)
 {
-    ::photon::vcpu_init();
+    ci_parse_env();
     ::testing::InitGoogleTest(&argc, arg);
     return RUN_ALL_TESTS();
 }

From 4488dc4830a22baa1af3d5d5c7e840721d4a5ea9 Mon Sep 17 00:00:00 2001
From: Huiba Li <lihuiba@gmail.com>
Date: Thu, 14 Dec 2023 16:00:19 +0800
Subject: [PATCH 29/76] Sequential mutex (#294)

seq_mutex
---
 thread/thread.cpp |  8 ++++----
 thread/thread.h   | 14 +++++++++++---
 2 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/thread/thread.cpp b/thread/thread.cpp
index bde8ab21..f9f1a67e 100644
--- a/thread/thread.cpp
+++ b/thread/thread.cpp
@@ -1544,12 +1544,12 @@ R"(
         auto splock = (spinlock*)s_;
         splock->unlock();
     }
-    int mutex::lock(uint64_t timeout)
-    {
-        for (int tries = 0; tries < MaxTries; ++tries) {
+    int mutex::lock(uint64_t timeout) {
+        if (try_lock() == 0) return 0;
+        for (auto re = retries; re; --re) {
+            thread_yield();
             if (try_lock() == 0)
                 return 0;
-            thread_yield();
         }
         splock.lock();
         if (try_lock() == 0) {
diff --git a/thread/thread.h b/thread/thread.h
index 69f64dce..9bb35347 100644
--- a/thread/thread.h
+++ b/thread/thread.h
@@ -226,8 +226,9 @@ namespace photon
     class mutex : protected waitq
     {
     public:
-        int lock(uint64_t timeout = -1);        // threads are guaranteed to get the lock
-        int try_lock();                         // in FIFO order, when there's contention
+        mutex(uint16_t max_retries = 100) : retries(max_retries) { }
+        int lock(uint64_t timeout = -1);
+        int try_lock();
         void unlock();
         ~mutex()
         {
@@ -235,13 +236,20 @@ namespace photon
         }
 
     protected:
-        static constexpr const int MaxTries = 100;
         std::atomic<thread*> owner{nullptr};
+        uint16_t retries;
         spinlock splock;
     };
 
+    class seq_mutex : protected mutex {
+    public:
+        // threads are guaranteed to get the lock in sequental order (FIFO)
+        seq_mutex() : mutex(0) { }
+    };
+
     class recursive_mutex : protected mutex {
     public:
+        using mutex::mutex;
         int lock(uint64_t timeout = -1);
         int try_lock();
         void unlock();

From 9245c32aba1ec9fdeccb806a7aa27a1d55ab044c Mon Sep 17 00:00:00 2001
From: Bob Chen <beef9999@qq.com>
Date: Fri, 15 Dec 2023 10:36:28 +0800
Subject: [PATCH 30/76] exit(0) requires stdlib.h (#303)

---
 io/signal.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/io/signal.cpp b/io/signal.cpp
index 63b1a751..7208b811 100644
--- a/io/signal.cpp
+++ b/io/signal.cpp
@@ -15,6 +15,7 @@ limitations under the License.
 */
 
 #include "signal.h"
+#include <cstdlib>
 #include <unistd.h>
 #include <pthread.h>
 #ifdef __APPLE__

From 0dbb0eb829d7334857fc48356dbf5cfd9fca3360 Mon Sep 17 00:00:00 2001
From: yuchen0cc <91253032+yuchen0cc@users.noreply.github.com>
Date: Fri, 15 Dec 2023 10:41:03 +0800
Subject: [PATCH 31/76] fix e2fs include dir name (#301)

Signed-off-by: yuchen.cc <yuchen.cc@alibaba-inc.com>
---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 9f86eb6e..b92e2c5d 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -212,7 +212,7 @@ if (PHOTON_ENABLE_FSTACK_DPDK)
     target_include_directories(photon_obj PRIVATE ${FSTACK_INCLUDE_DIRS})
 endif()
 if (PHOTON_ENABLE_EXTFS)
-    target_include_directories(photon_obj PRIVATE ${LIBE2FS_INCLUDE_DIRS})
+    target_include_directories(photon_obj PRIVATE ${E2FS_INCLUDE_DIRS})
 endif()
 if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 8.0)
     # This option is enabled by default after -std=c++17

From e9dedfb76d118daf4a980b9ed88267b48f7663d3 Mon Sep 17 00:00:00 2001
From: Bob Chen <beef9999@qq.com>
Date: Sun, 17 Dec 2023 20:32:05 +0800
Subject: [PATCH 32/76] minor fix for std-compat (#307)

* minor fix for std-compat

* rpc::string add empty method

* set io_uring eventfd non-blocking
---
 examples/sync-primitive/sync-primitive.cpp | 7 +++----
 io/iouring-wrapper.cpp                     | 8 +++-----
 rpc/serialize.h                            | 1 +
 thread/std-compat.h                        | 3 ++-
 4 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/examples/sync-primitive/sync-primitive.cpp b/examples/sync-primitive/sync-primitive.cpp
index 61977b9a..173e8ea4 100644
--- a/examples/sync-primitive/sync-primitive.cpp
+++ b/examples/sync-primitive/sync-primitive.cpp
@@ -43,7 +43,7 @@ struct Message {
     std::chrono::time_point<std::chrono::steady_clock> start;
 };
 
-static const int num_producers = 16, num_consumers = 16;
+static const int num_producers = 8, num_consumers = 8;
 static LockfreeMPMCRingQueue<Message*, 1024 * 1024> ring;
 static std::atomic<uint64_t> qps{0}, latency{0};
 
@@ -90,7 +90,7 @@ int main() {
                 Message message;
                 ring.push(&message);
                 {
-                    std::unique_lock l(message.mu);
+                    std::unique_lock<std::mutex> l(message.mu);
                     message.cv.wait(l, [&] { return message.done; });
                 }
                 auto end = std::chrono::steady_clock::now();
@@ -117,7 +117,7 @@ int main() {
                 m->sem.signal(1);
 #else
                 {
-                    std::unique_lock l(m->mu);
+                    std::unique_lock<std::mutex> l(m->mu);
                     m->done = true;
                     m->cv.notify_one();
                 }
@@ -128,7 +128,6 @@ int main() {
 
     // Show QPS and latency
     photon::thread_create11([&] {
-        photon::init(photon::INIT_EVENT_DEFAULT, photon::INIT_IO_NONE);
         while (true) {
             photon::thread_sleep(1);
             auto prev_qps = qps.exchange(0, std::memory_order_seq_cst);
diff --git a/io/iouring-wrapper.cpp b/io/iouring-wrapper.cpp
index d0e29059..77579daa 100644
--- a/io/iouring-wrapper.cpp
+++ b/io/iouring-wrapper.cpp
@@ -133,7 +133,7 @@ class iouringEngine : public MasterEventEngine, public CascadingEventEngine, pub
             }
         }
 
-        m_eventfd = eventfd(0, EFD_CLOEXEC);
+        m_eventfd = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK);
         if (m_eventfd < 0) {
             LOG_ERRNO_RETURN(0, -1, "iouring: failed to create eventfd");
         }
@@ -285,10 +285,8 @@ class iouringEngine : public MasterEventEngine, public CascadingEventEngine, pub
         if (ret < 0) {
             return errno == ETIMEDOUT ? 0 : -1;
         }
-        uint64_t value = 0;
-        if (eventfd_read(m_eventfd, &value)) {
-            LOG_ERROR("iouring: error reading cascading event fd, `", ERRNO());
-        }
+        uint64_t value;
+        eventfd_read(m_eventfd, &value);
 
         // Reap events
         size_t num = 0;
diff --git a/rpc/serialize.h b/rpc/serialize.h
index 0aa327bd..df5763db 100644
--- a/rpc/serialize.h
+++ b/rpc/serialize.h
@@ -71,6 +71,7 @@ namespace rpc
         const T& operator[](long i) const { return ((T*)_ptr)[i]; }
         const T& front() const    { return (*this)[0]; }
         const T& back() const     { return (*this)[(long)size() - 1]; }
+        bool empty() const { return _len == 0; }
         void assign(const T* x, size_t size) { buffer::assign(x, sizeof(*x) * size); }
         void assign(const std::vector<T>& vec) { assign(&vec[0], vec.size()); }
 
diff --git a/thread/std-compat.h b/thread/std-compat.h
index 91c56b5d..a71a3b18 100644
--- a/thread/std-compat.h
+++ b/thread/std-compat.h
@@ -222,9 +222,10 @@ class unique_lock {
     }
 
     Mutex* release() noexcept {
+        auto* mu = m_mutex;
         m_mutex = nullptr;
         m_owns = false;
-        return m_mutex;
+        return mu;
     }
 
     Mutex* mutex() const noexcept {

From 3e2d213dcd3da2154d256a69c4aee2cc03cec14e Mon Sep 17 00:00:00 2001
From: Bob Chen <beef9999@qq.com>
Date: Sun, 24 Dec 2023 08:35:14 +0800
Subject: [PATCH 33/76] more robust cmake script (#314)

---
 CMake/build-from-src.cmake | 28 +++++++++++++++++-----------
 1 file changed, 17 insertions(+), 11 deletions(-)

diff --git a/CMake/build-from-src.cmake b/CMake/build-from-src.cmake
index 646dc9f3..dad13278 100644
--- a/CMake/build-from-src.cmake
+++ b/CMake/build-from-src.cmake
@@ -12,7 +12,7 @@ function(build_from_src [dep])
                 URL_MD5 605237f35de238dfacc83bcae406d95d
                 BUILD_IN_SOURCE ON
                 CONFIGURE_COMMAND ""
-                BUILD_COMMAND make prefix=${BINARY_DIR} install -j
+                BUILD_COMMAND $(MAKE) prefix=${BINARY_DIR} install
                 INSTALL_COMMAND ""
         )
         set(AIO_INCLUDE_DIRS ${BINARY_DIR}/include PARENT_SCOPE)
@@ -26,8 +26,8 @@ function(build_from_src [dep])
                 URL_MD5 9b8aa094c4e5765dabf4da391f00d15c
                 BUILD_IN_SOURCE ON
                 CONFIGURE_COMMAND CFLAGS=-fPIC ./configure --prefix=${BINARY_DIR} --static
-                BUILD_COMMAND make -j
-                INSTALL_COMMAND make install
+                BUILD_COMMAND $(MAKE)
+                INSTALL_COMMAND $(MAKE) install
         )
         set(ZLIB_INCLUDE_DIRS ${BINARY_DIR}/include PARENT_SCOPE)
         set(ZLIB_LIBRARIES ${BINARY_DIR}/lib/libz.a PARENT_SCOPE)
@@ -40,8 +40,8 @@ function(build_from_src [dep])
                 URL_MD5 2e8c3c23795415475654346484f5c4b8
                 BUILD_IN_SOURCE ON
                 CONFIGURE_COMMAND ./configure --prefix=${BINARY_DIR}
-                BUILD_COMMAND V=1 CFLAGS=-fPIC make -C src
-                INSTALL_COMMAND make install
+                BUILD_COMMAND V=1 CFLAGS=-fPIC $(MAKE) -C src
+                INSTALL_COMMAND $(MAKE) install
         )
         set(URING_INCLUDE_DIRS ${BINARY_DIR}/include PARENT_SCOPE)
         set(URING_LIBRARIES ${BINARY_DIR}/lib/liburing.a PARENT_SCOPE)
@@ -83,10 +83,12 @@ function(build_from_src [dep])
                 URL_MD5 bad68bb6bd9908da75e2c8dedc536b29
                 BUILD_IN_SOURCE ON
                 CONFIGURE_COMMAND ./config -fPIC --prefix=${BINARY_DIR} --openssldir=${BINARY_DIR} shared
-                BUILD_COMMAND make -j ${NumCPU}
-                INSTALL_COMMAND make install
+                BUILD_COMMAND make -j 1  # https://github.com/openssl/openssl/issues/5762#issuecomment-376622684
+                INSTALL_COMMAND $(MAKE) install
+                LOG_CONFIGURE ON
+                LOG_BUILD ON
+                LOG_INSTALL ON
         )
-        ExternalProject_Get_Property(openssl SOURCE_DIR)
         set(OPENSSL_ROOT_DIR ${BINARY_DIR} PARENT_SCOPE)
         set(OPENSSL_INCLUDE_DIRS ${BINARY_DIR}/include PARENT_SCOPE)
         set(OPENSSL_LIBRARIES ${BINARY_DIR}/lib/libssl.a ${BINARY_DIR}/lib/libcrypto.a PARENT_SCOPE)
@@ -101,7 +103,7 @@ function(build_from_src [dep])
                 URL ${PHOTON_CURL_SOURCE}
                 URL_MD5 a66270f11e3fbfad709600bbd1686704
                 BUILD_IN_SOURCE ON
-                CONFIGURE_COMMAND autoreconf -i && ./configure --with-ssl=${OPENSSL_ROOT_DIR}
+                CONFIGURE_COMMAND autoreconf -i COMMAND ./configure --with-ssl=${OPENSSL_ROOT_DIR}
                     --without-libssh2 --enable-static --enable-shared=no --enable-optimize
                     --disable-manual --without-libidn
                     --disable-ftp --disable-file --disable-ldap --disable-ldaps
@@ -109,8 +111,12 @@ function(build_from_src [dep])
                     --disable-pop3 --disable-imap --disable-smb --disable-smtp
                     --disable-gopher --without-nghttp2 --enable-http --disable-verbose
                     --with-pic=PIC --prefix=${BINARY_DIR}
-                BUILD_COMMAND make -j ${NumCPU}
-                INSTALL_COMMAND make install
+                BUILD_COMMAND $(MAKE)
+                INSTALL_COMMAND $(MAKE) install
+                DEPENDS openssl
+                LOG_CONFIGURE ON
+                LOG_BUILD ON
+                LOG_INSTALL ON
         )
         set(CURL_INCLUDE_DIRS ${BINARY_DIR}/include PARENT_SCOPE)
         set(CURL_LIBRARIES ${BINARY_DIR}/lib/libcurl.a PARENT_SCOPE)

From 7cab63e26538e0f54e9c56b406115f86f3681816 Mon Sep 17 00:00:00 2001
From: HanLee13 <347363127@qq.com>
Date: Mon, 25 Dec 2023 10:19:38 +0800
Subject: [PATCH 34/76] Support ipv6 in http client (#310)

* Support ipv6 in http client.

- refactor KernelSocketClient.
- apply dns load balance in DefaultResolver.
- fix bug in DefaultResolver when timeout happen.

* Add big writes/uid/gid/du support to FUSE

- In Fuse 2.x, without big writes, write io size is limited to 4k. Turing it on can enlarge write io size to 128K at maximum.
- Support du by adding st_blocks to xmp_getattr result.
- Add default uid/gid to xmp_getattr to avoid permission problems.

* Fix bug when resolving thread timeout

* Fix unittest for default resovler
---
 fs/fuse_adaptor.cpp                    | 20 ++++++--
 net/http/client.cpp                    |  2 +-
 net/http/test/client_function_test.cpp | 10 ++++
 net/kernel_socket.cpp                  | 51 +++++++++---------
 net/test/test.cpp                      |  4 +-
 net/utils.cpp                          | 71 +++++++++++++++++---------
 net/utils.h                            |  3 +-
 thread/list.h                          | 51 ++++++++++++++++++
 8 files changed, 152 insertions(+), 60 deletions(-)

diff --git a/fs/fuse_adaptor.cpp b/fs/fuse_adaptor.cpp
index 655eb55a..f6821712 100644
--- a/fs/fuse_adaptor.cpp
+++ b/fs/fuse_adaptor.cpp
@@ -80,13 +80,21 @@ namespace photon {
 namespace fs{
 
 static IFileSystem* fs = nullptr;
+static uid_t cuid = 0, cgid = 0;
 
 #define CHECK_FS() if (!fs) return -EFAULT;
 
 static void* xmp_init(struct fuse_conn_info *conn)
 {
     REPORT_PERF(xmp_init, 1)
-    (void) conn;
+    cuid  = geteuid();
+    cgid  = getegid();
+    if ((unsigned int)conn->capable & FUSE_CAP_ATOMIC_O_TRUNC) {
+        conn->want |= FUSE_CAP_ATOMIC_O_TRUNC;
+    }
+    if ((unsigned int)conn->capable & FUSE_CAP_BIG_WRITES) {
+        conn->want |= FUSE_CAP_BIG_WRITES;
+    }
     return NULL;
 }
 
@@ -102,9 +110,13 @@ static int xmp_getattr(const char *path, struct stat *stbuf)
     CHECK_FS();
     errno = 0;
     int res = fs->lstat(path, stbuf);
-    if(res) \
-        LOG_ERROR_RETURN(0, errno ? -errno : res, VALUE(path));
-    errno = 0;
+    if(res) return -errno;
+    stbuf->st_blocks = stbuf->st_size / 512 + 1;  // du relies on this
+    stbuf->st_blksize = 4096;
+    // use current user/group when backendfs doesn't support uid/gid
+    if (stbuf->st_uid == 0) stbuf->st_uid = cuid;
+    if (stbuf->st_gid == 0) stbuf->st_gid = cgid;
+    if (stbuf->st_nlink == 0) stbuf->st_nlink = 1;
     return 0;
 }
 
diff --git a/net/http/client.cpp b/net/http/client.cpp
index 59f08faf..dc388f74 100644
--- a/net/http/client.cpp
+++ b/net/http/client.cpp
@@ -91,7 +91,7 @@ ISocketStream* PooledDialer::dial(std::string_view host, uint16_t port, bool sec
     if (ipaddr.undefined()) LOG_DEBUG("No connectable resolve result");
     // When failed, remove resolved result from dns cache so that following retries can try
     // different ips.
-    resolver->discard_cache(strhost.c_str());
+    resolver->discard_cache(strhost.c_str(), ipaddr);
     return nullptr;
 }
 
diff --git a/net/http/test/client_function_test.cpp b/net/http/test/client_function_test.cpp
index 1fb7cf9f..8e0b48cd 100644
--- a/net/http/test/client_function_test.cpp
+++ b/net/http/test/client_function_test.cpp
@@ -513,6 +513,16 @@ TEST(http_client, partial_body) {
     EXPECT_EQ(true, buf == "http_clien");
 }
 
+TEST(DISABLED_http_client, ipv6) {  // make sure runing in a ipv6-ready environment
+    auto client = new_http_client();
+    DEFER(delete client);
+    // here is an ipv6-only website
+    auto op = client->new_operation(Verb::GET, "http://test6.ustc.edu.cn");
+    DEFER(delete op);
+    op->call();
+    EXPECT_EQ(200, op->resp.status_code());
+}
+
 TEST(url, url_escape_unescape) {
     EXPECT_EQ(
         url_escape("?a=x:b&b=cd&c= feg&d=2/1[+]@alibaba.com&e='!bad';"),
diff --git a/net/kernel_socket.cpp b/net/kernel_socket.cpp
index 59749dd8..08715afa 100644
--- a/net/kernel_socket.cpp
+++ b/net/kernel_socket.cpp
@@ -191,9 +191,7 @@ class KernelSocketStream : public SocketStreamBase {
 
 class KernelSocketClient : public SocketClientBase {
 public:
-    KernelSocketClient(int socket_family, bool nonblocking) :
-            m_socket_family(socket_family),
-            m_nonblocking(nonblocking) {}
+    KernelSocketClient(bool nonblocking) : m_nonblocking(nonblocking) {}
 
     ISocketStream* connect(const char* path, size_t count) override {
         struct sockaddr_un addr_un;
@@ -211,11 +209,10 @@ class KernelSocketClient : public SocketClientBase {
     }
 
 protected:
-    int m_socket_family;
     bool m_nonblocking;
 
-    virtual KernelSocketStream* create_stream() {
-        return new KernelSocketStream(m_socket_family, m_nonblocking);
+    virtual KernelSocketStream* create_stream(int socket_family) {
+        return new KernelSocketStream(socket_family, m_nonblocking);
     }
 
     virtual int fd_connect(int fd, const sockaddr* remote, socklen_t addrlen) {
@@ -224,7 +221,7 @@ class KernelSocketClient : public SocketClientBase {
 
     ISocketStream* do_connect(const sockaddr* remote, socklen_t len_remote,
                               const sockaddr* local = nullptr, socklen_t len_local = 0) {
-        auto stream = create_stream();
+        auto stream = create_stream(remote->sa_family);
         auto deleter = [&](KernelSocketStream*) {
             auto errno_backup = errno;
             delete stream;
@@ -504,8 +501,8 @@ class ZeroCopySocketClient : public KernelSocketClient {
 public:
     using KernelSocketClient::KernelSocketClient;
 protected:
-    KernelSocketStream* create_stream() override {
-        return new ZeroCopySocketStream(m_socket_family, m_nonblocking);
+    KernelSocketStream* create_stream(int socket_family) override {
+        return new ZeroCopySocketStream(socket_family, m_nonblocking);
     }
 };
 
@@ -544,8 +541,8 @@ class IouringSocketClient : public KernelSocketClient {
     using KernelSocketClient::KernelSocketClient;
 
 protected:
-    KernelSocketStream* create_stream() override {
-        return new IouringSocketStream(m_socket_family, m_nonblocking);
+    KernelSocketStream* create_stream(int socket_family) override {
+        return new IouringSocketStream(socket_family, m_nonblocking);
     }
 
     int fd_connect(int fd, const sockaddr* remote, socklen_t addrlen) override {
@@ -614,8 +611,8 @@ class IouringFixedFileSocketClient : public IouringSocketClient {
 protected:
     using IouringSocketClient::IouringSocketClient;
 
-    KernelSocketStream* create_stream() override {
-        return new IouringFixedFileSocketStream(m_socket_family, m_nonblocking);
+    KernelSocketStream* create_stream(int socket_family) override {
+        return new IouringFixedFileSocketStream(socket_family, m_nonblocking);
     }
 };
 
@@ -683,8 +680,8 @@ class FstackDpdkSocketClient : public KernelSocketClient {
 protected:
     using KernelSocketClient::KernelSocketClient;
 
-    KernelSocketStream* create_stream() override {
-        return new FstackDpdkSocketStream(m_socket_family, m_nonblocking);
+    KernelSocketStream* create_stream(int socket_family) override {
+        return new FstackDpdkSocketStream(socket_family, m_nonblocking);
     }
 
     int fd_connect(int fd, const sockaddr* remote, socklen_t addrlen) override {
@@ -939,8 +936,8 @@ class ETKernelSocketClient : public KernelSocketClient {
     using KernelSocketClient::KernelSocketClient;
 
 protected:
-    KernelSocketStream* create_stream() override {
-        return new ETKernelSocketStream(m_socket_family, m_nonblocking);
+    KernelSocketStream* create_stream(int socket_family) override {
+        return new ETKernelSocketStream(socket_family, m_nonblocking);
     }
 };
 
@@ -974,10 +971,10 @@ class ETKernelSocketServer : public KernelSocketServer, public NotifyContext {
 /* ET Socket - End */
 
 extern "C" ISocketClient* new_tcp_socket_client() {
-    return new KernelSocketClient(AF_INET, true);
+    return new KernelSocketClient(true);
 }
 extern "C" ISocketClient* new_tcp_socket_client_ipv6() {
-    return new KernelSocketClient(AF_INET6, true);
+    return new KernelSocketClient(true);
 }
 extern "C" ISocketServer* new_tcp_socket_server() {
     return NewObj<KernelSocketServer>(AF_INET, false, true)->init();
@@ -986,7 +983,7 @@ extern "C" ISocketServer* new_tcp_socket_server_ipv6() {
     return NewObj<KernelSocketServer>(AF_INET6, false, true)->init();
 }
 extern "C" ISocketClient* new_uds_client() {
-    return new KernelSocketClient(AF_UNIX, true);
+    return new KernelSocketClient(true);
 }
 extern "C" ISocketServer* new_uds_server(bool autoremove) {
     return NewObj<KernelSocketServer>(AF_UNIX, autoremove, true)->init();
@@ -996,14 +993,14 @@ extern "C" ISocketServer* new_zerocopy_tcp_server() {
     return NewObj<ZeroCopySocketServer>(AF_INET, false, true)->init();
 }
 extern "C" ISocketClient* new_zerocopy_tcp_client() {
-    return new ZeroCopySocketClient(AF_INET, true);
+    return new ZeroCopySocketClient(true);
 }
 #ifdef PHOTON_URING
 extern "C" ISocketClient* new_iouring_tcp_client() {
     if (photon::iouring_register_files_enabled())
-        return new IouringFixedFileSocketClient(AF_INET, false);
+        return new IouringFixedFileSocketClient(false);
     else
-        return new IouringSocketClient(AF_INET, false);
+        return new IouringSocketClient(false);
 }
 extern "C" ISocketServer* new_iouring_tcp_server() {
     if (photon::iouring_register_files_enabled())
@@ -1013,23 +1010,23 @@ extern "C" ISocketServer* new_iouring_tcp_server() {
 }
 #endif // PHOTON_URING
 extern "C" ISocketClient* new_et_tcp_socket_client() {
-    return new ETKernelSocketClient(AF_INET, true);
+    return new ETKernelSocketClient(true);
 }
 extern "C" ISocketServer* new_et_tcp_socket_server() {
     return NewObj<ETKernelSocketServer>(AF_INET, false, true)->init();
 }
 extern "C" ISocketClient* new_smc_socket_client() {
-    return new KernelSocketClient(AF_SMC, true);
+    return new KernelSocketClient(true);
 }
 extern "C" ISocketServer* new_smc_socket_server() {
     return NewObj<KernelSocketServer>(AF_SMC, false, true)->init();
 }
 #ifdef ENABLE_FSTACK_DPDK
 extern "C" ISocketClient* new_fstack_dpdk_socket_client() {
-    return new FstackDpdkSocketClient(AF_INET, true);
+    return new FstackDpdkSocketClient(true);
 }
 extern "C" ISocketServer* new_fstack_dpdk_socket_server() {
-    return NewObj<FstackDpdkSocketServer>(AF_INET, false, true)->init();
+    return NewObj<FstackDpdkSocketServer>(false, true)->init();
 }
 #endif // ENABLE_FSTACK_DPDK
 #endif // __linux__
diff --git a/net/test/test.cpp b/net/test/test.cpp
index a61d6f72..ad182a06 100644
--- a/net/test/test.cpp
+++ b/net/test/test.cpp
@@ -714,9 +714,9 @@ TEST(utils, resolver) {
     DEFER(delete resolver);
     net::IPAddr localhost("127.0.0.1");
     net::IPAddr addr = resolver->resolve("localhost");
-    EXPECT_EQ(localhost.to_nl(), addr.to_nl());
+    if (addr.is_ipv4()) EXPECT_EQ(localhost.to_nl(), addr.to_nl());
     auto func = [&](net::IPAddr addr_){
-        EXPECT_EQ(localhost.to_nl(), addr_.to_nl());
+        if (addr_.is_ipv4()) EXPECT_EQ(localhost.to_nl(), addr_.to_nl());
     };
     resolver->resolve("localhost", func);
     resolver->discard_cache("non-exist-host.com");
diff --git a/net/utils.cpp b/net/utils.cpp
index 82985ffb..ad9209d8 100644
--- a/net/utils.cpp
+++ b/net/utils.cpp
@@ -21,6 +21,8 @@ limitations under the License.
 #include <sys/socket.h>
 #include <unistd.h>
 
+#include <chrono>
+#include <list>
 #include <thread>
 #include <string>
 
@@ -249,49 +251,68 @@ bool Base64Decode(std::string_view in, std::string &out) {
 }
 
 class DefaultResolver : public Resolver {
+protected:
+    struct IPAddrNode : public intrusive_list_node<IPAddrNode> {
+        IPAddr addr;
+        IPAddrNode(IPAddr addr) : addr(addr) {}
+    };
+    using IPAddrList = intrusive_list<IPAddrNode>;
 public:
     DefaultResolver(uint64_t cache_ttl, uint64_t resolve_timeout)
         : dnscache_(cache_ttl), resolve_timeout_(resolve_timeout) {}
-    ~DefaultResolver() { dnscache_.clear(); }
+    ~DefaultResolver() {
+        for (auto it : dnscache_) {
+            ((IPAddrList*)it->_obj)->delete_all();
+        }
+        dnscache_.clear();
+    }
 
     IPAddr resolve(const char *host) override {
-        auto ctr = [&]() -> IPAddr * {
-            std::vector<IPAddr> addrs;
+        auto ctr = [&]() -> IPAddrList* {
+            auto addrs = new IPAddrList();
             photon::semaphore sem;
             std::thread([&]() {
-                int ret = gethostbyname(host, addrs);
-                if (ret < 0) {
-                    addrs.clear();
+                auto now = std::chrono::steady_clock::now();
+                IPAddrList ret;
+                auto cb = [&](IPAddr addr) {
+                    ret.push_back(new IPAddrNode(addr));
+                    return 0;
+                };
+                _gethostbyname(host, cb);
+                auto time_elapsed = std::chrono::duration_cast<std::chrono::microseconds>(
+                                        std::chrono::steady_clock::now() - now).count();
+                if ((uint64_t)time_elapsed <= resolve_timeout_) {
+                    addrs->push_back(std::move(ret));
+                    sem.signal(1);
                 }
-                sem.signal(1);
             }).detach();
-            auto ret = sem.wait(1, resolve_timeout_);
-            if (ret < 0 && errno == ETIMEDOUT) {
-                LOG_WARN("Domain resolution for ` timeout!", host);
-                return new IPAddr;  // undefined addr
-            } else if (addrs.empty()) {
-                LOG_WARN("Domain resolution for ` failed", host);
-                return new IPAddr;  // undefined addr
-            }
-            // TODO: support ipv6
-            for (auto& ip : addrs) {
-                if (ip.is_ipv4())
-                    return new IPAddr(ip);
-            }
-            return new IPAddr;      // undefined addr
+            sem.wait(1, resolve_timeout_);
+            return addrs;
         };
-        return *(dnscache_.borrow(host, ctr));
+        auto ips = dnscache_.borrow(host, ctr);
+        if (ips->empty()) LOG_ERRNO_RETURN(0, IPAddr(), "Domain resolution for ` failed", host);
+        auto ret = ips->front();
+        ips->node = ret->next();  // access in round robin order
+        return ret->addr;
     }
 
     void resolve(const char *host, Delegate<void, IPAddr> func) override { func(resolve(host)); }
 
-    void discard_cache(const char *host) override {
+    void discard_cache(const char *host, IPAddr ip) override {
         auto ipaddr = dnscache_.borrow(host);
-        ipaddr.recycle(true);
+        if (ip.undefined() || ipaddr->empty()) ipaddr.recycle(true);
+        else {
+            for (auto itr = ipaddr->rbegin(); itr != ipaddr->rend(); itr++) {
+                if ((*itr)->addr == ip) {
+                    ipaddr->erase(*itr);
+                    break;
+                }
+            }
+        }
     }
 
 private:
-    ObjectCache<std::string, IPAddr *> dnscache_;
+    ObjectCache<std::string, IPAddrList*> dnscache_;
     uint64_t resolve_timeout_;
 };
 
diff --git a/net/utils.h b/net/utils.h
index 3f63f27c..bf61c3b0 100644
--- a/net/utils.h
+++ b/net/utils.h
@@ -157,11 +157,12 @@ class Resolver : public Object {
     // Normally dns servers return multiple ips in random order, choosing the first one should suffice.
     virtual IPAddr resolve(const char* host) = 0;
     virtual void resolve(const char* host, Delegate<void, IPAddr> func) = 0;
-    virtual void discard_cache(const char* host) = 0;  // discard current cache of host:ip
+    virtual void discard_cache(const char* host, IPAddr ip = IPAddr()) = 0;  // discard current cache of ip
 };
 
 /**
  * @brief A non-blocking Resolver based on gethostbyname.
+ * Currently, it's not thread safe.
  *
  * @param cache_ttl cache's lifetime in microseconds.
  * @param resolve_timeout timeout in microseconds for domain resolution.
diff --git a/thread/list.h b/thread/list.h
index 435f8aa8..3f42a8de 100644
--- a/thread/list.h
+++ b/thread/list.h
@@ -221,6 +221,48 @@ class intrusive_list_node : public __intrusive_list_node
     {
         return {nullptr, nullptr};
     }
+
+    struct reverse_iterator
+    {
+        __intrusive_list_node* ptr;
+        __intrusive_list_node* end;
+        T* operator*()
+        {
+            return static_cast<T*>(ptr);
+        }
+        reverse_iterator& operator++()
+        {
+            ptr = ptr->__prev_ptr;
+            if (ptr == end)
+                ptr = nullptr;
+            return *this;
+        }
+        reverse_iterator operator++(int)
+        {
+            auto rst = *this;
+            ptr = ptr->__prev_ptr;
+            if (ptr == end)
+                ptr = nullptr;
+            return rst;
+        }
+        bool operator == (const reverse_iterator& rhs) const
+        {
+            return ptr == rhs.ptr;
+        }
+        bool operator != (const reverse_iterator& rhs) const
+        {
+            return !(*this == rhs);
+        }
+    };
+
+    reverse_iterator rbegin()
+    {
+        return {this->__prev_ptr, this->__prev_ptr};
+    }
+    reverse_iterator rend()
+    {
+        return {nullptr, nullptr};
+    }
 };
 
 
@@ -321,6 +363,7 @@ class intrusive_list
         return node == nullptr;
     }
     typedef typename NodeType::iterator iterator;
+    typedef typename NodeType::reverse_iterator reverse_iterator;
     iterator begin()
     {
         return node ? node->begin() : end();
@@ -329,6 +372,14 @@ class intrusive_list
     {
         return {nullptr, nullptr};
     }
+    reverse_iterator rbegin()
+    {
+        return node ? node->rbegin() : rend();
+    }
+    reverse_iterator rend()
+    {
+        return {nullptr, nullptr};
+    }
     intrusive_list split_front_inclusive(NodeType* ptr)
     {
         auto ret = node;

From 1c0b39a5ed885e0a5797d7cdfa03084b00366ecd Mon Sep 17 00:00:00 2001
From: Huiba Li <huiba.lhb@alibaba-inc.com>
Date: Mon, 25 Dec 2023 10:22:25 +0800
Subject: [PATCH 35/76] Inlined defer (#311)

* make DEFER() always inline
---
 common/utility.h    | 12 ++++++------
 net/http/client.cpp |  4 +---
 2 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/common/utility.h b/common/utility.h
index afeb6fb0..3363d102 100644
--- a/common/utility.h
+++ b/common/utility.h
@@ -74,12 +74,15 @@ ptr_array_t<T> ptr_array(T* pbegin, size_t n)
     return {pbegin, pbegin + n};
 }
 
+#define __INLINE__ __attribute__((always_inline))
+#define __FORCE_INLINE__ __INLINE__ inline
+
 template<typename T>
 class Defer
 {
 public:
-    Defer(T fn) : m_func(fn) {}
-    ~Defer() { m_func(); }
+    Defer(T fn) __INLINE__ : m_func(fn) {}
+    ~Defer() __INLINE__ { m_func(); }
     void operator=(const Defer<T>&) = delete;
     operator bool () { return true; }   // if (DEFER(...)) { ... }
 
@@ -87,12 +90,9 @@ class Defer
     T m_func;
 };
 
-template<typename T>
+template<typename T> __FORCE_INLINE__
 Defer<T> make_defer(T func) { return Defer<T>(func); }
 
-#define __INLINE__ __attribute__((always_inline))
-#define __FORCE_INLINE__ __INLINE__ inline
-
 #define _CONCAT_(a, b) a##b
 #define _CONCAT(a, b) _CONCAT_(a, b)
 
diff --git a/net/http/client.cpp b/net/http/client.cpp
index dc388f74..57fac913 100644
--- a/net/http/client.cpp
+++ b/net/http/client.cpp
@@ -120,9 +120,7 @@ class ClientImpl : public Client {
     CommonHeaders<> m_common_headers;
     ICookieJar *m_cookie_jar;
     ClientImpl(ICookieJar *cookie_jar, TLSContext *tls_ctx) :
-        m_cookie_jar(cookie_jar),
-        m_dialer(tls_ctx) {
-    }
+        m_dialer(tls_ctx), m_cookie_jar(cookie_jar) { }
 
     using SocketStream_ptr = std::unique_ptr<ISocketStream>;
     int redirect(Operation* op) {

From c27805447df9117551b0999991d32cb35bb68dd0 Mon Sep 17 00:00:00 2001
From: Bob Chen <beef9999@qq.com>
Date: Tue, 26 Dec 2023 09:28:10 +0800
Subject: [PATCH 36/76] Don't use gtest_main and gflags_nothreads (#316)

---
 CMake/Findgflags.cmake     | 2 +-
 CMake/Findgoogletest.cmake | 4 ----
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/CMake/Findgflags.cmake b/CMake/Findgflags.cmake
index 13d8b5f5..c93c2374 100644
--- a/CMake/Findgflags.cmake
+++ b/CMake/Findgflags.cmake
@@ -1,6 +1,6 @@
 find_path(GFLAGS_INCLUDE_DIRS gflags/gflags.h)
 
-find_library(GFLAGS_LIBRARIES gflags_nothreads)
+find_library(GFLAGS_LIBRARIES gflags)
 
 find_package_handle_standard_args(gflags DEFAULT_MSG GFLAGS_LIBRARIES GFLAGS_INCLUDE_DIRS)
 
diff --git a/CMake/Findgoogletest.cmake b/CMake/Findgoogletest.cmake
index b88ede98..0de8d20f 100644
--- a/CMake/Findgoogletest.cmake
+++ b/CMake/Findgoogletest.cmake
@@ -1,15 +1,11 @@
 find_path(GOOGLETEST_INCLUDE_DIRS gtest/gtest.h gmock/gmock.h)
 
 find_library(GOOGLETEST_GTEST_LIBRARIES gtest)
-find_library(GOOGLETEST_GTEST_MAIN_LIBRARIES gtest_main)
 find_library(GOOGLETEST_GMOCK_LIBRARIES gmock)
-find_library(GOOGLETEST_GMOCK_MAIN_LIBRARIES gmock_main)
 
 set(GOOGLETEST_LIBRARIES
         ${GOOGLETEST_GTEST_LIBRARIES}
-        ${GOOGLETEST_GTEST_MAIN_LIBRARIES}
         ${GOOGLETEST_GMOCK_LIBRARIES}
-        ${GOOGLETEST_GMOCK_MAIN_LIBRARIES}
 )
 
 find_package_handle_standard_args(googletest DEFAULT_MSG GOOGLETEST_LIBRARIES GOOGLETEST_INCLUDE_DIRS)

From 51be942a4ddc5de2effa874222977d734de539f2 Mon Sep 17 00:00:00 2001
From: Bob Chen <beef9999@qq.com>
Date: Wed, 27 Dec 2023 10:41:58 +0800
Subject: [PATCH 37/76] Don't use gtest_main and gflags_nothreads (#318)

---
 CMake/build-from-src.cmake | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/CMake/build-from-src.cmake b/CMake/build-from-src.cmake
index dad13278..4e91b65f 100644
--- a/CMake/build-from-src.cmake
+++ b/CMake/build-from-src.cmake
@@ -10,6 +10,7 @@ function(build_from_src [dep])
                 aio
                 URL ${PHOTON_AIO_SOURCE}
                 URL_MD5 605237f35de238dfacc83bcae406d95d
+                UPDATE_DISCONNECTED ON
                 BUILD_IN_SOURCE ON
                 CONFIGURE_COMMAND ""
                 BUILD_COMMAND $(MAKE) prefix=${BINARY_DIR} install
@@ -24,6 +25,7 @@ function(build_from_src [dep])
                 zlib
                 URL ${PHOTON_ZLIB_SOURCE}
                 URL_MD5 9b8aa094c4e5765dabf4da391f00d15c
+                UPDATE_DISCONNECTED ON
                 BUILD_IN_SOURCE ON
                 CONFIGURE_COMMAND CFLAGS=-fPIC ./configure --prefix=${BINARY_DIR} --static
                 BUILD_COMMAND $(MAKE)
@@ -38,6 +40,7 @@ function(build_from_src [dep])
                 uring
                 URL ${PHOTON_URING_SOURCE}
                 URL_MD5 2e8c3c23795415475654346484f5c4b8
+                UPDATE_DISCONNECTED ON
                 BUILD_IN_SOURCE ON
                 CONFIGURE_COMMAND ./configure --prefix=${BINARY_DIR}
                 BUILD_COMMAND V=1 CFLAGS=-fPIC $(MAKE) -C src
@@ -59,7 +62,7 @@ function(build_from_src [dep])
         endif ()
         ExternalProject_Get_Property(gflags BINARY_DIR)
         set(GFLAGS_INCLUDE_DIRS ${BINARY_DIR}/include PARENT_SCOPE)
-        set(GFLAGS_LIBRARIES ${BINARY_DIR}/lib/libgflags${POSTFIX}.a ${BINARY_DIR}/lib/libgflags_nothreads${POSTFIX}.a PARENT_SCOPE)
+        set(GFLAGS_LIBRARIES ${BINARY_DIR}/lib/libgflags${POSTFIX}.a PARENT_SCOPE)
 
     elseif (dep STREQUAL "googletest")
         ExternalProject_Add(
@@ -72,8 +75,7 @@ function(build_from_src [dep])
         ExternalProject_Get_Property(googletest SOURCE_DIR)
         ExternalProject_Get_Property(googletest BINARY_DIR)
         set(GOOGLETEST_INCLUDE_DIRS ${SOURCE_DIR}/googletest/include ${SOURCE_DIR}/googlemock/include PARENT_SCOPE)
-        set(GOOGLETEST_LIBRARIES ${BINARY_DIR}/lib/libgmock.a ${BINARY_DIR}/lib/libgmock_main.a
-                ${BINARY_DIR}/lib/libgtest.a ${BINARY_DIR}/lib/libgtest_main.a PARENT_SCOPE)
+        set(GOOGLETEST_LIBRARIES ${BINARY_DIR}/lib/libgmock.a ${BINARY_DIR}/lib/libgtest.a PARENT_SCOPE)
 
     elseif (dep STREQUAL "openssl")
         set(BINARY_DIR ${PROJECT_BINARY_DIR}/openssl-build)
@@ -81,6 +83,7 @@ function(build_from_src [dep])
                 openssl
                 URL ${PHOTON_OPENSSL_SOURCE}
                 URL_MD5 bad68bb6bd9908da75e2c8dedc536b29
+                UPDATE_DISCONNECTED ON
                 BUILD_IN_SOURCE ON
                 CONFIGURE_COMMAND ./config -fPIC --prefix=${BINARY_DIR} --openssldir=${BINARY_DIR} shared
                 BUILD_COMMAND make -j 1  # https://github.com/openssl/openssl/issues/5762#issuecomment-376622684
@@ -102,6 +105,7 @@ function(build_from_src [dep])
                 curl
                 URL ${PHOTON_CURL_SOURCE}
                 URL_MD5 a66270f11e3fbfad709600bbd1686704
+                UPDATE_DISCONNECTED ON
                 BUILD_IN_SOURCE ON
                 CONFIGURE_COMMAND autoreconf -i COMMAND ./configure --with-ssl=${OPENSSL_ROOT_DIR}
                     --without-libssh2 --enable-static --enable-shared=no --enable-optimize

From 49a271009481a00319e12cde28294b3f151506e0 Mon Sep 17 00:00:00 2001
From: Bob Chen <beef9999@qq.com>
Date: Mon, 8 Jan 2024 09:36:07 +0800
Subject: [PATCH 38/76] Re-define an undefined endpoint (#322)

---
 CMake/build-from-src.cmake | 2 +-
 net/kernel_socket.cpp      | 4 ++--
 net/socket.h               | 5 +++--
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/CMake/build-from-src.cmake b/CMake/build-from-src.cmake
index 4e91b65f..abfcbe1b 100644
--- a/CMake/build-from-src.cmake
+++ b/CMake/build-from-src.cmake
@@ -87,7 +87,7 @@ function(build_from_src [dep])
                 BUILD_IN_SOURCE ON
                 CONFIGURE_COMMAND ./config -fPIC --prefix=${BINARY_DIR} --openssldir=${BINARY_DIR} shared
                 BUILD_COMMAND make -j 1  # https://github.com/openssl/openssl/issues/5762#issuecomment-376622684
-                INSTALL_COMMAND $(MAKE) install
+                INSTALL_COMMAND make install
                 LOG_CONFIGURE ON
                 LOG_BUILD ON
                 LOG_INSTALL ON
diff --git a/net/kernel_socket.cpp b/net/kernel_socket.cpp
index 08715afa..ada1c05a 100644
--- a/net/kernel_socket.cpp
+++ b/net/kernel_socket.cpp
@@ -1035,8 +1035,8 @@ extern "C" ISocketServer* new_fstack_dpdk_socket_server() {
 
 /* Implementations in socket.h */
 
-EndPoint::EndPoint(const char* s) {
-    estring_view ep(s);
+EndPoint::EndPoint(const char* _ep) {
+    estring_view ep(_ep);
     auto pos = ep.find_last_of(':');
     if (pos == estring::npos)
         return;
diff --git a/net/socket.h b/net/socket.h
index c0914ef8..93b45283 100644
--- a/net/socket.h
+++ b/net/socket.h
@@ -150,7 +150,8 @@ namespace net {
         uint16_t port = 0;
         EndPoint() = default;
         EndPoint(IPAddr ip, uint16_t port) : addr(ip), port(port) {}
-        explicit EndPoint(const char* s);
+        explicit EndPoint(const char* ep);
+        EndPoint(const char* ip, uint16_t port) : addr(ip), port(port) {}
         bool is_ipv4() const {
             return addr.is_ipv4();
         };
@@ -161,7 +162,7 @@ namespace net {
             return !operator==(rhs);
         }
         bool undefined() const {
-            return addr.undefined() && port == 0;
+            return addr.undefined();
         }
     };
 

From bd08579603765cb8814da89f7050c18c2e1464af Mon Sep 17 00:00:00 2001
From: Bob Chen <beef9999@qq.com>
Date: Wed, 10 Jan 2024 14:22:56 +0800
Subject: [PATCH 39/76] iouring: remove RLIMIT_MEMLOCK check for new kernel

---
 .github/workflows/ci.linux.x86.yml |  2 --
 io/iouring-wrapper.cpp             | 10 +++++++---
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/ci.linux.x86.yml b/.github/workflows/ci.linux.x86.yml
index d19e377b..b0101a2b 100644
--- a/.github/workflows/ci.linux.x86.yml
+++ b/.github/workflows/ci.linux.x86.yml
@@ -86,7 +86,6 @@ jobs:
 
     container:
       image: dokken/centos-stream-8:sha-40294ce
-      # In order to run io_uring, the docker daemon should add --default-ulimit memlock=-1:-1
       options: --cpus 4
 
     steps:
@@ -116,6 +115,5 @@ jobs:
       - name: Test
         run: |
           cd build
-          ulimit -l unlimited
           export PHOTON_CI_EV_ENGINE=io_uring
           ctest --timeout 3600 -V
diff --git a/io/iouring-wrapper.cpp b/io/iouring-wrapper.cpp
index 77579daa..ab3aadd1 100644
--- a/io/iouring-wrapper.cpp
+++ b/io/iouring-wrapper.cpp
@@ -76,10 +76,14 @@ class iouringEngine : public MasterEventEngine, public CascadingEventEngine, pub
     }
 
     int init() {
-        rlimit resource_limit{.rlim_cur = RLIM_INFINITY, .rlim_max = RLIM_INFINITY};
-        if (setrlimit(RLIMIT_MEMLOCK, &resource_limit) != 0) {
-            LOG_ERROR_RETURN(0, -1, "iouring: failed to set resource limit. Use command `ulimit -l unlimited`, or change to root");
+        int compare_result;
+        if (kernel_version_compare("5.11", compare_result) == 0 && compare_result <= 0) {
+            rlimit resource_limit{.rlim_cur = RLIM_INFINITY, .rlim_max = RLIM_INFINITY};
+            if (setrlimit(RLIMIT_MEMLOCK, &resource_limit) != 0)
+                LOG_ERROR_RETURN(0, -1, "iouring: failed to set resource limit. "
+                                        "Use command `ulimit -l unlimited`, or change to root");
         }
+
         check_register_file_support();
         check_cooperative_task_support();
         set_submit_wait_function();

From b57b2774cd528c8e2a7bc3b2103f034cdbb92ed5 Mon Sep 17 00:00:00 2001
From: Bob Chen <beef9999@qq.com>
Date: Mon, 15 Jan 2024 21:23:13 +0800
Subject: [PATCH 40/76] [http] Wrong domain name fail immediately

---
 net/http/client.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/net/http/client.cpp b/net/http/client.cpp
index 57fac913..7c9cf547 100644
--- a/net/http/client.cpp
+++ b/net/http/client.cpp
@@ -70,7 +70,7 @@ ISocketStream* PooledDialer::dial(std::string_view host, uint16_t port, bool sec
     std::string strhost(host);
     auto ipaddr = resolver->resolve(strhost.c_str());
     if (ipaddr.undefined()) {
-        LOG_ERROR_RETURN(0, nullptr, "DNS resolve failed, name = `", host)
+        LOG_ERROR_RETURN(ENOENT, nullptr, "DNS resolve failed, name = `", host)
     }
 
     EndPoint ep(ipaddr, port);
@@ -165,7 +165,7 @@ class ClientImpl : public Client {
                      ? m_dialer.dial(m_proxy_url, tmo.timeout())
                      : m_dialer.dial(req, tmo.timeout());
         if (!s) {
-            if (errno == ECONNREFUSED) {
+            if (errno == ECONNREFUSED || errno == ENOENT) {
                 LOG_ERROR_RETURN(0, ROUNDTRIP_FAST_RETRY, "connection refused")
             }
             LOG_ERROR_RETURN(0, ROUNDTRIP_NEED_RETRY, "connection failed");

From 94517910727d7f44cc991bc87711c68e82552601 Mon Sep 17 00:00:00 2001
From: Bob Chen <beef9999@qq.com>
Date: Thu, 18 Jan 2024 10:21:50 +0800
Subject: [PATCH 41/76] Treat compiler warning as error (#328)

---
 .github/workflows/ci.macos.arm.yml  |  6 +++---
 .github/workflows/ci.macos.yml      |  6 +++---
 CMakeLists.txt                      |  4 ++--
 examples/CMakeLists.txt             |  2 ++
 fs/subfs.cpp                        | 16 ++++++++--------
 net/basic_socket.cpp                |  2 +-
 net/http/headers.cpp                |  7 +++++++
 net/http/url.cpp                    |  2 +-
 net/security-context/tls-stream.cpp |  3 +++
 thread/thread.cpp                   |  8 ++++----
 10 files changed, 34 insertions(+), 22 deletions(-)

diff --git a/.github/workflows/ci.macos.arm.yml b/.github/workflows/ci.macos.arm.yml
index c8e17718..77ad0230 100644
--- a/.github/workflows/ci.macos.arm.yml
+++ b/.github/workflows/ci.macos.arm.yml
@@ -7,7 +7,7 @@ on:
     branches: [ "main", "release/*" ]
 
 jobs:
-  macOS-clang-debug:
+  macOS-clang-release:
     runs-on: [self-hosted, macOS, ARM64]
 
     steps:
@@ -26,9 +26,9 @@ jobs:
 
       - name: Build
         run: |
-          cmake -B ${{github.workspace}}/build -D PHOTON_BUILD_TESTING=ON -D CMAKE_BUILD_TYPE=Debug \
+          cmake -B ${{github.workspace}}/build -D PHOTON_BUILD_TESTING=ON -D CMAKE_BUILD_TYPE=Release \
             -D PHOTON_ENABLE_SASL=ON -DOPENSSL_ROOT_DIR=/usr/local/opt/openssl@3
-          cmake --build ${{github.workspace}}/build -j -- VERBOSE=1
+          cmake --build ${{github.workspace}}/build -j
 
       - name: Test
         working-directory: ${{github.workspace}}/build
diff --git a/.github/workflows/ci.macos.yml b/.github/workflows/ci.macos.yml
index 5e75d552..2ecfae9e 100644
--- a/.github/workflows/ci.macos.yml
+++ b/.github/workflows/ci.macos.yml
@@ -7,7 +7,7 @@ on:
     branches: [ "main", "release/*" ]
 
 jobs:
-  macOS-12-Monterey-debug:
+  macOS-12-Monterey-release:
     runs-on: macos-12
 
     steps:
@@ -26,9 +26,9 @@ jobs:
 
     - name: Build
       run: |
-        cmake -B ${{github.workspace}}/build -D PHOTON_BUILD_TESTING=ON -D CMAKE_BUILD_TYPE=Debug \
+        cmake -B ${{github.workspace}}/build -D PHOTON_BUILD_TESTING=ON -D CMAKE_BUILD_TYPE=Release \
           -D PHOTON_ENABLE_SASL=ON -DOPENSSL_ROOT_DIR=/usr/local/opt/openssl@3
-        cmake --build ${{github.workspace}}/build -j -- VERBOSE=1
+        cmake --build ${{github.workspace}}/build -j
 
     - name: Test
       working-directory: ${{github.workspace}}/build
diff --git a/CMakeLists.txt b/CMakeLists.txt
index b92e2c5d..f71d6c2f 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -45,8 +45,8 @@ if (NOT (${ARCH} STREQUAL x86_64) AND NOT (${ARCH} STREQUAL aarch64) AND NOT (${
 endif ()
 ProcessorCount(NumCPU)
 
-# Compiler options
-add_compile_options(-Wall)  # -Werror is not enable yet
+# Global compiler options, only effective within this project
+add_compile_options(-Wall -Werror -Wno-error=pragmas)
 
 set(CMAKE_CXX_STANDARD ${PHOTON_CXX_STANDARD})
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index 5c8b91e4..b0cc4a1e 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -1,3 +1,5 @@
+add_definitions(-w)
+
 add_executable(simple-example simple/simple.cpp)
 target_link_libraries(simple-example PRIVATE photon_static)
 
diff --git a/fs/subfs.cpp b/fs/subfs.cpp
index 102037f0..ab30c89b 100644
--- a/fs/subfs.cpp
+++ b/fs/subfs.cpp
@@ -231,56 +231,56 @@ namespace fs
             return underlayfs->mknod(path, mode, dev);
         }
 
-        virtual ssize_t getxattr(const char *path, const char *name, void *value, size_t size)
+        virtual ssize_t getxattr(const char *path, const char *name, void *value, size_t size) override
         {
             if (!underlay_xattrfs)
                 LOG_ERROR_RETURN(ENOTSUP, -1, "xattr is not supported by underlay fs");
             PathCat __(this, path);
             return underlay_xattrfs->getxattr(path, name, value, size);
         }
-        virtual ssize_t lgetxattr(const char *path, const char *name, void *value, size_t size)
+        virtual ssize_t lgetxattr(const char *path, const char *name, void *value, size_t size) override
         {
             if (!underlay_xattrfs)
                 LOG_ERROR_RETURN(ENOTSUP, -1, "xattr is not supported by underlay fs");
             PathCat __(this, path);
             return underlay_xattrfs->lgetxattr(path, name, value, size);
         }
-        virtual ssize_t listxattr(const char *path, char *list, size_t size)
+        virtual ssize_t listxattr(const char *path, char *list, size_t size) override
         {
             if (!underlay_xattrfs)
                 LOG_ERROR_RETURN(ENOTSUP, -1, "xattr is not supported by underlay fs");
             PathCat __(this, path);
             return underlay_xattrfs->listxattr(path, list, size);
         }
-        virtual ssize_t llistxattr(const char *path, char *list, size_t size)
+        virtual ssize_t llistxattr(const char *path, char *list, size_t size) override
         {
             if (!underlay_xattrfs)
                 LOG_ERROR_RETURN(ENOTSUP, -1, "xattr is not supported by underlay fs");
             PathCat __(this, path);
             return underlay_xattrfs->llistxattr(path, list, size);
         }
-        virtual int setxattr(const char *path, const char *name, const void *value, size_t size, int flags)
+        virtual int setxattr(const char *path, const char *name, const void *value, size_t size, int flags) override
         {
             if (!underlay_xattrfs)
                 LOG_ERROR_RETURN(ENOTSUP, -1, "xattr is not supported by underlay fs");
             PathCat __(this, path);
             return underlay_xattrfs->setxattr(path, name, value, size, flags);
         }
-        virtual int lsetxattr(const char *path, const char *name, const void *value, size_t size, int flags)
+        virtual int lsetxattr(const char *path, const char *name, const void *value, size_t size, int flags) override
         {
             if (!underlay_xattrfs)
                 LOG_ERROR_RETURN(ENOTSUP, -1, "xattr is not supported by underlay fs");
             PathCat __(this, path);
             return underlay_xattrfs->lsetxattr(path, name, value, size, flags);
         }
-        virtual int removexattr(const char *path, const char *name)
+        virtual int removexattr(const char *path, const char *name) override
         {
             if (!underlay_xattrfs)
                 LOG_ERROR_RETURN(ENOTSUP, -1, "xattr is not supported by underlay fs");
             PathCat __(this, path);
             return underlay_xattrfs->removexattr(path, name);
         }
-        virtual int lremovexattr(const char *path, const char *name)
+        virtual int lremovexattr(const char *path, const char *name) override
         {
             if (!underlay_xattrfs)
                 LOG_ERROR_RETURN(ENOTSUP, -1, "xattr is not supported by underlay fs");
diff --git a/net/basic_socket.cpp b/net/basic_socket.cpp
index 8446b8dd..5b5e8ac3 100644
--- a/net/basic_socket.cpp
+++ b/net/basic_socket.cpp
@@ -284,7 +284,7 @@ ssize_t ISocketStream::recv_at_least_mutable(struct iovec *iov, int iovcnt,
         if (ret == 0) break;    // EOF
         if ((n += ret) >= least) break;
         auto r = v.extract_front(ret);
-        assert(r == ret); (void)r;
+        assert((ssize_t) r == ret); (void)r;
     } while (v.iovcnt && v.iov->iov_len);
     return n;
 }
diff --git a/net/http/headers.cpp b/net/http/headers.cpp
index 39f262e4..ef997f5c 100644
--- a/net/http/headers.cpp
+++ b/net/http/headers.cpp
@@ -146,7 +146,14 @@ HeadersBase::KV* HeadersBase::kv_add_sort(KV kv) {
     if ((char*)(begin - 1) <= m_buf + m_buf_size)
         LOG_ERROR_RETURN(ENOBUFS, nullptr, "no buffer");
     auto it = std::lower_bound(begin, kv_end(), kv, HA(this));
+#ifndef __clang__
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wclass-memaccess"
+#endif
     memmove(begin - 1, begin, sizeof(KV) * (it - begin));
+#ifndef __clang__
+#pragma GCC diagnostic pop
+#endif
     m_kv_size++;
     *(it - 1) = kv;
     return it - 1;
diff --git a/net/http/url.cpp b/net/http/url.cpp
index 0c66f5f6..e5a534a6 100644
--- a/net/http/url.cpp
+++ b/net/http/url.cpp
@@ -27,7 +27,7 @@ void URL::fix_target() {
     if (m_target.size() == 0 || t.front() != '/') {
         m_tmp_target = (char*)malloc(m_target.size() + 1);
         m_tmp_target[0] = '/';
-        strncpy(m_tmp_target+1, t.data(), t.size());
+        memcpy(m_tmp_target+1, t.data(), t.size());
         m_target = rstring_view16(0, m_target.size()+1);
         m_path = rstring_view16(0, m_path.size()+1);
     }
diff --git a/net/security-context/tls-stream.cpp b/net/security-context/tls-stream.cpp
index dc3ffc59..ad12e61e 100644
--- a/net/security-context/tls-stream.cpp
+++ b/net/security-context/tls-stream.cpp
@@ -101,6 +101,8 @@ class TLSContextImpl : public TLSContext {
             case TLSVersion::SSL23:
                 method = SSLv23_method();
                 break;
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
             case TLSVersion::TLS11:
                 method = TLSv1_1_method();
                 break;
@@ -109,6 +111,7 @@ class TLSContextImpl : public TLSContext {
                 break;
             default:
                 method = TLSv1_2_method();
+#pragma GCC diagnostic pop
         }
         ctx = SSL_CTX_new(method);
         if (ctx == nullptr) {
diff --git a/thread/thread.cpp b/thread/thread.cpp
index f9f1a67e..9ddcd274 100644
--- a/thread/thread.cpp
+++ b/thread/thread.cpp
@@ -928,15 +928,15 @@ R"(
         size_t randomizer = (rand() % 32) * (1024 + 8);
         stack_size = align_up(randomizer + stack_size + sizeof(thread), PAGE_SIZE);
         char* ptr = (char*)photon_thread_alloc(stack_size);
-        auto p = ptr + stack_size - sizeof(thread) - randomizer;
-        (uint64_t&)p &= ~63;
-        auto th = new (p) thread;
+        uint64_t p = (uint64_t) ptr + stack_size - sizeof(thread) - randomizer;
+        p = align_down(p, 64);
+        auto th = new((char*) p) thread;
         th->buf = ptr;
         th->stackful_alloc_top = ptr;
         th->start = start;
         th->stack_size = stack_size;
         th->arg = arg;
-        auto sp = align_down((uint64_t)p - reserved_space, 64);
+        auto sp = align_down(p - reserved_space, 64);
         th->stack.init((void*)sp, &_photon_thread_stub, th);
         AtomicRunQ arq(rq);
         th->vcpu = arq.vcpu;

From 36974a8c5e78a65bc38caa35bc5e6da70350c3f1 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Fri, 19 Jan 2024 09:18:03 +0800
Subject: [PATCH 42/76] Fix HTTPFSv2 using proxy (#336) (#337)

Signed-off-by: Coldwings <coldwings@me.com>
Co-authored-by: Coldwings <coldwings@me.com>
---
 fs/httpfs/httpfs_v2.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fs/httpfs/httpfs_v2.cpp b/fs/httpfs/httpfs_v2.cpp
index 3aa3d881..e12f2a8c 100644
--- a/fs/httpfs/httpfs_v2.cpp
+++ b/fs/httpfs/httpfs_v2.cpp
@@ -151,8 +151,8 @@ class HttpFile_v2 : public fs::VirtualReadOnlyFile {
     again:
         estring url;
         url.appends(m_url, "?", m_url_param);
-        op.req.reset(net::http::Verb::GET, url);
         op.set_enable_proxy(m_fs->get_client()->has_proxy());
+        op.req.reset(net::http::Verb::GET, url, op.enable_proxy);
         op.req.headers.merge(m_common_header);
         op.req.headers.range(offset, offset + length - 1);
         op.req.headers.content_length(0);

From 0801372ec75b24cafb3e5057af56a53a4102fedf Mon Sep 17 00:00:00 2001
From: Bob Chen <beef9999@qq.com>
Date: Wed, 31 Jan 2024 15:09:12 +0800
Subject: [PATCH 43/76] Fix ipv6 test (#345)

---
 net/http/test/client_function_test.cpp |  2 +-
 net/http/test/server_function_test.cpp |  2 +-
 net/test/CMakeLists.txt                |  3 ++-
 net/test/test-ipv6.cpp                 | 12 +++++++-----
 net/utils.cpp                          |  1 +
 5 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/net/http/test/client_function_test.cpp b/net/http/test/client_function_test.cpp
index 8e0b48cd..4d68d3c1 100644
--- a/net/http/test/client_function_test.cpp
+++ b/net/http/test/client_function_test.cpp
@@ -580,7 +580,7 @@ int main(int argc, char** arg) {
     }
     DEFER(et_poller_fini());
 #endif
-    set_log_output_level(ALOG_DEBUG);
+    set_log_output_level(ALOG_INFO);
     ::testing::InitGoogleTest(&argc, arg);
     LOG_DEBUG("test result:`", RUN_ALL_TESTS());
 }
diff --git a/net/http/test/server_function_test.cpp b/net/http/test/server_function_test.cpp
index e50d547d..628f0ad9 100644
--- a/net/http/test/server_function_test.cpp
+++ b/net/http/test/server_function_test.cpp
@@ -531,7 +531,7 @@ int main(int argc, char** arg) {
     }
     DEFER(net::et_poller_fini());
 #endif
-    set_log_output_level(ALOG_DEBUG);
+    set_log_output_level(ALOG_INFO);
     ::testing::InitGoogleTest(&argc, arg);
     LOG_DEBUG("test result:`", RUN_ALL_TESTS());
 }
diff --git a/net/test/CMakeLists.txt b/net/test/CMakeLists.txt
index d8860fae..1d10876d 100644
--- a/net/test/CMakeLists.txt
+++ b/net/test/CMakeLists.txt
@@ -23,4 +23,5 @@ add_executable(test-client test-client.cpp)
 target_link_libraries(test-client PRIVATE photon_shared)
 
 add_executable(test-ipv6 test-ipv6.cpp)
-target_link_libraries(test-ipv6 PRIVATE photon_shared)
\ No newline at end of file
+target_link_libraries(test-ipv6 PRIVATE photon_shared)
+add_test(NAME test-ipv6 COMMAND $<TARGET_FILE:test-ipv6>)
\ No newline at end of file
diff --git a/net/test/test-ipv6.cpp b/net/test/test-ipv6.cpp
index 4e3cc809..bcc03f4c 100644
--- a/net/test/test-ipv6.cpp
+++ b/net/test/test-ipv6.cpp
@@ -54,21 +54,22 @@ TEST(ipv6, addr) {
     EXPECT_TRUE(b.is_link_local());
 }
 
+// GitHub CI doesn't support IPv6 container for now
 TEST(ipv6, get_host_by_peer) {
-    auto peer = photon::net::gethostbypeer(photon::net::IPAddr("2001:4860:4860::8888"));
+    auto peer = photon::net::gethostbypeer(photon::net::IPAddr("8.8.8.8"));
     ASSERT_TRUE(!peer.undefined());
-    ASSERT_TRUE(!peer.is_ipv4());
+    ASSERT_TRUE(peer.is_ipv4());
     LOG_INFO(peer);
 }
 
 TEST(ipv6, dns_lookup) {
     std::vector<photon::net::IPAddr> ret;
-    int num = photon::net::gethostbyname("github.com", ret);
+    int num = photon::net::gethostbyname("taobao.com", ret);
     ASSERT_GT(num, 0);
     ASSERT_EQ(num, ret.size());
     bool has_v6 = false;
     for (auto& each : ret) {
-        LOG_INFO("github.com IP addr `", each);
+        LOG_INFO("taobao.com IP addr `", each);
         if (!each.is_ipv4()) {
             has_v6 = true;
             break;
@@ -164,7 +165,8 @@ class V4ToV6Test : public DualStackTest {
     bool is_ipv6_client() override { return false; }
 };
 
-TEST_F(V6ToV6Test, run) {
+// GitHub CI doesn't support IPv6 container for now
+TEST_F(V6ToV6Test, DISABLED_run) {
     run();
 }
 
diff --git a/net/utils.cpp b/net/utils.cpp
index ad9209d8..f937c200 100644
--- a/net/utils.cpp
+++ b/net/utils.cpp
@@ -77,6 +77,7 @@ int _gethostbyname(const char* name, Delegate<int, IPAddr> append_op) {
     addrinfo* result = nullptr;
     addrinfo hints = {};
     hints.ai_socktype = SOCK_STREAM;
+    hints.ai_flags = AI_ALL | AI_V4MAPPED;
     hints.ai_family = AF_UNSPEC;
 
     int ret = getaddrinfo(name, nullptr, &hints, &result);

From 1bf5242c7d020065c9d55641b3177ec216e53c9f Mon Sep 17 00:00:00 2001
From: Coldwings <coldwings@me.com>
Date: Tue, 6 Feb 2024 20:15:20 +0800
Subject: [PATCH 44/76] Fix IPAddress packed layout and iovector flex array
 (#352)

* Fix IPAddr memory layout

* Ignore GCC13 new warning for flex array inheritance
---
 common/iovector.h | 6 +++++-
 net/socket.h      | 2 +-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/common/iovector.h b/common/iovector.h
index 08f6193f..97d583d4 100644
--- a/common/iovector.h
+++ b/common/iovector.h
@@ -236,6 +236,10 @@ inline void delete_iovector(iovector* ptr);
 
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Warray-bounds"
+#if __GNUC_MINOR__ >= 13
+#pragma GCC diagnostic ignored "-Wzero-length-bounds"
+#pragma GCC diagnostic ignored "-Wstrict-flex-arrays"
+#endif
 
 class iovector
 {
@@ -861,7 +865,7 @@ class iovector
 
     struct IOVAllocation_ : public IOAlloc
     {
-        void* bases[0];
+        void* bases[];
         void* do_allocate(int size, uint16_t& nbases, uint16_t capacity)
         {
             return do_allocate(size, size, nbases, capacity);
diff --git a/net/socket.h b/net/socket.h
index 93b45283..bf5c783c 100644
--- a/net/socket.h
+++ b/net/socket.h
@@ -47,7 +47,7 @@ namespace net {
         union {
             in6_addr addr = {};
             struct { uint16_t _1, _2, _3, _4, _5, _6; uint8_t a, b, c, d; };
-        };
+        } __attribute__((packed));
         // For compatibility, the default constructor is still 0.0.0.0 (IPv4)
         IPAddr() {
             map_v4(htonl(INADDR_ANY));

From 1417a40db67f38260130012f802dc6899e274070 Mon Sep 17 00:00:00 2001
From: Huiba Li <huiba.lhb@alibaba-inc.com>
Date: Sun, 18 Feb 2024 11:21:21 +0800
Subject: [PATCH 45/76] fix compile (#353)

---
 thread/test/perf_usleepdefer_semaphore.cpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/thread/test/perf_usleepdefer_semaphore.cpp b/thread/test/perf_usleepdefer_semaphore.cpp
index b4fd6fc8..9c056d10 100644
--- a/thread/test/perf_usleepdefer_semaphore.cpp
+++ b/thread/test/perf_usleepdefer_semaphore.cpp
@@ -21,6 +21,7 @@ limitations under the License.
 #include <thread>
 #include <photon/common/metric-meter/metrics.h>
 #include <photon/common/alog.h>
+#include <memory>
 
 Metric::AverageLatencyCounter ldefer, lsem;
 
@@ -37,7 +38,7 @@ void* task_defer(void* arg) {
 
 void* task_semaphore(void* arg) {
     SCOPE_LATENCY(*(Metric::AverageLatencyCounter*)(arg));
-    std::shared_ptr<photon::semaphore> sem=std::make_shared<photon::semaphore>(0);
+    auto sem = std::make_shared<photon::semaphore>(0);
     std::thread([&, sem]{
         sem->signal(1);
     }).detach();
@@ -59,7 +60,7 @@ void* ph_task_defer(void* arg) {
 
 void* ph_task_semaphore(void* arg) {
     SCOPE_LATENCY(*(Metric::AverageLatencyCounter*)(arg));
-    std::shared_ptr<photon::semaphore> sem=std::make_shared<photon::semaphore>(0);
+    auto sem = std::make_shared<photon::semaphore>(0);
     auto task = [&, sem]{
             sem->signal(1);
         };

From 71790a926d73a59f3f8ab5efb6e551a6eacf88b8 Mon Sep 17 00:00:00 2001
From: Coldwings <coldwings@me.com>
Date: Sun, 18 Feb 2024 11:19:52 +0800
Subject: [PATCH 46/76] FIX httpfs_v2 common header for file able to replace by
 ioctl (#351)

FIX httpfs common header for file able to replace by ioctl
---
 fs/httpfs/httpfs_v2.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/fs/httpfs/httpfs_v2.cpp b/fs/httpfs/httpfs_v2.cpp
index e12f2a8c..57e07072 100644
--- a/fs/httpfs/httpfs_v2.cpp
+++ b/fs/httpfs/httpfs_v2.cpp
@@ -103,7 +103,7 @@ class HttpFs_v2 : public fs::IFileSystem {
 class HttpFile_v2 : public fs::VirtualReadOnlyFile {
 public:
     std::string m_url;
-    net::http::CommonHeaders<> m_common_header;
+    unordered_map_string_key<std::string> m_common_header;
     HttpFs_v2* m_fs;
     struct stat m_stat;
     uint64_t m_stat_gettime = 0;
@@ -153,7 +153,8 @@ class HttpFile_v2 : public fs::VirtualReadOnlyFile {
         url.appends(m_url, "?", m_url_param);
         op.set_enable_proxy(m_fs->get_client()->has_proxy());
         op.req.reset(net::http::Verb::GET, url, op.enable_proxy);
-        op.req.headers.merge(m_common_header);
+        for (auto &kv : m_common_header)
+            op.req.headers.insert(kv.first, kv.second);
         op.req.headers.range(offset, offset + length - 1);
         op.req.headers.content_length(0);
         op.timeout = tmo.timeout();
@@ -228,11 +229,10 @@ class HttpFile_v2 : public fs::VirtualReadOnlyFile {
         return m_exists ? 0 : -1;
     }
 
-    //TODO: 这里是否需要考虑m_common_header被打爆的问题？
     void add_header(va_list args) {
         auto k = va_arg(args, const char*);
         auto v = va_arg(args, const char*);
-        m_common_header.insert(k, v);
+        m_common_header[k] = v;
     }
 
     void add_url_param(va_list args) { m_url_param = va_arg(args, const char*); }

From c3a3138ac7af087fcde5b47efc67be6ed564ffc6 Mon Sep 17 00:00:00 2001
From: Coldwings <coldwings@me.com>
Date: Wed, 21 Feb 2024 10:58:17 +0800
Subject: [PATCH 47/76] ALog Prologue now keeps only last part of source path
 (filename), using us timestamps (#364)

* Fix alog namedvalue print character array

Signed-off-by: Coldwings <coldwings@me.com>

* ALog prologue keeps only filename, uses us timestamp

Signed-off-by: Coldwings <coldwings@me.com>

* make `update_now` returns timeval, so as `alog_update_now()`

Signed-off-by: Coldwings <coldwings@me.com>

* Fix `make_named_value`

Signed-off-by: Coldwings <coldwings@me.com>

* `Prologue use const char* for string instead of uint64_t

Signed-off-by: Coldwings <coldwings@me.com>

* `prologue` with constexpr constructor

Signed-off-by: Coldwings <coldwings@me.com>

* prevent warning for make_named_value in clang

Signed-off-by: Coldwings <coldwings@me.com>

---------

Signed-off-by: Coldwings <coldwings@me.com>
---
 common/alog.cpp   | 19 ++++++++++---------
 common/alog.h     | 38 ++++++++++++++++++++++++--------------
 thread/thread.cpp | 21 +++++++++++++--------
 3 files changed, 47 insertions(+), 31 deletions(-)

diff --git a/common/alog.cpp b/common/alog.cpp
index 4d5a7aad..e371b801 100644
--- a/common/alog.cpp
+++ b/common/alog.cpp
@@ -228,11 +228,6 @@ struct tm* alog_update_time(time_t now)
     return &alog_time;
 }
 
-static struct tm* alog_update_time()
-{
-    return alog_update_time(time(0) - timezone);
-}
-
 class LogOutputFile final : public BaseLogOutput {
 public:
     uint64_t log_file_size_limit = 0;
@@ -494,28 +489,34 @@ static inline ALogInteger DEC_W2P0(uint64_t x)
     return DEC(x).width(2).padding('0');
 }
 
+namespace photon {
+struct timeval alog_update_now();
+}
+
 LogBuffer& operator << (LogBuffer& log, const Prologue& pro)
 {
 #ifdef LOG_BENCHMARK
     auto t = &alog_time;
 #else
-    auto t = alog_update_time();
+    auto ts = photon::alog_update_now();
+    auto t = alog_update_time(ts.tv_sec - timezone);
 #endif
     log.printf(t->tm_year, '/');
     log.printf(DEC_W2P0(t->tm_mon),  '/');
     log.printf(DEC_W2P0(t->tm_mday), ' ');
     log.printf(DEC_W2P0(t->tm_hour), ':');
     log.printf(DEC_W2P0(t->tm_min),  ':');
-    log.printf(DEC_W2P0(t->tm_sec));
+    log.printf(DEC_W2P0(t->tm_sec), '.');
+    log.printf(DEC(ts.tv_usec).width(6).padding('0'));
 
     static const char levels[] = "|DEBUG|th=|INFO |th=|WARN |th=|ERROR|th=|FATAL|th=|TEMP |th=|AUDIT|th=";
     log.reserved = pro.level;
     log.printf(ALogString(&levels[pro.level * 10], 10));
     log.printf(photon::CURRENT, '|');
     if (pro.level != ALOG_AUDIT) {
-        log.printf(ALogString((char*)pro.addr_file, pro.len_file), ':');
+        log.printf(ALogString(pro.addr_file, pro.len_file), ':');
         log.printf(pro.line, '|');
-        log.printf(ALogString((char*)pro.addr_func, pro.len_func), ':');
+        log.printf(ALogString(pro.addr_func, pro.len_func), ':');
     }
     return log;
 }
diff --git a/common/alog.h b/common/alog.h
index fe8ff2f0..247bdbd5 100644
--- a/common/alog.h
+++ b/common/alog.h
@@ -343,9 +343,19 @@ struct LogBuffer : public ALogBuffer
 
 struct Prologue
 {
-    uint64_t addr_func, addr_file;
+    const char *addr_func, *addr_file;
     int len_func, len_file;
     int line, level;
+
+    template <size_t N, typename FILEN>
+    constexpr Prologue(const char (&addr_func_)[N], FILEN addr_file_, int line_,
+                       int level_)
+        : addr_func(addr_func_),
+          addr_file(addr_file_.chars),
+          len_func(N - 1),
+          len_file(addr_file_.len - 1),
+          line(line_),
+          level(level_) {}
 };
 
 LogBuffer& operator << (LogBuffer& log, const Prologue& pro);
@@ -441,25 +451,19 @@ struct LogBuilder {
     }
 };
 
-#if __GNUC__ >= 9
-#define DEFINE_PROLOGUE(level, prolog)                                          \
-    static constexpr const char* _prologue_func = __func__;                     \
-    const static Prologue prolog{                                               \
-        (uint64_t) _prologue_func,  (uint64_t)__FILE__, sizeof(__func__) - 1,   \
-        sizeof(__FILE__) - 1, __LINE__, level};
-#else
-#define DEFINE_PROLOGUE(level, prolog)                                  \
-    const static Prologue prolog{                                       \
-        (uint64_t) __func__,  (uint64_t)__FILE__, sizeof(__func__) - 1, \
-        sizeof(__FILE__) - 1, __LINE__, level};
-#endif
+#define DEFINE_PROLOGUE(level)                                                 \
+    auto _prologue_file_r = TSTRING(__FILE__).reverse();                       \
+    auto _partial_file =                                                       \
+        ConstString::TSpliter<'/', ' ',                                        \
+                              decltype(_prologue_file_r)>::Current::reverse(); \
+    constexpr static Prologue prolog(__func__, _partial_file, __LINE__, level);
 
 #define _IS_LITERAL_STRING(x) \
     (sizeof(#x) > 2 && (#x[0] == '"') && (#x[sizeof(#x) - 2] == '"'))
 
 #define __LOG__(logger, level, first, ...)                             \
     ({                                                                 \
-        DEFINE_PROLOGUE(level, prolog);                                \
+        DEFINE_PROLOGUE(level);                                        \
         auto __build_lambda__ = [&](ILogOutput* __output_##__LINE__) { \
             if (_IS_LITERAL_STRING(first)) {                           \
                 return __log__(level, __output_##__LINE__, prolog,     \
@@ -523,6 +527,12 @@ inline NamedValue<T> make_named_value(const char (&name)[N], T&& value)
     return NamedValue<T> {ALogStringL(name), std::forward<T>(value)};
 }
 
+template <ssize_t N, ssize_t M>
+inline NamedValue<ALogString> make_named_value(const char (&name)[N],
+                                               char (&value)[M]) {
+    return {ALogStringL(name), alog_forwarding(value)};
+}
+
 #define VALUE(x) make_named_value(#x, x)
 
 template<typename T>
diff --git a/thread/thread.cpp b/thread/thread.cpp
index 9ddcd274..5b07e7ff 100644
--- a/thread/thread.cpp
+++ b/thread/thread.cpp
@@ -1033,7 +1033,7 @@ R"(
 
     volatile uint64_t now;
     static std::atomic<pthread_t> ts_updater(0);
-    static inline uint64_t update_now()
+    static inline struct timeval update_now()
     {
 #if defined(__x86_64__) && defined(__linux__) && defined(ENABLE_MIMIC_VDSO)
         if (likely(__mimic_vdso_time_x86))
@@ -1045,7 +1045,7 @@ R"(
         nnow *= 1000 * 1000;
         nnow += tv.tv_usec;
         now = nnow;
-        return nnow;
+        return tv;
     }
     __attribute__((always_inline))
     static inline uint32_t _rdtsc()
@@ -1070,23 +1070,28 @@ R"(
     #endif
     }
     static uint32_t last_tsc = 0;
-    static inline uint64_t if_update_now(bool accurate = false) {
+    static inline void if_update_now(bool accurate = false) {
 #if defined(__x86_64__) && defined(__linux__) && defined(ENABLE_MIMIC_VDSO)
         if (likely(__mimic_vdso_time_x86)) {
             return photon::now = __mimic_vdso_time_x86.get_now(accurate);
         }
 #endif
         if (likely(ts_updater.load(std::memory_order_relaxed))) {
-            return photon::now;
+            return;
+        }
+        if (unlikely(accurate)) {
+            update_now();
+            return;
         }
-        if (unlikely(accurate))
-            return update_now();
         uint32_t tsc = _rdtsc();
         if (unlikely(last_tsc != tsc)) {
             last_tsc = tsc;
-            return update_now();
+            update_now();
         }
-        return photon::now;
+    }
+    struct timeval alog_update_now() {
+        last_tsc = _rdtsc();
+        return update_now();
     }
     int timestamp_updater_init() {
         if (!ts_updater) {

From 27ffd3c4d7f0911180e570a5b87a77c49807646e Mon Sep 17 00:00:00 2001
From: Coldwings <coldwings@me.com>
Date: Wed, 21 Feb 2024 13:31:39 +0800
Subject: [PATCH 48/76] Fix prologue file field length (#369)

Signed-off-by: Coldwings <coldwings@me.com>
---
 common/alog.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common/alog.h b/common/alog.h
index 247bdbd5..f019ba86 100644
--- a/common/alog.h
+++ b/common/alog.h
@@ -353,7 +353,7 @@ struct Prologue
         : addr_func(addr_func_),
           addr_file(addr_file_.chars),
           len_func(N - 1),
-          len_file(addr_file_.len - 1),
+          len_file(addr_file_.len),
           line(line_),
           level(level_) {}
 };

From bac78b5e60243f24fd4959fb74daed78ee55f8df Mon Sep 17 00:00:00 2001
From: Lanzheng Liu <lanzheng.liulz@alibaba-inc.com>
Date: Wed, 21 Feb 2024 12:01:19 +0800
Subject: [PATCH 49/76] fix readdir for localfs

Signed-off-by: Lanzheng Liu <lanzheng.liulz@alibaba-inc.com>
---
 fs/localfs.cpp | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/fs/localfs.cpp b/fs/localfs.cpp
index 2c9dfbc7..e133bfad 100644
--- a/fs/localfs.cpp
+++ b/fs/localfs.cpp
@@ -294,6 +294,8 @@ namespace fs
         ::DIR* dirp;
         ::dirent* direntp;
         long loc;
+        ::dirent m_dirent;
+
         LocalDIR(::DIR* dirp) : dirp(dirp)
         {
             next();
@@ -313,6 +315,10 @@ namespace fs
         }
         virtual dirent* get() override
         {
+            if (direntp) {
+                memcpy(&m_dirent, direntp, sizeof(m_dirent));
+                return &m_dirent;
+            }
             return direntp;
         }
         virtual int next() override

From 1ecf33a416aafa0c57f0c9a70e7d236c927351f2 Mon Sep 17 00:00:00 2001
From: Huiba Li <huiba.lhb@alibaba-inc.com>
Date: Wed, 21 Feb 2024 16:43:16 +0800
Subject: [PATCH 50/76] fix

---
 thread/test/perf_usleepdefer_semaphore.cpp | 68 +++++++++-------------
 1 file changed, 27 insertions(+), 41 deletions(-)

diff --git a/thread/test/perf_usleepdefer_semaphore.cpp b/thread/test/perf_usleepdefer_semaphore.cpp
index 9c056d10..8302345a 100644
--- a/thread/test/perf_usleepdefer_semaphore.cpp
+++ b/thread/test/perf_usleepdefer_semaphore.cpp
@@ -21,12 +21,8 @@ limitations under the License.
 #include <thread>
 #include <photon/common/metric-meter/metrics.h>
 #include <photon/common/alog.h>
-#include <memory>
-
-Metric::AverageLatencyCounter ldefer, lsem;
 
 void* task_defer(void* arg) {
-    SCOPE_LATENCY(*(Metric::AverageLatencyCounter*)(arg));
     auto th = photon::CURRENT;
     photon::thread_usleep_defer(-1, [&]{
         std::thread([&]{
@@ -37,67 +33,57 @@ void* task_defer(void* arg) {
 }
 
 void* task_semaphore(void* arg) {
-    SCOPE_LATENCY(*(Metric::AverageLatencyCounter*)(arg));
-    auto sem = std::make_shared<photon::semaphore>(0);
-    std::thread([&, sem]{
-        sem->signal(1);
+    photon::semaphore sem(0);
+    std::thread([&]{
+        sem.signal(1);
     }).detach();
-    sem->wait(1);
+    sem.wait(1);
     return 0;
 }
 
 void* ph_task_defer(void* arg) {
-    SCOPE_LATENCY(*(Metric::AverageLatencyCounter*)(arg));
     auto th = photon::CURRENT;
     auto task = [&]{
-            photon::thread_interrupt(th);
-        };
+        photon::thread_interrupt(th);
+    };
     photon::thread_usleep_defer(-1, [&]{
-        photon::thread_create11(&decltype(task)::operator(), &task);
+        photon::thread_create11(task);
     });
     return 0;
 }
 
 void* ph_task_semaphore(void* arg) {
-    SCOPE_LATENCY(*(Metric::AverageLatencyCounter*)(arg));
-    auto sem = std::make_shared<photon::semaphore>(0);
-    auto task = [&, sem]{
-            sem->signal(1);
-        };
-    photon::thread_create11(&decltype(task)::operator(), &task);
-    sem->wait(1);
+    photon::semaphore sem(0);
+    auto task = [&]{
+        sem.signal(1);
+    };
+    photon::thread_create11(task);
+    sem.wait(1);
     return 0;
 }
 
-using TestTask = void*(*)(void*);
-template<TestTask task, Metric::AverageLatencyCounter* arg, int ROUND=10000>
-void do_test() {
-    std::vector<photon::join_handle*> jh;
-    for (auto i=0;i<ROUND;i++) {
-        jh.emplace_back(
-            photon::thread_enable_join(
-                photon::thread_create(task, arg)
-            )
-        );
-    }
-    for (auto &j : jh) {
-        photon::thread_join(j);
+const int ROUND = 10000;
+void do_test(ALogStringL name, photon::thread_entry task) {
+    Metric::AverageLatencyCounter lat; {
+        SCOPE_LATENCY(lat);
+        photon::threads_create_join(ROUND, task, nullptr);
     }
+    LOG_INFO("average latency = ` (`)", lat.val(), name);
 }
 
+#define DO_TEST(task) do_test(#task, & task);
+
 TEST(perf, task_in_thread) {
-    do_test<task_defer, &ldefer>();
-    LOG_INFO(VALUE(ldefer.val()));
-    do_test<task_semaphore, &lsem>();
-    LOG_INFO(VALUE(lsem.val()));
+    LOG_INFO(ROUND, " rounds");
+    DO_TEST(task_defer);
+    DO_TEST(task_semaphore);
     photon::thread_usleep(1);
 }
 
 TEST(perf, task_in_photon) {
-    do_test<ph_task_defer, &ldefer>();
-    LOG_INFO(VALUE(ldefer.val()));
-    do_test<ph_task_semaphore, &lsem>();
-    LOG_INFO(VALUE(lsem.val()));
+    LOG_INFO(ROUND, " rounds");
+    DO_TEST(ph_task_defer);
+    DO_TEST(ph_task_semaphore);
     photon::thread_usleep(1);
 }
 

From 95187e02830671cff28f4361dca8f024b3cc88f4 Mon Sep 17 00:00:00 2001
From: Bob Chen <beef9999@qq.com>
Date: Wed, 21 Feb 2024 17:57:53 +0800
Subject: [PATCH 51/76] timer.h remove include alog.h (#371)

---
 common/identity-pool.cpp      | 2 +-
 common/test/test_objcache.cpp | 1 +
 thread/timer.h                | 1 -
 3 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/common/identity-pool.cpp b/common/identity-pool.cpp
index d942d597..75ea8202 100644
--- a/common/identity-pool.cpp
+++ b/common/identity-pool.cpp
@@ -16,7 +16,7 @@ limitations under the License.
 
 #include "identity-pool.h"
 #include "../thread/timer.h"
-// #include "alog.h"
+#include "alog.h"
 
 void* IdentityPoolBase::get()
 {
diff --git a/common/test/test_objcache.cpp b/common/test/test_objcache.cpp
index 0c90cf82..28538607 100644
--- a/common/test/test_objcache.cpp
+++ b/common/test/test_objcache.cpp
@@ -23,6 +23,7 @@ limitations under the License.
 #undef protected
 #include <gtest/gtest.h>
 #include <photon/thread/thread.h>
+#include <photon/common/alog.h>
 
 #include <thread>
 
diff --git a/thread/timer.h b/thread/timer.h
index 32bf504a..c8e3dda5 100644
--- a/thread/timer.h
+++ b/thread/timer.h
@@ -19,7 +19,6 @@ limitations under the License.
 #include <cassert>
 #include <photon/thread/thread.h>
 #include <photon/common/callback.h>
-#include <photon/common/alog.h>
 
 namespace photon
 {

From c3301458c6394676ae6bc9e7342706ccfa2c3f23 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Fri, 23 Feb 2024 13:47:30 +0800
Subject: [PATCH 52/76] Auto PR from release/0.6 to release/0.7 (#376)

* Fix HTTPFSv2 using proxy (#336)

Signed-off-by: Coldwings <coldwings@me.com>

* FIX httpfs_v2 common header for file able to replace by ioctl (#351)

FIX httpfs common header for file able to replace by ioctl

* ALog Prologue now keeps only last part of source path (filename), using us timestamps (#364)

* Fix alog namedvalue print character array

Signed-off-by: Coldwings <coldwings@me.com>

* ALog prologue keeps only filename, uses us timestamp

Signed-off-by: Coldwings <coldwings@me.com>

* make `update_now` returns timeval, so as `alog_update_now()`

Signed-off-by: Coldwings <coldwings@me.com>

* Fix `make_named_value`

Signed-off-by: Coldwings <coldwings@me.com>

* `Prologue use const char* for string instead of uint64_t

Signed-off-by: Coldwings <coldwings@me.com>

* `prologue` with constexpr constructor

Signed-off-by: Coldwings <coldwings@me.com>

* prevent warning for make_named_value in clang

Signed-off-by: Coldwings <coldwings@me.com>

---------

Signed-off-by: Coldwings <coldwings@me.com>

* fix readdir for localfs

Signed-off-by: Lanzheng Liu <lanzheng.liulz@alibaba-inc.com>

* Fix prologue file field length (#369)

Signed-off-by: Coldwings <coldwings@me.com>

* Fix: make_named_tuple maked ref to temporary alogstring (#375)

Signed-off-by: Coldwings <coldwings@me.com>

---------

Signed-off-by: Coldwings <coldwings@me.com>
Signed-off-by: Lanzheng Liu <lanzheng.liulz@alibaba-inc.com>
Co-authored-by: Coldwings <coldwings@me.com>
Co-authored-by: Lanzheng Liu <lanzheng.liulz@alibaba-inc.com>
---
 common/alog.h | 16 +++++-----------
 1 file changed, 5 insertions(+), 11 deletions(-)

diff --git a/common/alog.h b/common/alog.h
index f019ba86..c3a7de17 100644
--- a/common/alog.h
+++ b/common/alog.h
@@ -528,24 +528,18 @@ inline NamedValue<T> make_named_value(const char (&name)[N], T&& value)
 }
 
 template <ssize_t N, ssize_t M>
-inline NamedValue<ALogString> make_named_value(const char (&name)[N],
-                                               char (&value)[M]) {
-    return {ALogStringL(name), alog_forwarding(value)};
+inline NamedValue<const char*> make_named_value(const char (&name)[N],
+                                                char (&value)[M]) {
+    return {ALogStringL(name), value};
 }
 
 #define VALUE(x) make_named_value(#x, x)
 
-template<typename T>
-inline LogBuffer& operator << (LogBuffer& log, const NamedValue<T>& v)
-{
+template <typename T>
+inline LogBuffer& operator<<(LogBuffer& log, const NamedValue<T>& v) {
     return log.printf('[', v.name, '=', v.value, ']');
 }
 
-inline LogBuffer& operator << (LogBuffer& log, const NamedValue<ALogString>& v)
-{
-    return log.printf('[', v.name, '=', '"', v.value, '"', ']');
-}
-
 // output a log message, set errno, then return a value
 // keep errno unchaged if new_errno == 0
 #define LOG_ERROR_RETURN(new_errno, retv, ...) {    \

From 47de87b9ab3a29298423286ce59f3ffda17250f1 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Fri, 1 Mar 2024 17:58:37 +0800
Subject: [PATCH 53/76] Auto PR from release/0.6 to release/0.7 (#385)

* Fix HTTPFSv2 using proxy (#336)

Signed-off-by: Coldwings <coldwings@me.com>

* FIX httpfs_v2 common header for file able to replace by ioctl (#351)

FIX httpfs common header for file able to replace by ioctl

* ALog Prologue now keeps only last part of source path (filename), using us timestamps (#364)

* Fix alog namedvalue print character array

Signed-off-by: Coldwings <coldwings@me.com>

* ALog prologue keeps only filename, uses us timestamp

Signed-off-by: Coldwings <coldwings@me.com>

* make `update_now` returns timeval, so as `alog_update_now()`

Signed-off-by: Coldwings <coldwings@me.com>

* Fix `make_named_value`

Signed-off-by: Coldwings <coldwings@me.com>

* `Prologue use const char* for string instead of uint64_t

Signed-off-by: Coldwings <coldwings@me.com>

* `prologue` with constexpr constructor

Signed-off-by: Coldwings <coldwings@me.com>

* prevent warning for make_named_value in clang

Signed-off-by: Coldwings <coldwings@me.com>

---------

Signed-off-by: Coldwings <coldwings@me.com>

* fix readdir for localfs

Signed-off-by: Lanzheng Liu <lanzheng.liulz@alibaba-inc.com>

* Fix prologue file field length (#369)

Signed-off-by: Coldwings <coldwings@me.com>

* Fix: make_named_tuple maked ref to temporary alogstring (#375)

Signed-off-by: Coldwings <coldwings@me.com>

* Fix compile in gcc492 (#384)

* Fix compile in gcc492, ignore some designed warning with diagnostic pragma

Signed-off-by: Coldwings <coldwings@me.com>

* Only add no-packed-bitfield-compat for GCC

Signed-off-by: Coldwings <coldwings@me.com>

---------

Signed-off-by: Coldwings <coldwings@me.com>

---------

Signed-off-by: Coldwings <coldwings@me.com>
Signed-off-by: Lanzheng Liu <lanzheng.liulz@alibaba-inc.com>
Co-authored-by: Coldwings <coldwings@me.com>
Co-authored-by: Lanzheng Liu <lanzheng.liulz@alibaba-inc.com>
---
 CMakeLists.txt             |  4 ++++
 common/alog.h              | 13 +++++++------
 examples/perf/net-perf.cpp |  1 -
 net/http/client.cpp        |  4 +++-
 net/utils.cpp              |  3 +++
 5 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index f71d6c2f..6f67a2fd 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -61,6 +61,10 @@ set(CMAKE_BUILD_RPATH_USE_ORIGIN ON)
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 set(CMAKE_BUILD_PARALLEL_LEVEL ${NumCPU})
 
+if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-packed-bitfield-compat")
+endif()
+
 if (${ARCH} STREQUAL x86_64)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse4.2")
 elseif (${ARCH} STREQUAL aarch64)
diff --git a/common/alog.h b/common/alog.h
index c3a7de17..c1ca1790 100644
--- a/common/alog.h
+++ b/common/alog.h
@@ -347,13 +347,13 @@ struct Prologue
     int len_func, len_file;
     int line, level;
 
-    template <size_t N, typename FILEN>
-    constexpr Prologue(const char (&addr_func_)[N], FILEN addr_file_, int line_,
-                       int level_)
+    template <size_t N, size_t M>
+    constexpr Prologue(const char (&addr_func_)[N], const char (&addr_file_)[M],
+                       int line_, int level_)
         : addr_func(addr_func_),
-          addr_file(addr_file_.chars),
+          addr_file(addr_file_),
           len_func(N - 1),
-          len_file(addr_file_.len),
+          len_file(M - 1),
           line(line_),
           level(level_) {}
 };
@@ -456,7 +456,8 @@ struct LogBuilder {
     auto _partial_file =                                                       \
         ConstString::TSpliter<'/', ' ',                                        \
                               decltype(_prologue_file_r)>::Current::reverse(); \
-    constexpr static Prologue prolog(__func__, _partial_file, __LINE__, level);
+    constexpr static Prologue prolog(__func__, _partial_file.chars, __LINE__,  \
+                                     level);
 
 #define _IS_LITERAL_STRING(x) \
     (sizeof(#x) > 2 && (#x[0] == '"') && (#x[sizeof(#x) - 2] == '"'))
diff --git a/examples/perf/net-perf.cpp b/examples/perf/net-perf.cpp
index c2b94edd..eb24e28a 100644
--- a/examples/perf/net-perf.cpp
+++ b/examples/perf/net-perf.cpp
@@ -35,7 +35,6 @@ DEFINE_uint64(port, 9527, "port");
 DEFINE_uint64(buf_size, 512, "buffer size");
 DEFINE_uint64(vcpu_num, 1, "server vcpu num. Increase this value to enable multi-vcpu scheduling");
 
-static int event_engine = 0;
 static bool stop_test = false;
 static uint64_t qps = 0;
 static uint64_t time_cost = 0;
diff --git a/net/http/client.cpp b/net/http/client.cpp
index 7c9cf547..8271b660 100644
--- a/net/http/client.cpp
+++ b/net/http/client.cpp
@@ -120,7 +120,9 @@ class ClientImpl : public Client {
     CommonHeaders<> m_common_headers;
     ICookieJar *m_cookie_jar;
     ClientImpl(ICookieJar *cookie_jar, TLSContext *tls_ctx) :
-        m_dialer(tls_ctx), m_cookie_jar(cookie_jar) { }
+        m_dialer(tls_ctx),
+        m_cookie_jar(cookie_jar) {
+    }
 
     using SocketStream_ptr = std::unique_ptr<ISocketStream>;
     int redirect(Operation* op) {
diff --git a/net/utils.cpp b/net/utils.cpp
index f937c200..f90e49a0 100644
--- a/net/utils.cpp
+++ b/net/utils.cpp
@@ -118,7 +118,10 @@ void base64_translate_3to4(const char *in, char *out)  {
     static const unsigned char tbl[] =
         "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
     auto v = htonl(*(uint32_t *)in);
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wstrict-aliasing"
     auto x = (xlator*) &v;
+#pragma GCC diagnostic pop
     *(uint32_t *)out = ((tbl[x->a] << 24) + (tbl[x->b] << 16) +
                         (tbl[x->c] << 8) + (tbl[x->d] << 0));
 }

From f65027d87430c7e0b0a703917f822d4779b799bd Mon Sep 17 00:00:00 2001
From: Bob Chen <beef9999@qq.com>
Date: Sun, 3 Mar 2024 21:34:49 +0800
Subject: [PATCH 54/76] Fix merge

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 2b79868c..67c1634d 100644
--- a/README.md
+++ b/README.md
@@ -35,3 +35,4 @@
 <img src="/doc/static/img/slack.svg" width="20"> Join Slack: [link](https://join.slack.com/t/photonlibos/shared_invite/zt-25wauq8g1-iK_oHMrXetcvWNNhIt8Nkg)
 
 <img src="/doc/static/img/dingtalk.svg" width="20"> Join DingTalk group: 55690000272
+

From ada6e2d4075cb742e992abc5e5bf7d83ca0ea885 Mon Sep 17 00:00:00 2001
From: Bob Chen <beef9999@qq.com>
Date: Sat, 2 Mar 2024 18:48:33 +0800
Subject: [PATCH 55/76] Fix endpoint undefined situation

---
 net/kernel_socket.cpp  | 2 --
 net/test/test-ipv6.cpp | 7 +++++++
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/net/kernel_socket.cpp b/net/kernel_socket.cpp
index ada1c05a..d9c851b7 100644
--- a/net/kernel_socket.cpp
+++ b/net/kernel_socket.cpp
@@ -1043,8 +1043,6 @@ EndPoint::EndPoint(const char* _ep) {
     // Detect IPv6 or IPv4
     estring ip_str = ep[pos - 1] == ']' ? ep.substr(1, pos - 2) : ep.substr(0, pos);
     auto ip = IPAddr(ip_str.c_str());
-    if (ip.undefined())
-        return;
     auto port_str = ep.substr(pos + 1);
     if (!port_str.all_digits())
         return;
diff --git a/net/test/test-ipv6.cpp b/net/test/test-ipv6.cpp
index bcc03f4c..7e989c39 100644
--- a/net/test/test-ipv6.cpp
+++ b/net/test/test-ipv6.cpp
@@ -14,6 +14,13 @@ TEST(ipv6, endpoint) {
     EXPECT_FALSE(c.undefined());
     c = photon::net::EndPoint("[::1]:8888");
     EXPECT_FALSE(c.undefined());
+    c = photon::net::EndPoint("0.0.0.0:8888");
+    EXPECT_TRUE(c.undefined());
+    EXPECT_EQ(8888, c.port);
+    c = photon::net::EndPoint("::", 8888);
+    EXPECT_TRUE(!c.undefined());
+    EXPECT_TRUE(!c.is_ipv4());
+    EXPECT_EQ(8888, c.port);
 }
 
 TEST(ipv6, addr) {

From 930ae4951b4f51fd3373c24343de8053425be823 Mon Sep 17 00:00:00 2001
From: Bob Chen <beef9999@qq.com>
Date: Tue, 5 Mar 2024 14:45:29 +0800
Subject: [PATCH 56/76] throttle support update

---
 common/throttle.h | 30 ++++++++++++++++++------------
 1 file changed, 18 insertions(+), 12 deletions(-)

diff --git a/common/throttle.h b/common/throttle.h
index 9e8fd646..7585d67d 100644
--- a/common/throttle.h
+++ b/common/throttle.h
@@ -8,16 +8,16 @@ class throttle {
     photon::semaphore sem;
     uint64_t last_retrieve = 0;
     uint64_t m_limit = -1UL;
-    uint64_t m_limit_slice;
+    uint64_t m_limit_per_slice;
     uint64_t m_time_window;
-    uint64_t m_time_slice;
+    uint64_t m_time_window_per_slice;
     uint64_t m_slice_num;
 
     void try_signal() {
         auto duration = photon::now - last_retrieve;
         if (duration > m_time_window) duration = m_time_window;
-        if (duration >= m_time_slice) {
-            auto free = m_limit_slice * (duration / m_time_slice);
+        if (duration >= m_time_window_per_slice) {
+            auto free = m_limit_per_slice * (duration / m_time_window_per_slice);
             auto current = photon::sat_sub(m_limit, sem.count());
             if (current < free) {
                 free = current;
@@ -28,22 +28,28 @@ class throttle {
     }
 
 public:
-    throttle(uint64_t limit, uint64_t time_window = 1000UL * 1000,
-             uint64_t slice = 10) {
-        m_slice_num = slice;
-        m_limit_slice = photon::sat_add(limit, (slice - 1)) / slice;
-        m_limit = m_limit_slice * slice;
-        m_time_slice = photon::sat_add(time_window, (slice - 1)) / slice;
-        m_time_window = m_time_slice * slice;
+    /**
+     * @param limit -1UL means no limit, 0 means lowest speed (hang)
+     */
+    explicit throttle(uint64_t limit, uint64_t time_window = 1000UL * 1000,
+             uint64_t slice = 10) : m_slice_num(slice) {
+        update(limit);
+        m_time_window_per_slice = photon::sat_add(time_window, (m_slice_num - 1)) / m_slice_num;
+        m_time_window = m_time_window_per_slice * m_slice_num;
         sem.signal(m_limit);
     }
 
+    void update(uint64_t limit) {
+        m_limit_per_slice = photon::sat_add(limit, (m_slice_num - 1)) / m_slice_num;
+        m_limit = m_limit_per_slice * m_slice_num;
+    }
+
     int consume(uint64_t amount) {
         int ret = 0;
         int err = 0;
         do {
             try_signal();
-            ret = sem.wait(amount, m_time_slice);
+            ret = sem.wait(amount, m_time_window_per_slice);
             err = errno;
         } while (ret < 0 && err == ETIMEDOUT);
         if (ret < 0) {

From ecbb2fa7fb02057e4c45d74499afaffcbda4b6af Mon Sep 17 00:00:00 2001
From: Bob Chen <beef9999@qq.com>
Date: Fri, 15 Mar 2024 17:55:33 +0800
Subject: [PATCH 57/76] fix test case ssl include dir

---
 net/http/test/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/net/http/test/CMakeLists.txt b/net/http/test/CMakeLists.txt
index 810990f1..582c4269 100644
--- a/net/http/test/CMakeLists.txt
+++ b/net/http/test/CMakeLists.txt
@@ -26,5 +26,6 @@ target_link_libraries(headers_test PRIVATE photon_shared)
 add_test(NAME headers_test COMMAND $<TARGET_FILE:headers_test>)
 
 add_executable(client_tls_test client_tls_test.cpp)
+target_include_directories(client_tls_test PRIVATE ${OPENSSL_INCLUDE_DIRS})
 target_link_libraries(client_tls_test PRIVATE photon_shared ${testing_libs})
 add_test(NAME client_tls_test COMMAND $<TARGET_FILE:client_tls_test>)

From a7597a918110d4e48ad2d0e93bec6c84a491e760 Mon Sep 17 00:00:00 2001
From: Huiba Li <huiba.lhb@alibaba-inc.com>
Date: Sun, 17 Mar 2024 16:21:12 +0800
Subject: [PATCH 58/76] Realize true one-shot semantics for epoll with
 EPOLLONESHOT (#410)

* realize ONE_SHOT for epoll

* fix

* fix

* improve branching

* fix

* fix
---
 io/epoll.cpp      | 174 +++++++++++++++++++++++++---------------------
 io/fd-events.h    |   3 +-
 net/test/test.cpp |  14 ++--
 net/utils.h       |   2 +-
 4 files changed, 104 insertions(+), 89 deletions(-)

diff --git a/io/epoll.cpp b/io/epoll.cpp
index 4100bd69..ca9a2493 100644
--- a/io/epoll.cpp
+++ b/io/epoll.cpp
@@ -110,62 +110,81 @@ class EventEngineEPoll : public MasterEventEngine, public CascadingEventEngine,
         }
         return 0;
     }
+
     std::vector<InFlightEvent> _inflight_events;
     virtual int add_interest(Event e) override {
         if (e.fd < 0)
             LOG_ERROR_RETURN(EINVAL, -1, "invalid file descriptor ", e.fd);
-        if ((size_t)e.fd >= _inflight_events.size())
+        if (unlikely((size_t)e.fd >= _inflight_events.size()))
             _inflight_events.resize(e.fd * 2);
         auto& entry = _inflight_events[e.fd];
-        if (e.interests & entry.interests) {
-            if (((e.interests & entry.interests & EVENT_READ) &&
-                 (entry.reader_data != e.data)) ||
-                ((e.interests & entry.interests & EVENT_WRITE) &&
-                 (entry.writer_data != e.data)) ||
-                ((e.interests & entry.interests & EVENT_ERROR) &&
-                 (entry.error_data != e.data))) {
-                LOG_ERROR_RETURN(EALREADY, -1, "conflicted interest(s)");
+        auto intersection = e.interests & entry.interests & EVENT_RWE;
+        auto data = (entry.reader_data != e.data) * EVENT_READ  |
+                    (entry.writer_data != e.data) * EVENT_WRITE |
+                    (entry.error_data  != e.data) * EVENT_ERROR ;
+        if (intersection & data)
+            LOG_ERROR_RETURN(EALREADY, -1, "conflicted interest(s)");
+
+        int ret;
+        auto eint = entry.interests;
+        auto x = (eint | e.interests) & EVENT_RWE;
+        auto events = evmap.translate_bitwisely(x);
+        if (likely(e.interests & ONE_SHOT)) {
+            events |= EPOLLONESHOT;
+            if (likely(eint & ONE_SHOT)) {
+                ret = ctl(e.fd, EPOLL_CTL_MOD, events);
+                if (unlikely(ret < 0 && errno == ENOENT))
+                    ret = ctl(e.fd, EPOLL_CTL_ADD, events);
+            } else {
+                if (eint != 0) LOG_ERROR_RETURN(EINVAL, -1, "conflicted interest(s) regarding ONE_SHOT");
+                ret = ctl(e.fd, EPOLL_CTL_ADD, events);
             }
+        } else {
+            auto op = (eint & EVENT_RWE) ? EPOLL_CTL_MOD : EPOLL_CTL_ADD;
+            ret = ctl(e.fd, op, events);
         }
+        if (ret < 0)
+            LOG_ERROR_RETURN(0, ret, "failed to add_interest()");
 
-        if (e.interests & EVENT_READ) entry.reader_data = e.data;
+        entry.interests |= e.interests;
+        if (e.interests & EVENT_READ)  entry.reader_data = e.data;
         if (e.interests & EVENT_WRITE) entry.writer_data = e.data;
-        if (e.interests & EVENT_ERROR) entry.error_data = e.data;
-        auto eint = entry.interests & (EVENT_READ | EVENT_WRITE | EVENT_ERROR);
-        auto op = eint ? EPOLL_CTL_MOD : EPOLL_CTL_ADD;
-        if (op == EPOLL_CTL_MOD &&
-            (e.interests & ONE_SHOT) != (entry.interests & ONE_SHOT)) {
-            LOG_ERROR_RETURN(
-                EINVAL, -1,
-                "do not support ONE_SHOT on no-oneshot interested fd");
-        }
-        auto x = entry.interests |= e.interests;
-        x &= (EVENT_READ | EVENT_WRITE | EVENT_ERROR);
-        // since epoll oneshot shows totally different meanning of ONESHOT in
-        // photon all epoll action keeps no oneshot
-        auto events = evmap.translate_bitwisely(x);
-        return ctl(e.fd, op, events);
+        if (e.interests & EVENT_ERROR) entry.error_data  = e.data;
+        return 0;
     }
+
     virtual int rm_interest(Event e) override {
         if (e.fd < 0 || (size_t)e.fd >= _inflight_events.size())
             LOG_ERROR_RETURN(EINVAL, -1, "invalid file descriptor ", e.fd);
+        if (unlikely(!e.interests)) return 0;
         auto& entry = _inflight_events[e.fd];
-        auto intersection = e.interests & entry.interests &
-                            (EVENT_READ | EVENT_WRITE | EVENT_ERROR);
+        auto intersection = e.interests & entry.interests & EVENT_RWE;
         if (intersection == 0) return 0;
 
-        auto x = (entry.interests ^= intersection) &
-                 (EVENT_READ | EVENT_WRITE | EVENT_ERROR);
-        if (e.interests & EVENT_READ) entry.reader_data = nullptr;
-        if (e.interests & EVENT_WRITE) entry.writer_data = nullptr;
-        if (e.interests & EVENT_ERROR) entry.error_data = nullptr;
-        auto op = x ? EPOLL_CTL_MOD : EPOLL_CTL_DEL;
-        auto events = evmap.translate_bitwisely(x);
-        if (op == EPOLL_CTL_DEL) {
-            entry.interests = 0;
+        int ret, op = 0;    // ^ is to flip intersected bits
+        auto x = (entry.interests ^ intersection) & EVENT_RWE;
+        if (likely(e.interests & ONE_SHOT)) {
+            if (!x) {
+                ret = 0; // no need to epoll_ctl()
+            } else {
+                auto events = evmap.translate_bitwisely(x);
+                ret = ctl(e.fd, EPOLL_CTL_MOD, events); // re-arm other interests
+            }
+        } else {
+            op = x ? EPOLL_CTL_MOD : EPOLL_CTL_DEL;
+            auto events = evmap.translate_bitwisely(x);
+            ret = ctl(e.fd, op, events);
         }
-        return ctl(e.fd, op, events);
+        if (ret < 0)
+            LOG_ERROR_RETURN(0, ret, "failed to rm_interest()");
+                                                    // ^ is to flip intersected bits
+        entry.interests = (op == EPOLL_CTL_DEL) ? 0 : (entry.interests ^ intersection);
+        if (intersection & EVENT_READ)  entry.reader_data = nullptr;
+        if (intersection & EVENT_WRITE) entry.writer_data = nullptr;
+        if (intersection & EVENT_ERROR) entry.error_data  = nullptr;
+        return 0;
     }
+
     epoll_event _events[16];
     uint16_t _events_remain = 0;
     int do_epoll_wait(uint64_t timeout) {
@@ -210,32 +229,23 @@ class EventEngineEPoll : public MasterEventEngine, public CascadingEventEngine,
             assert(e.data.u64 < _inflight_events.size());
             if (e.data.u64 >= _inflight_events.size()) continue;
             auto& entry = _inflight_events[e.data.u64];
+            uint32_t events = 0;
             if ((e.events & ERRBIT) && (entry.interests & EVENT_ERROR)) {
-                auto data = entry.error_data;
-                if (entry.interests & ONE_SHOT) {
-                    rm_interest({.fd = (int)e.data.u64,
-                                 .interests = EVENT_ERROR | ONE_SHOT,
-                                 .data = nullptr});
-                }
-                datacb(data);
+                events |= EVENT_ERROR;
+                datacb(entry.error_data);
             }
             if ((e.events & READBITS) && (entry.interests & EVENT_READ)) {
-                auto data = entry.reader_data;
-                if (entry.interests & ONE_SHOT) {
-                    rm_interest({.fd = (int)e.data.u64,
-                                 .interests = EVENT_READ | ONE_SHOT,
-                                 .data = nullptr});
-                }
-                datacb(data);
+                events |= EVENT_READ;
+                datacb(entry.reader_data);
             }
             if ((e.events & WRITEBITS) && (entry.interests & EVENT_WRITE)) {
-                auto data = entry.writer_data;
-                if (entry.interests & ONE_SHOT) {
-                    rm_interest({.fd = (int)e.data.u64,
-                                 .interests = EVENT_WRITE | ONE_SHOT,
-                                 .data = nullptr});
-                }
-                datacb(data);
+                events |= EVENT_WRITE;
+                datacb(entry.writer_data);
+            }
+            if (events && (entry.interests & ONE_SHOT)) {
+                rm_interest({.fd = (int)e.data.u64,
+                             .interests = events | ONE_SHOT,
+                             .data = nullptr});
             }
         }
     }
@@ -248,12 +258,11 @@ class EventEngineEPoll : public MasterEventEngine, public CascadingEventEngine,
         }
         auto ptr = data;
         auto end = data + count;
-        wait_for_events(
-            0, [&](void* data) __INLINE__ { *ptr++ = data; },
-            [&]()
-                __INLINE__ {  // make sure each fd receives all possible events
-                    return (end - ptr) >= 3;
-                });
+        wait_for_events(0,  // pass timeout as 0 to avoid another wait
+            [&](void* data) __INLINE__ { *ptr++ = data; },
+            [&]() __INLINE__ {  // make sure each fd receives all possible events
+                return (end - ptr) >= 3;
+            });
         if (ptr == data) {
             return 0;
         }
@@ -261,8 +270,7 @@ class EventEngineEPoll : public MasterEventEngine, public CascadingEventEngine,
     }
     virtual ssize_t wait_and_fire_events(uint64_t timeout = -1) override {
         ssize_t n = 0;
-        wait_for_events(
-            timeout,
+        wait_for_events(timeout,
             [&](void* data) __INLINE__ {
                 assert(data);
                 thread_interrupt((thread*)data, EOK);
@@ -273,35 +281,39 @@ class EventEngineEPoll : public MasterEventEngine, public CascadingEventEngine,
     }
     virtual int cancel_wait() override { return eventfd_write(_evfd, 1); }
 
-    int wait_for_fd(int fd, uint32_t interests, uint64_t timeout) override {
-        Event event{fd, interests | ONE_SHOT, CURRENT};
-        int ret = add_interest(event);
+    int wait_for_fd(int fd, uint32_t interest, uint64_t timeout) override {
+        if (fd < 0)
+            LOG_ERROR_RETURN(EINVAL, -1, "invalid fd");
+        if (interest & (interest-1))
+            LOG_ERROR_RETURN(EINVAL, -1, "can not wait for multiple interests");
+        if (unlikely(!interest))
+            return rm_interest({fd, EVENT_RWE, 0}); // remove fd from epoll
+        int ret = add_interest({fd, interest | ONE_SHOT, CURRENT});
         if (ret < 0) LOG_ERROR_RETURN(0, -1, "failed to add event interest");
         ret = thread_usleep(timeout);
         ERRNO err;
         if (ret == -1 && err.no == EOK) {
             return 0;  // Event arrived
-        } else if (ret == 0) {
-            rm_interest(event);  // Timeout
-            errno = ETIMEDOUT;
-            return -1;
-        } else {
-            rm_interest(event);  // Interrupted by other thread
-            errno = err.no;
-            return -1;
         }
+        rm_interest({fd, interest, 0}); // no ONE_SHOT, to reconfig epoll
+        errno = (ret == 0) ? ETIMEDOUT :    // Timeout
+                             err.no; // Interrupted by other thread
+        return -1;
     }
 };
 
-__attribute__((noinline)) static EventEngineEPoll* new_epoll_engine() {
-    LOG_INFO("Init event engine: epoll");
+__attribute__((noinline)) static
+EventEngineEPoll* new_epoll_engine(ALogStringL role) {
+    LOG_INFO("Init epoll event engine: ", role);
     return NewObj<EventEngineEPoll>()->init();
 }
 
-MasterEventEngine* new_epoll_master_engine() { return new_epoll_engine(); }
+MasterEventEngine* new_epoll_master_engine() {
+    return new_epoll_engine("master");
+}
 
 CascadingEventEngine* new_epoll_cascading_engine() {
-    return new_epoll_engine();
+    return new_epoll_engine("cascading");
 }
 
 }  // namespace photon
diff --git a/io/fd-events.h b/io/fd-events.h
index 0bcd9986..d201a5be 100644
--- a/io/fd-events.h
+++ b/io/fd-events.h
@@ -24,6 +24,7 @@ namespace photon {
 const static uint32_t EVENT_READ = 1;
 const static uint32_t EVENT_WRITE = 2;
 const static uint32_t EVENT_ERROR = 4;
+const static uint32_t EVENT_RWE = EVENT_READ | EVENT_WRITE | EVENT_ERROR;
 const static uint32_t EDGE_TRIGGERED = 0x4000;
 const static uint32_t ONE_SHOT = 0x8000;
 
@@ -44,7 +45,7 @@ class MasterEventEngine {
      * @return 0 for success, which means event arrived in time
      *         -1 for failure, could be timeout or interrupted by another thread
      */
-    virtual int wait_for_fd(int fd, uint32_t interests, uint64_t timeout) = 0;
+    virtual int wait_for_fd(int fd, uint32_t interest, uint64_t timeout) = 0;
 
     int wait_for_fd_readable(int fd, uint64_t timeout = -1) {
         return wait_for_fd(fd, EVENT_READ, timeout);
diff --git a/net/test/test.cpp b/net/test/test.cpp
index ad182a06..dcf755fe 100644
--- a/net/test/test.cpp
+++ b/net/test/test.cpp
@@ -695,18 +695,20 @@ void* start_server(void*) {
 
 TEST(utils, gethostbyname) {
     net::IPAddr localhost("127.0.0.1");
-    net::IPAddr addr;
-    net::gethostbyname("localhost", &addr);
-    EXPECT_EQ(localhost.to_nl(), addr.to_nl());
     std::vector<net::IPAddr> addrs;
     net::gethostbyname("localhost", addrs);
     EXPECT_GT((int)addrs.size(), 0);
-    EXPECT_EQ(localhost.to_nl(), addrs[0].to_nl());
+
     net::IPAddr host = net::gethostbypeer("localhost");
-    EXPECT_EQ(localhost.to_nl(), host.to_nl());
-    for (auto &x : addrs) {
+    bool found_localhost = false, found_host = false;
+    for (auto& x: addrs) {
         LOG_INFO(VALUE(x));
+        EXPECT_TRUE(x.is_loopback());
+        found_localhost |= (x == localhost);
+        found_host |= (x == host);
     }
+    EXPECT_TRUE(found_localhost);
+    EXPECT_TRUE(found_host);
 }
 
 TEST(utils, resolver) {
diff --git a/net/utils.h b/net/utils.h
index bf61c3b0..7b73625e 100644
--- a/net/utils.h
+++ b/net/utils.h
@@ -94,7 +94,7 @@ inline int gethostbyname(const char* name, IPAddr* buf, int bufsize = 1) {
     int i = 0;
     auto cb = [&](IPAddr addr) {
         if (i < bufsize) buf[i++] = addr;
-        return 0;
+        return (i < bufsize) ? 0 : -1;
     };
     return _gethostbyname(name, cb);
 }

From a9e311fd6c0f800228e1f0f24748a46ff5d84d6f Mon Sep 17 00:00:00 2001
From: Bob Chen <beef9999@qq.com>
Date: Mon, 18 Mar 2024 17:20:27 +0800
Subject: [PATCH 59/76] When rm_interest needs to re-arm other interests, add
 EPOLLONESHOT

---
 io/epoll.cpp           | 6 ++++--
 io/iouring-wrapper.cpp | 2 ++
 io/kqueue.cpp          | 2 ++
 net/kernel_socket.cpp  | 1 +
 4 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/io/epoll.cpp b/io/epoll.cpp
index ca9a2493..42d9e6ab 100644
--- a/io/epoll.cpp
+++ b/io/epoll.cpp
@@ -156,7 +156,7 @@ class EventEngineEPoll : public MasterEventEngine, public CascadingEventEngine,
     virtual int rm_interest(Event e) override {
         if (e.fd < 0 || (size_t)e.fd >= _inflight_events.size())
             LOG_ERROR_RETURN(EINVAL, -1, "invalid file descriptor ", e.fd);
-        if (unlikely(!e.interests)) return 0;
+        if (unlikely(e.interests == 0)) return 0;
         auto& entry = _inflight_events[e.fd];
         auto intersection = e.interests & entry.interests & EVENT_RWE;
         if (intersection == 0) return 0;
@@ -164,10 +164,12 @@ class EventEngineEPoll : public MasterEventEngine, public CascadingEventEngine,
         int ret, op = 0;    // ^ is to flip intersected bits
         auto x = (entry.interests ^ intersection) & EVENT_RWE;
         if (likely(e.interests & ONE_SHOT)) {
+            // If e is ONE_SHOT, the entry must be ONE_SHOT as well
             if (!x) {
                 ret = 0; // no need to epoll_ctl()
             } else {
                 auto events = evmap.translate_bitwisely(x);
+                events |= EPOLLONESHOT;
                 ret = ctl(e.fd, EPOLL_CTL_MOD, events); // re-arm other interests
             }
         } else {
@@ -286,7 +288,7 @@ class EventEngineEPoll : public MasterEventEngine, public CascadingEventEngine,
             LOG_ERROR_RETURN(EINVAL, -1, "invalid fd");
         if (interest & (interest-1))
             LOG_ERROR_RETURN(EINVAL, -1, "can not wait for multiple interests");
-        if (unlikely(!interest))
+        if (unlikely(interest == 0))
             return rm_interest({fd, EVENT_RWE, 0}); // remove fd from epoll
         int ret = add_interest({fd, interest | ONE_SHOT, CURRENT});
         if (ret < 0) LOG_ERROR_RETURN(0, -1, "failed to add event interest");
diff --git a/io/iouring-wrapper.cpp b/io/iouring-wrapper.cpp
index ab3aadd1..e1b0329e 100644
--- a/io/iouring-wrapper.cpp
+++ b/io/iouring-wrapper.cpp
@@ -224,6 +224,8 @@ class iouringEngine : public MasterEventEngine, public CascadingEventEngine, pub
     }
 
     int wait_for_fd(int fd, uint32_t interests, uint64_t timeout) override {
+        if (unlikely(interests == 0))
+            return 0;
         unsigned poll_mask = evmap.translate_bitwisely(interests);
         // The io_uring_prep_poll_add's return value is the same as poll(2)'s revents.
         int ret = async_io(&io_uring_prep_poll_add, timeout, 0, fd, poll_mask);
diff --git a/io/kqueue.cpp b/io/kqueue.cpp
index 4e314585..aea5d63f 100644
--- a/io/kqueue.cpp
+++ b/io/kqueue.cpp
@@ -88,6 +88,8 @@ class KQueue : public MasterEventEngine, public CascadingEventEngine, public Res
     }
 
     int wait_for_fd(int fd, uint32_t interests, uint64_t timeout) override {
+        if (unlikely(interests == 0))
+            return 0;
         short ev = (interests == EVENT_READ) ? EVFILT_READ : EVFILT_WRITE;
         enqueue(fd, ev, EV_ADD | EV_ONESHOT, 0, CURRENT);
         int ret = thread_usleep(timeout);
diff --git a/net/kernel_socket.cpp b/net/kernel_socket.cpp
index d9c851b7..02aa8bfe 100644
--- a/net/kernel_socket.cpp
+++ b/net/kernel_socket.cpp
@@ -135,6 +135,7 @@ class KernelSocketStream : public SocketStreamBase {
         return (Object*) (uint64_t) fd;
     }
     int close() final {
+        get_vcpu()->master_event_engine->wait_for_fd(fd, 0, -1UL);
         auto ret = ::close(fd);
         fd = -1;
         return ret;

From a34a0fe6b21fa58833a0d7a47f726cd755e2575b Mon Sep 17 00:00:00 2001
From: lihuiba <lihuiba@gmail.com>
Date: Tue, 19 Mar 2024 21:49:51 +0800
Subject: [PATCH 60/76] fix_epoll_oneshot

---
 io/epoll.cpp | 36 ++++++++++++++++--------------------
 1 file changed, 16 insertions(+), 20 deletions(-)

diff --git a/io/epoll.cpp b/io/epoll.cpp
index 42d9e6ab..ded985f4 100644
--- a/io/epoll.cpp
+++ b/io/epoll.cpp
@@ -158,29 +158,25 @@ class EventEngineEPoll : public MasterEventEngine, public CascadingEventEngine,
             LOG_ERROR_RETURN(EINVAL, -1, "invalid file descriptor ", e.fd);
         if (unlikely(e.interests == 0)) return 0;
         auto& entry = _inflight_events[e.fd];
-        auto intersection = e.interests & entry.interests & EVENT_RWE;
+        auto eint = entry.interests & (EVENT_RWE | ONE_SHOT);
+        auto intersection = e.interests & eint;
         if (intersection == 0) return 0;
 
-        int ret, op = 0;    // ^ is to flip intersected bits
-        auto x = (entry.interests ^ intersection) & EVENT_RWE;
-        if (likely(e.interests & ONE_SHOT)) {
-            // If e is ONE_SHOT, the entry must be ONE_SHOT as well
-            if (!x) {
-                ret = 0; // no need to epoll_ctl()
+        auto remain = eint ^ intersection;  // ^ is to flip intersected bits
+        if (remain == ONE_SHOT) {/* no need to epoll_ctl() */} else {
+            uint32_t op, events = 0;
+            if (remain) {
+                events = evmap.translate_bitwisely(remain);
+                if (remain & ONE_SHOT) events |= EPOLLONESHOT;
+                op = EPOLL_CTL_MOD;
             } else {
-                auto events = evmap.translate_bitwisely(x);
-                events |= EPOLLONESHOT;
-                ret = ctl(e.fd, EPOLL_CTL_MOD, events); // re-arm other interests
+                op = EPOLL_CTL_DEL;
             }
-        } else {
-            op = x ? EPOLL_CTL_MOD : EPOLL_CTL_DEL;
-            auto events = evmap.translate_bitwisely(x);
-            ret = ctl(e.fd, op, events);
+           if (ctl(e.fd, op, events) < 0)
+               LOG_ERROR_RETURN(0, -1, "failed to rm_interest()");
         }
-        if (ret < 0)
-            LOG_ERROR_RETURN(0, ret, "failed to rm_interest()");
-                                                    // ^ is to flip intersected bits
-        entry.interests = (op == EPOLL_CTL_DEL) ? 0 : (entry.interests ^ intersection);
+
+        entry.interests ^= intersection;
         if (intersection & EVENT_READ)  entry.reader_data = nullptr;
         if (intersection & EVENT_WRITE) entry.writer_data = nullptr;
         if (intersection & EVENT_ERROR) entry.error_data  = nullptr;
@@ -246,7 +242,7 @@ class EventEngineEPoll : public MasterEventEngine, public CascadingEventEngine,
             }
             if (events && (entry.interests & ONE_SHOT)) {
                 rm_interest({.fd = (int)e.data.u64,
-                             .interests = events | ONE_SHOT,
+                             .interests = events,
                              .data = nullptr});
             }
         }
@@ -289,7 +285,7 @@ class EventEngineEPoll : public MasterEventEngine, public CascadingEventEngine,
         if (interest & (interest-1))
             LOG_ERROR_RETURN(EINVAL, -1, "can not wait for multiple interests");
         if (unlikely(interest == 0))
-            return rm_interest({fd, EVENT_RWE, 0}); // remove fd from epoll
+            return rm_interest({fd, EVENT_RWE| ONE_SHOT, 0}); // remove fd from epoll
         int ret = add_interest({fd, interest | ONE_SHOT, CURRENT});
         if (ret < 0) LOG_ERROR_RETURN(0, -1, "failed to add event interest");
         ret = thread_usleep(timeout);

From fa89fc09b681e97ccc295362b83789aae3fb10ea Mon Sep 17 00:00:00 2001
From: Huiba Li <huiba.lhb@alibaba-inc.com>
Date: Thu, 21 Mar 2024 14:46:25 +0800
Subject: [PATCH 61/76] add judgement for conflicted ONE_SHOT flag in
 add_interests() (#422)

improve logic in add_interests() and rm_interests()
---
 io/epoll.cpp | 99 ++++++++++++++++++++++++++++++----------------------
 1 file changed, 57 insertions(+), 42 deletions(-)

diff --git a/io/epoll.cpp b/io/epoll.cpp
index ded985f4..7029606c 100644
--- a/io/epoll.cpp
+++ b/io/epoll.cpp
@@ -49,6 +49,8 @@ struct InFlightEvent {
     void* error_data;
 };
 
+const static uint32_t EVENT_RWEO = EVENT_RWE | ONE_SHOT;
+
 class EventEngineEPoll : public MasterEventEngine, public CascadingEventEngine, public ResetHandle {
 public:
     int _evfd = -1;
@@ -89,24 +91,25 @@ class EventEngineEPoll : public MasterEventEngine, public CascadingEventEngine,
         DEFER(fd = -1);
         return close(fd);
     }
-    int ctl(int fd, int op, uint32_t events, int no_log_errno_1 = 0,
-            int no_log_errno_2 = 0) {
+    int ctl(int fd, int op, uint32_t events,
+            int ignore_error_1 = 0, int ignore_error_2 = 0) {
         struct epoll_event ev;
         ev.events = events;  // EPOLLERR | EPOLLHUP always included
         ev.data.u64 = fd;
         int ret = epoll_ctl(_engine_fd, op, fd, &ev);
         if (ret < 0) {
             ERRNO err;
-            if (err.no != no_log_errno_1 &&
-                err.no != no_log_errno_2) {  // deleting a non-existing fd is
-                                             // considered OK
+            // some errno may be ignored, such as deleting a non-existing fd, etc.
+            if ((ignore_error_1 == 0 || ignore_error_1 != err.no) &&
+                (ignore_error_2 == 0 || ignore_error_2 != err.no)) {
                 auto events = HEX(ev.events);
                 auto data = ev.data.ptr;
                 LOG_WARN("failed to call epoll_ctl(`, `, `, {`, `})",
                          VALUE(_engine_fd), VALUE(op), VALUE(fd), VALUE(events),
                          VALUE(data), err);
+                return -err.no;
             }
-            return -err.no;
+            return 1; // error ignored
         }
         return 0;
     }
@@ -115,38 +118,51 @@ class EventEngineEPoll : public MasterEventEngine, public CascadingEventEngine,
     virtual int add_interest(Event e) override {
         if (e.fd < 0)
             LOG_ERROR_RETURN(EINVAL, -1, "invalid file descriptor ", e.fd);
+        if (unlikely(!e.interests))
+            return 0;
         if (unlikely((size_t)e.fd >= _inflight_events.size()))
             _inflight_events.resize(e.fd * 2);
+
+        e.interests &= EVENT_RWEO;
         auto& entry = _inflight_events[e.fd];
-        auto intersection = e.interests & entry.interests & EVENT_RWE;
-        auto data = (entry.reader_data != e.data) * EVENT_READ  |
-                    (entry.writer_data != e.data) * EVENT_WRITE |
-                    (entry.error_data  != e.data) * EVENT_ERROR ;
-        if (intersection & data)
-            LOG_ERROR_RETURN(EALREADY, -1, "conflicted interest(s)");
+        auto eint = entry.interests & EVENT_RWEO;
+        int op;
+        if (!eint) {
+            eint = e.interests;
+            op = EPOLL_CTL_ADD;
+        } else {
+            if ((eint ^ e.interests) & ONE_SHOT)
+                LOG_ERROR_RETURN(EALREADY, -1, "conflicted ONE_SHOT flag");
+            auto intersection = e.interests & eint;
+            auto data = (entry.reader_data != e.data) * EVENT_READ  |
+                        (entry.writer_data != e.data) * EVENT_WRITE |
+                        (entry.error_data  != e.data) * EVENT_ERROR ;
+            if (intersection & data)
+                LOG_ERROR_RETURN(EALREADY, -1, "conflicted interest(s)");
+
+            eint |= e.interests;
+            op = EPOLL_CTL_MOD;
+        }
 
-        int ret;
-        auto eint = entry.interests;
-        auto x = (eint | e.interests) & EVENT_RWE;
-        auto events = evmap.translate_bitwisely(x);
-        if (likely(e.interests & ONE_SHOT)) {
+        auto events = evmap.translate_bitwisely(eint);
+        if (likely(eint & ONE_SHOT)) {
             events |= EPOLLONESHOT;
-            if (likely(eint & ONE_SHOT)) {
-                ret = ctl(e.fd, EPOLL_CTL_MOD, events);
-                if (unlikely(ret < 0 && errno == ENOENT))
-                    ret = ctl(e.fd, EPOLL_CTL_ADD, events);
-            } else {
-                if (eint != 0) LOG_ERROR_RETURN(EINVAL, -1, "conflicted interest(s) regarding ONE_SHOT");
-                ret = ctl(e.fd, EPOLL_CTL_ADD, events);
+            if (likely(op == EPOLL_CTL_MOD)) {
+                // This may falsely fail with errno == ENOENT,
+                // if the fd was closed and created again.
+                // We should suppress that specific error log
+                int ret = ctl(e.fd, op, events, ENOENT);
+                if (likely(ret == 0)) goto ok;
+                // in such cases, and `EPOLL_CTL_ADD` it again.
+                else if (ret > 0) op = EPOLL_CTL_ADD;
+                else /*if (ret < 0)*/ goto fail;
             }
-        } else {
-            auto op = (eint & EVENT_RWE) ? EPOLL_CTL_MOD : EPOLL_CTL_ADD;
-            ret = ctl(e.fd, op, events);
         }
-        if (ret < 0)
-            LOG_ERROR_RETURN(0, ret, "failed to add_interest()");
+        if (ctl(e.fd, op, events) < 0) { fail:
+            LOG_ERROR_RETURN(0, -1, "failed to add_interest()");
+        }
 
-        entry.interests |= e.interests;
+ok:     entry.interests |= eint;
         if (e.interests & EVENT_READ)  entry.reader_data = e.data;
         if (e.interests & EVENT_WRITE) entry.writer_data = e.data;
         if (e.interests & EVENT_ERROR) entry.error_data  = e.data;
@@ -158,25 +174,24 @@ class EventEngineEPoll : public MasterEventEngine, public CascadingEventEngine,
             LOG_ERROR_RETURN(EINVAL, -1, "invalid file descriptor ", e.fd);
         if (unlikely(e.interests == 0)) return 0;
         auto& entry = _inflight_events[e.fd];
-        auto eint = entry.interests & (EVENT_RWE | ONE_SHOT);
+        auto eint = entry.interests & EVENT_RWEO;
         auto intersection = e.interests & eint;
         if (intersection == 0) return 0;
 
         auto remain = eint ^ intersection;  // ^ is to flip intersected bits
-        if (remain == ONE_SHOT) {/* no need to epoll_ctl() */} else {
-            uint32_t op, events = 0;
-            if (remain) {
-                events = evmap.translate_bitwisely(remain);
-                if (remain & ONE_SHOT) events |= EPOLLONESHOT;
-                op = EPOLL_CTL_MOD;
-            } else {
-                op = EPOLL_CTL_DEL;
+        if (likely(remain == ONE_SHOT)) {
+            /* no need to epoll_ctl() */
+        } else if (likely(!remain)) {
+            if (ctl(e.fd, EPOLL_CTL_DEL, 0, ENOENT) < 0) { fail:
+                LOG_ERROR_RETURN(0, -1, "failed to rm_interest()");
             }
-           if (ctl(e.fd, op, events) < 0)
-               LOG_ERROR_RETURN(0, -1, "failed to rm_interest()");
+        } else {
+            auto events = evmap.translate_bitwisely(remain);
+            if (remain & ONE_SHOT) events |= EPOLLONESHOT;
+            if (ctl(e.fd, EPOLL_CTL_MOD, events) < 0) goto fail;
         }
 
-        entry.interests ^= intersection;
+        entry.interests ^= intersection; // ^ is to flip intersected bits
         if (intersection & EVENT_READ)  entry.reader_data = nullptr;
         if (intersection & EVENT_WRITE) entry.writer_data = nullptr;
         if (intersection & EVENT_ERROR) entry.error_data  = nullptr;

From c30e53338dc785b4785f4f62b50d72d560d82b3d Mon Sep 17 00:00:00 2001
From: Huiba Li <huiba.lhb@alibaba-inc.com>
Date: Thu, 28 Mar 2024 15:02:27 +0800
Subject: [PATCH 62/76] fix gcc13 (#429)

---
 common/iovector.h     | 5 +++++
 net/datagram_socket.h | 2 ++
 rpc/rpc.cpp           | 4 ++++
 3 files changed, 11 insertions(+)

diff --git a/common/iovector.h b/common/iovector.h
index 97d583d4..2dc11f0f 100644
--- a/common/iovector.h
+++ b/common/iovector.h
@@ -39,6 +39,10 @@ limitations under the License.
 #include <photon/common/callback.h>
 #include <photon/common/io-alloc.h>
 
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunknown-warning-option"
+#pragma GCC diagnostic ignored "-Wzero-length-bounds"
+
 inline bool operator == (const iovec& a, const iovec& b)
 {
     return a.iov_base == b.iov_base && a.iov_len == b.iov_len;
@@ -1074,3 +1078,4 @@ class SmartCloneIOV
 
 #undef IF_ASSERT_RETURN
 
+#pragma GCC diagnostic pop
\ No newline at end of file
diff --git a/net/datagram_socket.h b/net/datagram_socket.h
index a7dbfc28..d9284467 100644
--- a/net/datagram_socket.h
+++ b/net/datagram_socket.h
@@ -50,6 +50,8 @@ class IDatagramSocket : public IMessage,
     ssize_t recv(B* buf, S count, int flags = 0) {
         return cast()->recv(buf, count, nullptr, nullptr, flags);
     }
+    using IMessage::recv;
+    using IMessage::send;
 };
 
 class UDPSocket : public IDatagramSocket {
diff --git a/rpc/rpc.cpp b/rpc/rpc.cpp
index d55a778e..0fc401dc 100644
--- a/rpc/rpc.cpp
+++ b/rpc/rpc.cpp
@@ -324,8 +324,12 @@ namespace rpc {
             if (unlikely(!m_running))
                 return -1;
 
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunknown-warning-option"
+#pragma GCC diagnostic ignored "-Wdangling-pointer"
             ThreadLink node;
             m_list.push_back(&node);
+#pragma GCC diagnostic pop
             DEFER(m_list.erase(&node));
             // stream serve refcount
             int stream_serv_count = 0;

From 71626d66047fa41d55fb1b3e3df13f460e567eff Mon Sep 17 00:00:00 2001
From: Bob Chen <beef9999@qq.com>
Date: Sun, 31 Mar 2024 09:30:33 +0800
Subject: [PATCH 63/76] Closure of callback.h add photon namespace (#431)

---
 common/callback.h | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/common/callback.h b/common/callback.h
index 60d74f6a..14e65b4e 100644
--- a/common/callback.h
+++ b/common/callback.h
@@ -137,7 +137,9 @@ struct Delegate : public Delegate_Base
 template<typename...Ts>
 using Callback = Delegate<int, Ts...>;
 
-// a Closure encapslates a Delegate together
+namespace photon {
+
+// a Closure encapsulates a Delegate together
 // with a ptr to a functor (or lambda object),
 // and delete it if DELETE_CLOSURE is returned.
 
@@ -172,6 +174,8 @@ struct Closure : public Delegate<int64_t, ARGS...> {
 
 using Closure0 = struct Closure<>;
 
+} // namespace photon
+
 /*
 inline int __Examples_of_Callback(void*, int, double, long)
 {

From 53095bacf345b6df01225ffd74818de0889ea79b Mon Sep 17 00:00:00 2001
From: Bob Chen <beef9999@qq.com>
Date: Tue, 2 Apr 2024 09:24:16 +0800
Subject: [PATCH 64/76] httpfs add error log (#436)

---
 fs/httpfs/httpfs_v2.cpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/fs/httpfs/httpfs_v2.cpp b/fs/httpfs/httpfs_v2.cpp
index 57e07072..3a0b3fc8 100644
--- a/fs/httpfs/httpfs_v2.cpp
+++ b/fs/httpfs/httpfs_v2.cpp
@@ -206,6 +206,9 @@ class HttpFile_v2 : public fs::VirtualReadOnlyFile {
         }
         m_authorized = true;
         ret = op.resp.readv(iovec, iovcnt);
+        if (ret < 0) {
+            LOG_ERROR("HttpFs: read body failed, ", ERRNO());
+        }
         return ret;
     }
 

From b5f9430621aacc873abb4f3217fd8e40fcf117db Mon Sep 17 00:00:00 2001
From: Coldwings <coldwings@me.com>
Date: Wed, 3 Apr 2024 16:38:56 +0800
Subject: [PATCH 65/76] Remove some outdated libcurl option usage in curl
 wrapper header (#440) (#443)

---
 net/curl.h | 2 --
 1 file changed, 2 deletions(-)

diff --git a/net/curl.h b/net/curl.h
index 37fee819..8a4a4db5 100644
--- a/net/curl.h
+++ b/net/curl.h
@@ -240,7 +240,6 @@ class cURL {
             return;
         }
         setopt(CURLOPT_ERRORBUFFER, m_errmsg);
-        setopt(CURLOPT_DNS_USE_GLOBAL_CACHE, 0L);
         setopt(CURLOPT_NOSIGNAL, 1L);
         setopt(CURLOPT_TCP_NODELAY, 1L);
         m_errmsg[0] = '\0';
@@ -413,7 +412,6 @@ class cURL {
         set_read_stream(rstream);
         set_write_stream(wstream);
         setopt(CURLOPT_UPLOAD, 1L);
-        setopt(CURLOPT_PUT, 1L);
         setopt(CURLOPT_URL, url);
         setopt(CURLOPT_HTTPHEADER, headers.list);
 #if LIBCURL_VERSION_MAJOR > 7 || LIBCURL_VERSION_MAJOR == 7 && LIBCURL_VERSION_MINOR >= 37

From 5cbfb92987c6a018ab322c8eb2626e93fe0d7aa3 Mon Sep 17 00:00:00 2001
From: Bob Chen <beef9999@qq.com>
Date: Wed, 10 Apr 2024 20:55:13 +0800
Subject: [PATCH 66/76] zlib and uring source build didn't have -O3 (#451)

---
 CMake/build-from-src.cmake | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/CMake/build-from-src.cmake b/CMake/build-from-src.cmake
index abfcbe1b..caeb2598 100644
--- a/CMake/build-from-src.cmake
+++ b/CMake/build-from-src.cmake
@@ -27,7 +27,7 @@ function(build_from_src [dep])
                 URL_MD5 9b8aa094c4e5765dabf4da391f00d15c
                 UPDATE_DISCONNECTED ON
                 BUILD_IN_SOURCE ON
-                CONFIGURE_COMMAND CFLAGS=-fPIC ./configure --prefix=${BINARY_DIR} --static
+                CONFIGURE_COMMAND sh -c "CFLAGS=\"-fPIC -O3\" ./configure --prefix=${BINARY_DIR} --static"
                 BUILD_COMMAND $(MAKE)
                 INSTALL_COMMAND $(MAKE) install
         )
@@ -43,7 +43,7 @@ function(build_from_src [dep])
                 UPDATE_DISCONNECTED ON
                 BUILD_IN_SOURCE ON
                 CONFIGURE_COMMAND ./configure --prefix=${BINARY_DIR}
-                BUILD_COMMAND V=1 CFLAGS=-fPIC $(MAKE) -C src
+                BUILD_COMMAND sh -c "V=1 CFLAGS=\"-fPIC -g -O3 -Wall -Wextra -fno-stack-protector\" $(MAKE) -C src"
                 INSTALL_COMMAND $(MAKE) install
         )
         set(URING_INCLUDE_DIRS ${BINARY_DIR}/include PARENT_SCOPE)

From cd03fc000465a5e6c0226ea673a0ac04871b509d Mon Sep 17 00:00:00 2001
From: Lanzheng Liu <lanzheng.liulz@alibaba-inc.com>
Date: Thu, 18 Apr 2024 17:41:27 +0800
Subject: [PATCH 67/76] fix dns discard (#459)

Signed-off-by: liulanzheng <lanzheng.liulz@alibaba-inc.com>
---
 net/utils.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/net/utils.cpp b/net/utils.cpp
index f90e49a0..ed514ed9 100644
--- a/net/utils.cpp
+++ b/net/utils.cpp
@@ -312,6 +312,7 @@ class DefaultResolver : public Resolver {
                     break;
                 }
             }
+            ipaddr.recycle(ipaddr->empty());
         }
     }
 

From 76e032dbe8a657af2e7ad4149fe9798a07f2a275 Mon Sep 17 00:00:00 2001
From: Bob Chen <beef9999@qq.com>
Date: Fri, 19 Apr 2024 21:50:15 +0800
Subject: [PATCH 68/76] http client support uds (#461)

---
 fs/test/CMakeLists.txt                 |  6 +--
 net/http/client.cpp                    | 22 +++++++++--
 net/http/client.h                      | 15 ++++++--
 net/http/test/client_function_test.cpp | 52 +++++++++++++++++++++++++-
 4 files changed, 84 insertions(+), 11 deletions(-)

diff --git a/fs/test/CMakeLists.txt b/fs/test/CMakeLists.txt
index 6eb5b9c5..fb7421b1 100644
--- a/fs/test/CMakeLists.txt
+++ b/fs/test/CMakeLists.txt
@@ -12,6 +12,6 @@ add_executable(test-filecopy test_filecopy.cpp)
 target_link_libraries(test-filecopy PRIVATE photon_shared)
 add_test(NAME test-filecopy COMMAND $<TARGET_FILE:test-filecopy>)
 
-add_executable(test-throttled test_throttledfile.cpp)
-target_link_libraries(test-throttled PRIVATE photon_shared)
-add_test(NAME test-throttled COMMAND $<TARGET_FILE:test-throttled>)
\ No newline at end of file
+add_executable(test-throttle-file test_throttledfile.cpp)
+target_link_libraries(test-throttle-file PRIVATE photon_shared)
+add_test(NAME test-throttle-file COMMAND $<TARGET_FILE:test-throttle-file>)
\ No newline at end of file
diff --git a/net/http/client.cpp b/net/http/client.cpp
index 23ea5c25..802eb873 100644
--- a/net/http/client.cpp
+++ b/net/http/client.cpp
@@ -38,6 +38,7 @@ class PooledDialer {
     bool tls_ctx_ownership;
     std::unique_ptr<ISocketClient> tcpsock;
     std::unique_ptr<ISocketClient> tlssock;
+    std::unique_ptr<ISocketClient> udssock;
     std::unique_ptr<Resolver> resolver;
 
     //etsocket seems not support multi thread very well, use tcp_socket now. need to find out why
@@ -49,6 +50,7 @@ class PooledDialer {
         auto tls_cli = new_tls_client(tls_ctx, new_tcp_socket_client(), true);
         tcpsock.reset(new_tcp_socket_pool(tcp_cli, -1, true));
         tlssock.reset(new_tcp_socket_pool(tls_cli, -1, true));
+        udssock.reset(new_uds_client());
     }
 
     ~PooledDialer() {
@@ -63,6 +65,8 @@ class PooledDialer {
     ISocketStream* dial(const T& x, uint64_t timeout = -1UL) {
         return dial(x.host_no_port(), x.port(), x.secure(), timeout);
     }
+
+    ISocketStream* dial(std::string_view uds_path, uint64_t timeout = -1UL);
 };
 
 ISocketStream* PooledDialer::dial(std::string_view host, uint16_t port, bool secure, uint64_t timeout) {
@@ -96,6 +100,14 @@ ISocketStream* PooledDialer::dial(std::string_view host, uint16_t port, bool sec
     return nullptr;
 }
 
+ISocketStream* PooledDialer::dial(std::string_view uds_path, uint64_t timeout) {
+    udssock->timeout(timeout);
+    auto stream = udssock->connect(uds_path.data());
+    if (!stream)
+        LOG_ERRNO_RETURN(0, nullptr, "failed to dial to unix socket `", uds_path);
+    return stream;
+}
+
 constexpr uint64_t code3xx() { return 0; }
 template<typename...Ts>
 constexpr uint64_t code3xx(uint64_t x, Ts...xs)
@@ -164,9 +176,13 @@ class ClientImpl : public Client {
         if (tmo.timeout() == 0)
             LOG_ERROR_RETURN(ETIMEDOUT, ROUNDTRIP_FAILED, "connection timedout");
         auto &req = op->req;
-        auto s = (m_proxy && !m_proxy_url.empty())
-                     ? m_dialer.dial(m_proxy_url, tmo.timeout())
-                     : m_dialer.dial(req, tmo.timeout());
+        ISocketStream* s;
+        if (m_proxy && !m_proxy_url.empty())
+            s = m_dialer.dial(m_proxy_url, tmo.timeout());
+        else if (!op->uds_path.empty())
+            s = m_dialer.dial(op->uds_path, tmo.timeout());
+        else
+            s = m_dialer.dial(req, tmo.timeout());
         if (!s) {
             if (errno == ECONNREFUSED || errno == ENOENT) {
                 LOG_ERROR_RETURN(0, ROUNDTRIP_FAST_RETRY, "connection refused")
diff --git a/net/http/client.h b/net/http/client.h
index 350766e3..c1a80cb3 100644
--- a/net/http/client.h
+++ b/net/http/client.h
@@ -48,7 +48,9 @@ class Client : public Object {
         uint16_t retry = 5;                       // default retry: 5 at most
         Response resp;                            // response
         int status_code = -1;                     // status code in response
-        bool enable_proxy;
+        bool enable_proxy = false;
+        std::string_view uds_path;                // If set, Unix Domain Socket will be used instead of TCP.
+                                                  // URL should still be the format of http://localhost/xxx
         IStream* body_stream = nullptr;                 // use body_stream as body
         using BodyWriter = Delegate<ssize_t, Request*>; // or call body_writer if body_stream
         BodyWriter body_writer = {};                    // is not set
@@ -68,6 +70,11 @@ class Client : public Object {
             if (!_client) return -1;
             return _client->call(this);
         }
+        int call(std::string_view unix_socket_path) {
+            if (!_client) return -1;
+            uds_path = unix_socket_path;
+            return _client->call(this);
+        }
 
     protected:
         Client* _client;
@@ -83,15 +90,15 @@ class Client : public Object {
         Operation(uint16_t buf_size) : req(_buf, buf_size) {}
     };
 
-    Operation* new_operation(Verb v, std::string_view url, uint16_t buf_size = 64 * 1024 - 1) {
+    Operation* new_operation(Verb v, std::string_view url, uint16_t buf_size = UINT16_MAX) {
         return Operation::create(this, v, url, buf_size);
     }
 
-    Operation* new_operation(uint16_t buf_size = 64 * 1024 - 1) {
+    Operation* new_operation(uint16_t buf_size = UINT16_MAX) {
         return Operation::create(this, buf_size);
     }
 
-    template<uint16_t BufferSize>
+    template<uint16_t BufferSize = UINT16_MAX>
     class OperationOnStack : public Operation {
         char _buf[BufferSize];
     public:
diff --git a/net/http/test/client_function_test.cpp b/net/http/test/client_function_test.cpp
index 4d68d3c1..494fa249 100644
--- a/net/http/test/client_function_test.cpp
+++ b/net/http/test/client_function_test.cpp
@@ -61,6 +61,21 @@ int timeout_writer(void *self, IStream* stream) {
     return 0;
 }
 
+class SimpleHandler : public http::HTTPHandler {
+public:
+    int handle_request(http::Request& req, http::Response& resp, std::string_view) {
+        std::string url_path(req.target());
+        resp.set_result(200);
+        resp.headers.content_length(url_path.size());
+        resp.headers.insert("Content-Type", "application/octet-stream");
+        auto n = resp.write(url_path.data(), url_path.size());
+        if (n != (ssize_t) url_path.size()) {
+            LOG_ERRNO_RETURN(0, -1, "send body failed");
+        }
+        return 0;
+    }
+};
+
 TEST(http_client, get) {
     system("mkdir -p /tmp/ease_ut/http_test/");
     system("echo \"this is a http_client request body text for socket stream\" > /tmp/ease_ut/http_test/ease-httpclient-gettestfile");
@@ -523,6 +538,41 @@ TEST(DISABLED_http_client, ipv6) {  // make sure runing in a ipv6-ready environm
     EXPECT_EQ(200, op->resp.status_code());
 }
 
+TEST(http_client, unix_socket) {
+    const char* uds_path = "test-http-client.sock";
+
+    auto http_server = new_http_server();
+    DEFER(delete http_server);
+    http_server->add_handler(new SimpleHandler, true, "/simple-api");
+
+    auto socket_server = new_uds_server(true);
+    DEFER(delete socket_server);
+
+    socket_server->set_handler(http_server->get_connection_handler());
+    ASSERT_EQ(0, socket_server->bind(uds_path));
+    ASSERT_EQ(0, socket_server->listen());
+    ASSERT_EQ(0, socket_server->start_loop(false));
+
+    auto client = new_http_client();
+    DEFER(delete client);
+
+    Client::OperationOnStack<> op(client, Verb::GET, "http://localhost/simple-api");
+    int ret = op.call(uds_path);
+    ASSERT_EQ(0, ret);
+    ASSERT_EQ(200, op.resp.status_code());
+
+    char buf[UINT16_MAX];
+    ssize_t n = op.resp.read(buf, op.resp.body_size());
+    ASSERT_EQ(n, (ssize_t) op.resp.body_size());
+    LOG_INFO(buf);
+
+    // A wrong hostname or HTTPS doesn't have effect on unix socket
+    Client::OperationOnStack<> op2(client, Verb::GET, "https://www.wrong.hostname/simple-api");
+    ret = op2.call(uds_path);
+    ASSERT_EQ(0, ret);
+    ASSERT_EQ(200, op2.resp.status_code());
+}
+
 TEST(url, url_escape_unescape) {
     EXPECT_EQ(
         url_escape("?a=x:b&b=cd&c= feg&d=2/1[+]@alibaba.com&e='!bad';"),
@@ -582,5 +632,5 @@ int main(int argc, char** arg) {
 #endif
     set_log_output_level(ALOG_INFO);
     ::testing::InitGoogleTest(&argc, arg);
-    LOG_DEBUG("test result:`", RUN_ALL_TESTS());
+    return RUN_ALL_TESTS();
 }

From 8aff3851f78ca0fcff3a8676d6cdae17039cff1e Mon Sep 17 00:00:00 2001
From: Huiba Li <huiba.lhb@alibaba-inc.com>
Date: Thu, 9 May 2024 12:49:24 +0800
Subject: [PATCH 69/76] udpate_now() to force update `now` (#473)

* udpate_now() to force update `now`

and make it compilable under macos + arm
---
 CMakeLists.txt    |  8 +++++---
 common/alog.cpp   | 24 +++++-------------------
 thread/thread.cpp | 19 +++++++++++++------
 thread/thread.h   |  8 +++++++-
 4 files changed, 30 insertions(+), 29 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 6f67a2fd..63794d65 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -71,9 +71,11 @@ elseif (${ARCH} STREQUAL aarch64)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcpu=generic+crc -fsigned-char -fno-stack-protector -fomit-frame-pointer")
 endif ()
 
-check_cxx_compiler_flag(-mcrc32 COMPILER_HAS_MCRC32_FLAG)
-if (COMPILER_HAS_MCRC32_FLAG)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcrc32")
+if (${ARCH} STREQUAL x86_64)
+    check_cxx_compiler_flag(-mcrc32 COMPILER_HAS_MCRC32_FLAG)
+    if (COMPILER_HAS_MCRC32_FLAG)
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcrc32")
+    endif ()
 endif ()
 
 set(CMAKE_C_FLAGS ${CMAKE_CXX_FLAGS})
diff --git a/common/alog.cpp b/common/alog.cpp
index e371b801..43e68431 100644
--- a/common/alog.cpp
+++ b/common/alog.cpp
@@ -206,7 +206,7 @@ void LogFormatter::put_integer_dec(ALogBuffer& buf, ALogInteger x)
 __attribute__((constructor)) static void __initial_timezone() { tzset(); }
 static time_t dayid = 0;
 static struct tm alog_time = {0};
-struct tm* alog_update_time(time_t now)
+static struct tm* alog_update_time(time_t now)
 {
     auto now0 = now;
     int sec = now % 60;    now /= 60;
@@ -478,36 +478,22 @@ int log_output_file_close() {
     return 0;
 }
 
-namespace photon
-{
-    struct thread;
-    extern __thread thread* CURRENT;
-}
-
-static inline ALogInteger DEC_W2P0(uint64_t x)
-{
-    return DEC(x).width(2).padding('0');
-}
-
-namespace photon {
-struct timeval alog_update_now();
-}
-
 LogBuffer& operator << (LogBuffer& log, const Prologue& pro)
 {
 #ifdef LOG_BENCHMARK
     auto t = &alog_time;
 #else
-    auto ts = photon::alog_update_now();
-    auto t = alog_update_time(ts.tv_sec - timezone);
+    auto ts = photon::__update_now();
+    auto t = alog_update_time(ts.sec() - timezone);
 #endif
+#define DEC_W2P0(x) DEC(x).width(2).padding('0')
     log.printf(t->tm_year, '/');
     log.printf(DEC_W2P0(t->tm_mon),  '/');
     log.printf(DEC_W2P0(t->tm_mday), ' ');
     log.printf(DEC_W2P0(t->tm_hour), ':');
     log.printf(DEC_W2P0(t->tm_min),  ':');
     log.printf(DEC_W2P0(t->tm_sec), '.');
-    log.printf(DEC(ts.tv_usec).width(6).padding('0'));
+    log.printf(DEC(ts.usec()).width(6).padding('0'));
 
     static const char levels[] = "|DEBUG|th=|INFO |th=|WARN |th=|ERROR|th=|FATAL|th=|TEMP |th=|AUDIT|th=";
     log.reserved = pro.level;
diff --git a/thread/thread.cpp b/thread/thread.cpp
index 5b07e7ff..e6e2379d 100644
--- a/thread/thread.cpp
+++ b/thread/thread.cpp
@@ -847,6 +847,11 @@ R"(
 )"
     );
 
+#ifdef __clang__
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Winline-asm"
+#endif
+
     inline void switch_context(thread* from, thread* to) {
         prepare_switch(from, to);
         auto _t_ = to->stack.pointer_ref();
@@ -881,6 +886,9 @@ R"(
                        "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16",
                        "x17", "x18");
     }
+#ifdef __clang__
+#pragma GCC diagnostic pop
+#endif
 
 #endif  // x86 or arm
 
@@ -1033,7 +1041,7 @@ R"(
 
     volatile uint64_t now;
     static std::atomic<pthread_t> ts_updater(0);
-    static inline struct timeval update_now()
+    static inline NowTime update_now()
     {
 #if defined(__x86_64__) && defined(__linux__) && defined(ENABLE_MIMIC_VDSO)
         if (likely(__mimic_vdso_time_x86))
@@ -1041,11 +1049,10 @@ R"(
 #endif
         struct timeval tv;
         gettimeofday(&tv, NULL);
-        uint64_t nnow = tv.tv_sec;
-        nnow *= 1000 * 1000;
-        nnow += tv.tv_usec;
+        uint64_t nnow = tv.tv_sec * 1000ul * 1000ul + tv.tv_usec;
         now = nnow;
-        return tv;
+        assert(tv.tv_sec <= UINT32_MAX && tv.tv_usec < 1000000);
+        return {nnow, ((uint64_t)tv.tv_sec << 32) | (uint32_t)tv.tv_usec};
     }
     __attribute__((always_inline))
     static inline uint32_t _rdtsc()
@@ -1089,7 +1096,7 @@ R"(
             update_now();
         }
     }
-    struct timeval alog_update_now() {
+    NowTime __update_now() {
         last_tsc = _rdtsc();
         return update_now();
     }
diff --git a/thread/thread.h b/thread/thread.h
index 9bb35347..d9748c04 100644
--- a/thread/thread.h
+++ b/thread/thread.h
@@ -36,7 +36,13 @@ namespace photon
 
     struct thread;
     extern __thread thread* CURRENT;
-    extern volatile uint64_t now;
+    extern volatile uint64_t now;   // a coarse-grained timestamp in unit of us
+    struct NowTime {
+        uint64_t now, _sec_usec;
+        uint32_t  sec() { return _sec_usec >> 32; }
+        uint32_t usec() { auto p = (uint32_t*)&_sec_usec; return *p; }
+    };
+    NowTime __update_now();    // update `now`
 
     enum states
     {

From 80cfc2ef118eeae70e3e4989970e0eeb25cb640a Mon Sep 17 00:00:00 2001
From: Lanzheng Liu <lanzheng.liulz@alibaba-inc.com>
Date: Thu, 16 May 2024 11:45:36 +0800
Subject: [PATCH 70/76] support customized UserAgent in http client (#479)

Signed-off-by: liulanzheng <lanzheng.liulz@alibaba-inc.com>
---
 net/http/client.cpp                    |  5 ++--
 net/http/client.h                      |  4 +++
 net/http/test/client_function_test.cpp | 38 ++++++++++++++++++++++++++
 3 files changed, 45 insertions(+), 2 deletions(-)

diff --git a/net/http/client.cpp b/net/http/client.cpp
index 802eb873..6ec2bea8 100644
--- a/net/http/client.cpp
+++ b/net/http/client.cpp
@@ -252,9 +252,10 @@ class ClientImpl : public Client {
             LOG_ERROR_RETURN(EINVAL, ROUNDTRIP_FAILED,
                             "Content-Length and Transfer-Encoding conflicted");
         }
-        op->req.headers.insert("User-Agent", USERAGENT);
-        op->req.headers.insert("Connection", "keep-alive");
         op->req.headers.merge(m_common_headers);
+        op->req.headers.insert("User-Agent", m_user_agent.empty() ? std::string_view(USERAGENT)
+                                                                  : std::string_view(m_user_agent));
+        op->req.headers.insert("Connection", "keep-alive");
         if (m_cookie_jar && m_cookie_jar->set_cookies_to_headers(&op->req) != 0)
             LOG_ERROR_RETURN(0, -1, "set_cookies_to_headers failed");
         Timeout tmo(std::min(op->timeout.timeout(), m_timeout));
diff --git a/net/http/client.h b/net/http/client.h
index c1a80cb3..53a27ae6 100644
--- a/net/http/client.h
+++ b/net/http/client.h
@@ -116,6 +116,9 @@ class Client : public Object {
         m_proxy_url.from_string(proxy);
         m_proxy = true;
     }
+    void set_user_agent(std::string_view user_agent) {
+        m_user_agent = std::string(user_agent);
+    }
     StoredURL* get_proxy() {
         return &m_proxy_url;
     }
@@ -136,6 +139,7 @@ class Client : public Object {
                                     bool secure = false, uint64_t timeout = -1UL) = 0;
 protected:
     StoredURL m_proxy_url;
+    std::string m_user_agent;
     uint64_t m_timeout = -1UL;
     bool m_proxy = false;
 };
diff --git a/net/http/test/client_function_test.cpp b/net/http/test/client_function_test.cpp
index 494fa249..79af5284 100644
--- a/net/http/test/client_function_test.cpp
+++ b/net/http/test/client_function_test.cpp
@@ -573,6 +573,44 @@ TEST(http_client, unix_socket) {
     ASSERT_EQ(200, op2.resp.status_code());
 }
 
+int ua_check_handler(void*, Request &req, Response &resp, std::string_view) {
+    auto ua = req.headers["User-Agent"];
+    LOG_DEBUG(VALUE(ua));
+    EXPECT_EQ(ua, "TEST_UA");
+    resp.set_result(200);
+    std::string str = "success";
+    resp.headers.content_length(7);
+    resp.write((void*)str.data(), str.size());
+    return 0;
+}
+
+TEST(http_client, user_agent) {
+    auto tcpserver = new_tcp_socket_server();
+    DEFER(delete tcpserver);
+    tcpserver->bind(18731);
+    tcpserver->listen();
+    auto server = new_http_server();
+    DEFER(delete server);
+    server->add_handler({nullptr, &ua_check_handler});
+    tcpserver->set_handler(server->get_connection_handler());
+    tcpserver->start_loop();
+
+    std::string target_get = "http://localhost:18731/file";
+    auto client = new_http_client();
+    client->set_user_agent("TEST_UA");
+    DEFER(delete client);
+    auto op = client->new_operation(Verb::GET, target_get);
+    DEFER(delete op);
+    op->req.headers.content_length(0);
+    client->call(op);
+    EXPECT_EQ(op->status_code, 200);
+    std::string buf;
+    buf.resize(op->resp.headers.content_length());
+    op->resp.read((void*)buf.data(), op->resp.headers.content_length());
+    LOG_DEBUG(VALUE(buf));
+    EXPECT_EQ(true, buf == "success");
+}
+
 TEST(url, url_escape_unescape) {
     EXPECT_EQ(
         url_escape("?a=x:b&b=cd&c= feg&d=2/1[+]@alibaba.com&e='!bad';"),

From 58fab4c5fe82a6bd18fbf25a5fa734b212bf846b Mon Sep 17 00:00:00 2001
From: lihuiba <huiba.lhb@alibaba-inc.com>
Date: Mon, 20 May 2024 16:54:08 +0800
Subject: [PATCH 71/76] fix gcc13

---
 common/iovector.h    | 3 ++-
 common/test/test.cpp | 5 +++++
 rpc/rpc.cpp          | 4 +++-
 thread/thread11.h    | 2 +-
 4 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/common/iovector.h b/common/iovector.h
index 2dc11f0f..404d2492 100644
--- a/common/iovector.h
+++ b/common/iovector.h
@@ -40,7 +40,8 @@ limitations under the License.
 #include <photon/common/io-alloc.h>
 
 #pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wunknown-warning-option"
+#if __GNUC__ >= 13
+// #pragma GCC diagnostic ignored "-Wunknown-warning-option"
 #pragma GCC diagnostic ignored "-Wzero-length-bounds"
 
 inline bool operator == (const iovec& a, const iovec& b)
diff --git a/common/test/test.cpp b/common/test/test.cpp
index e9b2b27e..28aa6a22 100644
--- a/common/test/test.cpp
+++ b/common/test/test.cpp
@@ -243,7 +243,12 @@ TEST(Callback, virtual_function)
     Callback<int> dd(lambda);
 //    Callback<int> ee([&](int x){ return RET + x/2; });
 
+#pragma GCC diagnostic push
+#if __GNUC__ >= 13
+#pragma GCC diagnostic ignored "-Wdangling-pointer"
+#endif
     THIS = (BB*)&c;
+#pragma GCC diagnostic pop
     LOG_DEBUG(VALUE(THIS), VALUE(&c));
 
     for (int i=0; i<100; ++i)
diff --git a/rpc/rpc.cpp b/rpc/rpc.cpp
index 0fc401dc..a0a7950b 100644
--- a/rpc/rpc.cpp
+++ b/rpc/rpc.cpp
@@ -325,8 +325,10 @@ namespace rpc {
                 return -1;
 
 #pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wunknown-warning-option"
+#if __GNUC__ >= 13
+// #pragma GCC diagnostic ignored "-Wunknown-warning-option"
 #pragma GCC diagnostic ignored "-Wdangling-pointer"
+#endif
             ThreadLink node;
             m_list.push_back(&node);
 #pragma GCC diagnostic pop
diff --git a/thread/thread11.h b/thread/thread11.h
index 7bd6fd1b..7f4cf34c 100644
--- a/thread/thread11.h
+++ b/thread/thread11.h
@@ -135,7 +135,7 @@ namespace photon {
 
         void asdf()
         {
-            int a; char b;
+            int a = 0; char b = 0;
             auto func = &__Example_of_Thread11__::member_function;
             auto cfunc = &__Example_of_Thread11__::const_member_function;
 

From e5a7cac0b1f4f086a2f585acca0c758282eff3ba Mon Sep 17 00:00:00 2001
From: lihuiba <huiba.lhb@alibaba-inc.com>
Date: Mon, 20 May 2024 17:09:23 +0800
Subject: [PATCH 72/76] fix

---
 common/iovector.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/common/iovector.h b/common/iovector.h
index 404d2492..27a24c9c 100644
--- a/common/iovector.h
+++ b/common/iovector.h
@@ -43,6 +43,7 @@ limitations under the License.
 #if __GNUC__ >= 13
 // #pragma GCC diagnostic ignored "-Wunknown-warning-option"
 #pragma GCC diagnostic ignored "-Wzero-length-bounds"
+#endif
 
 inline bool operator == (const iovec& a, const iovec& b)
 {

From 40507a627db156107fc5ea5e4cca4ad7ab38938a Mon Sep 17 00:00:00 2001
From: Bob Chen <beef9999@qq.com>
Date: Thu, 23 May 2024 13:57:17 +0800
Subject: [PATCH 73/76] cherry-pick fix for timer stack overflow (#489)

---
 thread/timer.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/thread/timer.h b/thread/timer.h
index c8e3dda5..d8e789b2 100644
--- a/thread/timer.h
+++ b/thread/timer.h
@@ -35,7 +35,7 @@ namespace photon
         // it has a `stack_size`, and the `on_timer` is invoked within the thread's context.
         // The timer object is deleted automatically after it is finished.
         Timer(uint64_t default_timeout, Entry on_timer, bool repeating = true,
-              uint64_t stack_size = 1024 * 64)
+              uint64_t stack_size = DEFAULT_STACK_SIZE)
         {
             _on_timer = on_timer;
             _default_timeout = default_timeout;

From 2244d0d2e2757cb92421fd270ee93328bf782838 Mon Sep 17 00:00:00 2001
From: Lanzheng Liu <lanzheng.liulz@alibaba-inc.com>
Date: Fri, 31 May 2024 01:45:59 +0800
Subject: [PATCH 74/76] fix dns resolver discard_cache (#497)

Signed-off-by: liulanzheng <lanzheng.liulz@alibaba-inc.com>
---
 net/utils.cpp | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/net/utils.cpp b/net/utils.cpp
index ed514ed9..010678f3 100644
--- a/net/utils.cpp
+++ b/net/utils.cpp
@@ -304,8 +304,10 @@ class DefaultResolver : public Resolver {
 
     void discard_cache(const char *host, IPAddr ip) override {
         auto ipaddr = dnscache_.borrow(host);
-        if (ip.undefined() || ipaddr->empty()) ipaddr.recycle(true);
-        else {
+        if (ip.undefined() || ipaddr->empty()) {
+            ipaddr->delete_all();
+            ipaddr.recycle(true);
+        } else {
             for (auto itr = ipaddr->rbegin(); itr != ipaddr->rend(); itr++) {
                 if ((*itr)->addr == ip) {
                     ipaddr->erase(*itr);

From 8261b9307ba21561bb70e8391d2c827657833ded Mon Sep 17 00:00:00 2001
From: Huiba Li <huiba.lhb@alibaba-inc.com>
Date: Mon, 3 Jun 2024 17:18:11 +0800
Subject: [PATCH 75/76] new CI workflow for linux.x86 (#494)

new CI workflow for linux.x86
---
 .github/workflows/ci.linux.arm.yml            |   8 +-
 .github/workflows/ci.linux.x86-64.yml         | 230 ++++++++++++++
 .github/workflows/ci.linux.x86.yml            | 119 -------
 .github/workflows/ci.macos.arm.yml            |  18 +-
 .../{ci.macos.yml => ci.macos.x86.yml}        |  14 +-
 CMake/Findgoogletest.cmake                    |   1 +
 CMake/build-from-src.cmake                    |   2 +-
 CMake/generate-ctest-packed-script.cmake      |  17 +
 CMakeLists.txt                                |  21 +-
 common/checksum/test/test_checksum.cpp        |   3 +
 common/executor/test/CMakeLists.txt           |  24 ++
 common/executor/test/test_async.cpp           |  12 +-
 common/executor/test/test_easyexport.cpp      |   2 +-
 .../executor/test/test_export_as_executor.cpp |   1 -
 common/lockfree_queue.h                       |   7 +-
 common/memory-stream/test/CMakeLists.txt      |   6 +
 common/memory-stream/test/test.cpp            |   2 +
 common/stream-messenger/messenger.cpp         |  89 ------
 common/stream-messenger/messenger.h           |  41 ---
 common/stream-messenger/test/test.cpp         | 101 ------
 common/test/CMakeLists.txt                    |   6 +-
 common/test/perf_objcache.cpp                 |   1 +
 common/test/test.cpp                          |   4 +
 common/test/test_alog.cpp                     |   2 +
 common/test/test_constexprstr.cpp             |   2 +
 common/test/test_lockfree.cpp                 |  36 +--
 common/test/test_objcache.cpp                 |  17 +-
 common/test/test_scalepool.cpp                |   5 +-
 common/test/test_throttle.cpp                 |   5 +-
 fs/exportfs.cpp                               |   1 +
 fs/test/CMakeLists.txt                        |   2 +-
 fs/test/test.cpp                              |   7 +-
 fs/test/test_exportfs.cpp                     |  75 ++---
 fs/test/test_throttledfile.cpp                |   2 +
 io/fd-events.h                                |   2 +-
 io/test/CMakeLists.txt                        |   2 +-
 io/test/test-iouring.cpp                      |   4 +-
 net/http/test/CMakeLists.txt                  |   4 +-
 net/test/CMakeLists.txt                       |   2 +-
 photon.cpp                                    |  28 +-
 rpc/test/test.cpp                             |   3 +-
 test/ci-tools.cpp                             | 112 +++++--
 test/ci-tools.h                               |  10 +-
 thread/test/CMakeLists.txt                    |   8 +-
 thread/test/test-multi-vcpu-locking.cpp       |   3 +-
 thread/test/test-pool.cpp                     | 297 ++++++++++++++++++
 thread/test/test.cpp                          | 275 +---------------
 thread/thread.cpp                             |   5 +-
 48 files changed, 836 insertions(+), 802 deletions(-)
 create mode 100644 .github/workflows/ci.linux.x86-64.yml
 delete mode 100644 .github/workflows/ci.linux.x86.yml
 rename .github/workflows/{ci.macos.yml => ci.macos.x86.yml} (74%)
 create mode 100644 CMake/generate-ctest-packed-script.cmake
 create mode 100644 common/executor/test/CMakeLists.txt
 create mode 100644 common/memory-stream/test/CMakeLists.txt
 delete mode 100644 common/stream-messenger/messenger.cpp
 delete mode 100644 common/stream-messenger/messenger.h
 delete mode 100644 common/stream-messenger/test/test.cpp
 create mode 100644 thread/test/test-pool.cpp

diff --git a/.github/workflows/ci.linux.arm.yml b/.github/workflows/ci.linux.arm.yml
index 5418a3e3..fbe35f61 100644
--- a/.github/workflows/ci.linux.arm.yml
+++ b/.github/workflows/ci.linux.arm.yml
@@ -37,12 +37,12 @@ jobs:
           source /opt/rh/gcc-toolset-9/enable
           cmake -B build -D CMAKE_BUILD_TYPE=MinSizeRel -D PHOTON_BUILD_TESTING=ON \
             -D PHOTON_ENABLE_SASL=ON -D PHOTON_ENABLE_FUSE=ON -D PHOTON_ENABLE_EXTFS=ON
-          cmake --build build -j -- VERBOSE=1
+          cmake --build build -j $(nproc) -- VERBOSE=1
 
       - name: Test
         run: |
           cd build
-          ctest --timeout 3600 -V
+          ctest -E test-lockfree --timeout 3600 -V
 
   centos8-gcc921-epoll-debug:
     runs-on: [self-hosted, Linux, ARM64]
@@ -74,9 +74,9 @@ jobs:
           source /opt/rh/gcc-toolset-9/enable
           cmake -B build -D CMAKE_BUILD_TYPE=Debug -D PHOTON_BUILD_TESTING=ON \
             -D PHOTON_ENABLE_SASL=ON -D PHOTON_ENABLE_FUSE=ON -D PHOTON_ENABLE_EXTFS=ON
-          cmake --build build -j -- VERBOSE=1
+          cmake --build build -j $(nproc) -- VERBOSE=1
 
       - name: Test
         run: |
           cd build
-          ctest --timeout 3600 -V
+          ctest -E test-lockfree --timeout 3600 -V
diff --git a/.github/workflows/ci.linux.x86-64.yml b/.github/workflows/ci.linux.x86-64.yml
new file mode 100644
index 00000000..453dff50
--- /dev/null
+++ b/.github/workflows/ci.linux.x86-64.yml
@@ -0,0 +1,230 @@
+name: Linux x86 (new)
+
+on:
+  push:
+    branches: [ "main", "release/*" ]
+  pull_request:
+    branches: [ "main", "release/*" ]
+
+jobs:
+  gcc850:
+    runs-on: [self-hosted, compiler]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Build850
+        run: |
+          rm -fr build
+          cmake -B build -D CMAKE_BUILD_TYPE=MinSizeRel   \
+                         -D PHOTON_BUILD_TESTING=ON       \
+                         -D PHOTON_ENABLE_SASL=ON         \
+                         -D PHOTON_ENABLE_FUSE=ON         \
+                         -D PHOTON_ENABLE_URING=ON        \
+                         -D PHOTON_ENABLE_EXTFS=ON        \
+                        #  -D PHOTON_BUILD_DEPENDENCIES=ON  \
+                        #  -D PHOTON_AIO_SOURCE=""          \
+                        #  -D PHOTON_ZLIB_SOURCE=""         \
+                        #  -D PHOTON_CURL_SOURCE=""         \
+                        #  -D PHOTON_OPENSSL_SOURCE=""      \
+                        #  -D PHOTON_GFLAGS_SOURCE=""       \
+                        #  -D PHOTON_GOOGLETEST_SOURCE=""   \
+                        #  -D PHOTON_URING_SOURCE=https://github.com/axboe/liburing/archive/refs/tags/liburing-2.3.tar.gz
+
+          cmake --build build -j $(nproc) --clean-first -- VERBOSE=1
+          ln -f common/checksum/test/checksum.in build/output/
+          tar -c -h --use-compress-program=zstdmt -f output850.tzs build/output/
+      - name: Upload850
+        uses: actions/upload-artifact@v4
+        with:
+          name: output850
+          path: output850.tzs
+          retention-days: 5
+          compression-level: 0
+
+  test850:
+    needs: gcc850
+    runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/coldwings/photon-ut-base:latest
+      options: --cpus 4
+    steps:
+      - uses: szenius/set-timezone@v1.2
+        with:
+          timezoneLinux: "Asia/Shanghai"
+          timezoneMacos: "Asia/Shanghai"
+          timezoneWindows: "China Standard Time"
+      - uses: actions/download-artifact@v4
+        with:
+          name: output850
+      - name: test
+        run: |
+          tar -x --use-compress-program=zstdmt -f output850.tzs
+          cd build/output/
+          ctest -E test-lockfree --timeout 3600 -V
+          export PHOTON_CI_EV_ENGINE=io_uring
+          ctest -E test-lockfree --timeout 3600 -V
+
+  gcc921:
+    needs: gcc850
+    runs-on: [self-hosted, compiler]
+    steps:
+      - name: Build921
+        run: |
+          source /opt/rh/gcc-toolset-9/enable
+          cmake --build build -j $(nproc) --clean-first -- VERBOSE=1
+          ln -f common/checksum/test/checksum.in build/output/
+          tar -c --use-compress-program=zstdmt -f output921.tzs build/output/
+      - name: Upload921
+        uses: actions/upload-artifact@v4
+        with:
+          name: output921
+          path: output921.tzs
+          retention-days: 5
+          compression-level: 0
+
+  test921:
+    needs: gcc921
+    runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/coldwings/photon-ut-base:latest
+      options: --cpus 4
+    steps:
+      - uses: szenius/set-timezone@v1.2
+        with:
+          timezoneLinux: "Asia/Shanghai"
+          timezoneMacos: "Asia/Shanghai"
+          timezoneWindows: "China Standard Time"
+      - uses: actions/download-artifact@v4
+        with:
+          name: output921
+      - name: test
+        run: |
+          tar -x --use-compress-program=zstdmt -f output921.tzs
+          cd build/output/
+          ctest -E test-lockfree --timeout 3600 -V
+          export PHOTON_CI_EV_ENGINE=io_uring
+          ctest -E test-lockfree --timeout 3600 -V
+
+  gcc1031:
+    needs: gcc921
+    runs-on: [self-hosted, compiler]
+    steps:
+      - name: Build1031
+        run: |
+          source /opt/rh/gcc-toolset-10/enable
+          cmake --build build -j $(nproc) --clean-first -- VERBOSE=1
+          ln -f common/checksum/test/checksum.in build/output/
+          tar -c --use-compress-program=zstdmt -f output1031.tzs build/output/
+      - name: Upload1031
+        uses: actions/upload-artifact@v4
+        with:
+          name: output1031
+          path: output1031.tzs
+          retention-days: 5
+          compression-level: 0
+
+  test1031:
+    needs: gcc1031
+    runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/coldwings/photon-ut-base:latest
+      options: --cpus 4
+    steps:
+      - uses: szenius/set-timezone@v1.2
+        with:
+          timezoneLinux: "Asia/Shanghai"
+          timezoneMacos: "Asia/Shanghai"
+          timezoneWindows: "China Standard Time"
+      - uses: actions/download-artifact@v4
+        with:
+          name: output1031
+      - name: test
+        run: |
+          tar -x --use-compress-program=zstdmt -f output1031.tzs
+          cd build/output/
+          ctest -E test-lockfree --timeout 3600 -V
+          export PHOTON_CI_EV_ENGINE=io_uring
+          ctest -E test-lockfree --timeout 3600 -V
+
+  gcc1121:
+    needs: gcc1031
+    runs-on: [self-hosted, compiler]
+    steps:
+      - name: Build1121
+        run: |
+          source /opt/rh/gcc-toolset-10/enable
+          cmake --build build -j --clean-first -- VERBOSE=1
+          ln -f common/checksum/test/checksum.in build/output/
+          tar -c --use-compress-program=zstdmt -f output1121.tzs build/output/
+      - name: Upload1121
+        uses: actions/upload-artifact@v4
+        with:
+          name: output1121
+          path: output1121.tzs
+          retention-days: 5
+          compression-level: 0
+
+  test1121:
+    needs: gcc1121
+    runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/coldwings/photon-ut-base:latest
+      options: --cpus 4
+    steps:
+      - uses: szenius/set-timezone@v1.2
+        with:
+          timezoneLinux: "Asia/Shanghai"
+          timezoneMacos: "Asia/Shanghai"
+          timezoneWindows: "China Standard Time"
+      - uses: actions/download-artifact@v4
+        with:
+          name: output1121
+      - name: test
+        run: |
+          tar -x --use-compress-program=zstdmt -f output1121.tzs
+          cd build/output/
+          ctest -E test-lockfree --timeout 3600 -V
+          export PHOTON_CI_EV_ENGINE=io_uring
+          ctest -E test-lockfree --timeout 3600 -V
+
+  gcc1211:
+    needs: gcc1121
+    runs-on: [self-hosted, compiler]
+    steps:
+      - name: Build1211
+        run: |
+          source /opt/rh/gcc-toolset-10/enable
+          cmake --build build -j $(nproc) --clean-first -- VERBOSE=1
+          ln -f common/checksum/test/checksum.in build/output/
+          tar -c --use-compress-program=zstdmt -f output1211.tzs build/output/
+      - name: Upload1211
+        uses: actions/upload-artifact@v4
+        with:
+          name: output1211
+          path: output1211.tzs
+          retention-days: 5
+          compression-level: 0
+
+  test1211:
+    needs: gcc1211
+    runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/coldwings/photon-ut-base:latest
+      options: --cpus 4
+    steps:
+      - uses: szenius/set-timezone@v1.2
+        with:
+          timezoneLinux: "Asia/Shanghai"
+          timezoneMacos: "Asia/Shanghai"
+          timezoneWindows: "China Standard Time"
+      - uses: actions/download-artifact@v4
+        with:
+          name: output1211
+      - name: test
+        run: |
+          tar -x --use-compress-program=zstdmt -f output1211.tzs
+          cd build/output/
+          ctest -E test-lockfree --timeout 3600 -V
+          export PHOTON_CI_EV_ENGINE=io_uring
+          ctest -E test-lockfree --timeout 3600 -V
+
+
diff --git a/.github/workflows/ci.linux.x86.yml b/.github/workflows/ci.linux.x86.yml
deleted file mode 100644
index b0101a2b..00000000
--- a/.github/workflows/ci.linux.x86.yml
+++ /dev/null
@@ -1,119 +0,0 @@
-name: Linux x86
-
-on:
-  push:
-    branches: [ "main", "release/*" ]
-  pull_request:
-    branches: [ "main", "release/*" ]
-
-jobs:
-  centos8-gcc921-epoll-release:
-    runs-on: ubuntu-latest
-
-    container:
-      image: dokken/centos-stream-8:sha-40294ce
-      options: --cpus 4
-
-    steps:
-      - uses: szenius/set-timezone@v1.2
-        with:
-          timezoneLinux: "Asia/Shanghai"
-          timezoneMacos: "Asia/Shanghai"
-          timezoneWindows: "China Standard Time"
-
-      - uses: actions/checkout@v3
-
-      - name: Install Dependencies
-        run: |
-          dnf install -y git gcc-c++ cmake 'dnf-command(config-manager)'
-          dnf install -y gcc-toolset-9-gcc-c++
-          dnf install -y openssl-devel libcurl-devel libaio-devel
-          dnf install -y epel-release
-          dnf config-manager --set-enabled powertools
-          dnf install -y gtest-devel gmock-devel gflags-devel fuse-devel libgsasl-devel e2fsprogs-devel
-
-      - name: Build
-        run: |
-          source /opt/rh/gcc-toolset-9/enable
-          cmake -B build -D CMAKE_BUILD_TYPE=MinSizeRel -D PHOTON_BUILD_TESTING=ON \
-            -D PHOTON_ENABLE_SASL=ON -D PHOTON_ENABLE_FUSE=ON -D PHOTON_ENABLE_EXTFS=ON
-          cmake --build build -j -- VERBOSE=1
-
-      - name: Test
-        run: |
-          cd build
-          ctest --timeout 3600 -V
-
-  centos8-gcc921-epoll-debug:
-    runs-on: ubuntu-latest
-
-    container:
-      image: dokken/centos-stream-8:sha-40294ce
-      options: --cpus 4
-
-    steps:
-      - uses: szenius/set-timezone@v1.2
-        with:
-          timezoneLinux: "Asia/Shanghai"
-          timezoneMacos: "Asia/Shanghai"
-          timezoneWindows: "China Standard Time"
-
-      - uses: actions/checkout@v3
-
-      - name: Install Dependencies
-        run: |
-          dnf install -y git gcc-c++ cmake 'dnf-command(config-manager)'
-          dnf install -y gcc-toolset-9-gcc-c++
-          dnf install -y openssl-devel libcurl-devel libaio-devel
-          dnf install -y epel-release
-          dnf config-manager --set-enabled powertools
-          dnf install -y gtest-devel gmock-devel gflags-devel fuse-devel libgsasl-devel e2fsprogs-devel
-
-      - name: Build
-        run: |
-          source /opt/rh/gcc-toolset-9/enable
-          cmake -B build -D CMAKE_BUILD_TYPE=Debug -D PHOTON_BUILD_TESTING=ON \
-            -D PHOTON_ENABLE_SASL=ON -D PHOTON_ENABLE_FUSE=ON -D PHOTON_ENABLE_EXTFS=ON
-          cmake --build build -j -- VERBOSE=1
-
-      - name: Test
-        run: |
-          cd build
-          ctest --timeout 3600 -V
-
-  centos8-gcc921-iouring-release:
-    runs-on: [self-hosted, Linux, X64, io_uring]
-
-    container:
-      image: dokken/centos-stream-8:sha-40294ce
-      options: --cpus 4
-
-    steps:
-      - uses: szenius/set-timezone@v1.2
-        with:
-          timezoneLinux: "Asia/Shanghai"
-          timezoneMacos: "Asia/Shanghai"
-          timezoneWindows: "China Standard Time"
-
-      - uses: actions/checkout@v3
-
-      - name: Install Dependencies
-        run: |
-          dnf install -y git gcc-c++ cmake
-          dnf install -y gcc-toolset-9-gcc-c++
-          dnf install -y autoconf automake libtool
-
-      - name: Build
-        run: |
-          source /opt/rh/gcc-toolset-9/enable
-          cmake -B build -D CMAKE_BUILD_TYPE=MinSizeRel \
-            -D PHOTON_BUILD_DEPENDENCIES=ON \
-            -D PHOTON_BUILD_TESTING=ON \
-            -D PHOTON_ENABLE_URING=ON
-          cmake --build build -j
-
-      - name: Test
-        run: |
-          cd build
-          export PHOTON_CI_EV_ENGINE=io_uring
-          ctest --timeout 3600 -V
diff --git a/.github/workflows/ci.macos.arm.yml b/.github/workflows/ci.macos.arm.yml
index 77ad0230..a8084fbd 100644
--- a/.github/workflows/ci.macos.arm.yml
+++ b/.github/workflows/ci.macos.arm.yml
@@ -7,15 +7,15 @@ on:
     branches: [ "main", "release/*" ]
 
 jobs:
-  macOS-clang-release:
-    runs-on: [self-hosted, macOS, ARM64]
+  macOS14-arm:
+    runs-on: macos-14
 
     steps:
-#      - uses: szenius/set-timezone@v1.2
-#        with:
-#          timezoneLinux: "Asia/Shanghai"
-#          timezoneMacos: "Asia/Shanghai"
-#          timezoneWindows: "China Standard Time"
+      - uses: szenius/set-timezone@v1.2
+        with:
+          timezoneLinux: "Asia/Shanghai"
+          timezoneMacos: "Asia/Shanghai"
+          timezoneWindows: "China Standard Time"
 
       - uses: actions/checkout@v3
 
@@ -28,8 +28,8 @@ jobs:
         run: |
           cmake -B ${{github.workspace}}/build -D PHOTON_BUILD_TESTING=ON -D CMAKE_BUILD_TYPE=Release \
             -D PHOTON_ENABLE_SASL=ON -DOPENSSL_ROOT_DIR=/usr/local/opt/openssl@3
-          cmake --build ${{github.workspace}}/build -j
+          cmake --build ${{github.workspace}}/build -j $(nproc)
 
       - name: Test
         working-directory: ${{github.workspace}}/build
-        run: ctest --timeout 3600 -V
+        run: ctest -E test-lockfree --timeout 3600 -V
diff --git a/.github/workflows/ci.macos.yml b/.github/workflows/ci.macos.x86.yml
similarity index 74%
rename from .github/workflows/ci.macos.yml
rename to .github/workflows/ci.macos.x86.yml
index 2ecfae9e..4ca1ef90 100644
--- a/.github/workflows/ci.macos.yml
+++ b/.github/workflows/ci.macos.x86.yml
@@ -7,8 +7,8 @@ on:
     branches: [ "main", "release/*" ]
 
 jobs:
-  macOS-12-Monterey-release:
-    runs-on: macos-12
+  macOS13-x86:
+    runs-on: macos-13
 
     steps:
     - uses: szenius/set-timezone@v1.2
@@ -20,17 +20,17 @@ jobs:
     - uses: actions/checkout@v3
 
     - name: Install Dependencies
-      shell: bash  
+      shell: bash
       run: |
-        brew install cmake openssl gflags googletest gsasl 
+        brew install cmake openssl gflags googletest gsasl
 
     - name: Build
       run: |
         cmake -B ${{github.workspace}}/build -D PHOTON_BUILD_TESTING=ON -D CMAKE_BUILD_TYPE=Release \
           -D PHOTON_ENABLE_SASL=ON -DOPENSSL_ROOT_DIR=/usr/local/opt/openssl@3
-        cmake --build ${{github.workspace}}/build -j
+        cmake --build ${{github.workspace}}/build -j $(nproc)
 
     - name: Test
       working-directory: ${{github.workspace}}/build
-      run: ctest --timeout 3600 -V
-      
+      run: ctest -E test-lockfree --timeout 3600 -V
+
diff --git a/CMake/Findgoogletest.cmake b/CMake/Findgoogletest.cmake
index 0de8d20f..e1e82570 100644
--- a/CMake/Findgoogletest.cmake
+++ b/CMake/Findgoogletest.cmake
@@ -1,6 +1,7 @@
 find_path(GOOGLETEST_INCLUDE_DIRS gtest/gtest.h gmock/gmock.h)
 
 find_library(GOOGLETEST_GTEST_LIBRARIES gtest)
+find_library(GOOGLETEST_GTEST_MAIN_LIBRARIES gtest_main)
 find_library(GOOGLETEST_GMOCK_LIBRARIES gmock)
 
 set(GOOGLETEST_LIBRARIES
diff --git a/CMake/build-from-src.cmake b/CMake/build-from-src.cmake
index caeb2598..c7bc59d7 100644
--- a/CMake/build-from-src.cmake
+++ b/CMake/build-from-src.cmake
@@ -43,7 +43,7 @@ function(build_from_src [dep])
                 UPDATE_DISCONNECTED ON
                 BUILD_IN_SOURCE ON
                 CONFIGURE_COMMAND ./configure --prefix=${BINARY_DIR}
-                BUILD_COMMAND sh -c "V=1 CFLAGS=\"-fPIC -g -O3 -Wall -Wextra -fno-stack-protector\" $(MAKE) -C src"
+                BUILD_COMMAND sh -c "V=1 CFLAGS=\"-fPIC -O3 -Wall -Wextra -fno-stack-protector\" $(MAKE) -C src"
                 INSTALL_COMMAND $(MAKE) install
         )
         set(URING_INCLUDE_DIRS ${BINARY_DIR}/include PARENT_SCOPE)
diff --git a/CMake/generate-ctest-packed-script.cmake b/CMake/generate-ctest-packed-script.cmake
new file mode 100644
index 00000000..ab0d6e8e
--- /dev/null
+++ b/CMake/generate-ctest-packed-script.cmake
@@ -0,0 +1,17 @@
+function(__GenerateStandaloneCTestScript DIR FILEPATH)
+    get_property(TGTS DIRECTORY "${DIR}" PROPERTY TESTS)
+    foreach(TGT IN LISTS TGTS)
+        file(APPEND ${FILEPATH} "add_test(${TGT} \"./${TGT}\")\n")
+    endforeach()
+
+    get_property(SUBDIRS DIRECTORY "${DIR}" PROPERTY SUBDIRECTORIES)
+    foreach(SUBDIR IN LISTS SUBDIRS)
+        __GenerateStandaloneCTestScript("${SUBDIR}" ${FILEPATH})
+    endforeach()
+endfunction()
+
+
+function(GenerateStandaloneCTestScript DIR FILEPATH)
+    file(WRITE ${FILEPATH} "")
+    __GenerateStandaloneCTestScript(${DIR} ${FILEPATH})
+endfunction()
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 63794d65..7c844c89 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -55,7 +55,7 @@ set(CMAKE_CXX_EXTENSIONS OFF)
 set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g")
 set(CMAKE_CXX_FLAGS_RELEASE "-O2 -DNDEBUG")
 set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -DNDEBUG -g")
-set(CMAKE_CXX_FLAGS_MINSIZEREL "-O2 -g")    # Only for CI test
+set(CMAKE_CXX_FLAGS_MINSIZEREL "-O2")    # Only for CI test
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 set(CMAKE_BUILD_RPATH_USE_ORIGIN ON)
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
@@ -159,7 +159,6 @@ file(GLOB PHOTON_SRC
         common/checksum/*.cpp
         common/executor/*.cpp
         common/memory-stream/*.cpp
-        common/stream-messenger/*.cpp
         fs/aligned-file.cpp
         fs/async_filesystem.cpp
         fs/exportfs.cpp
@@ -203,9 +202,8 @@ endif ()
 
 # An object library compiles source files but does not archive or link their object files.
 add_library(photon_obj OBJECT ${PHOTON_SRC})
-target_include_directories(photon_obj PRIVATE include ${OPENSSL_INCLUDE_DIRS} ${AIO_INCLUDE_DIRS}
-        ${ZLIB_INCLUDE_DIRS} ${CURL_INCLUDE_DIRS}
-)
+target_include_directories(photon_obj PRIVATE include)
+
 target_compile_definitions(photon_obj PRIVATE _FILE_OFFSET_BITS=64 FUSE_USE_VERSION=29)
 if (PHOTON_ENABLE_URING)
     target_include_directories(photon_obj PRIVATE ${URING_INCLUDE_DIRS})
@@ -334,16 +332,24 @@ endif ()
 
 # Build test cases
 if (PHOTON_BUILD_TESTING)
+    include_directories(photon_static ${GFLAGS_INCLUDE_DIRS} ${GOOGLETEST_INCLUDE_DIRS})
+    set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/examples-output)
+    link_libraries(${GFLAGS_LIBRARIES})
+    add_subdirectory(examples)
+
     include(CTest)
+    set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/output)
+    include(generate-ctest-packed-script)
 
     add_library(ci-tools STATIC test/ci-tools.cpp)
+    target_include_directories(ci-tools PRIVATE include)
 
-    include_directories(photon_static ${GFLAGS_INCLUDE_DIRS} ${GOOGLETEST_INCLUDE_DIRS})
     link_libraries(${GFLAGS_LIBRARIES} ${GOOGLETEST_LIBRARIES} ci-tools)
 
-    add_subdirectory(examples)
     add_subdirectory(common/checksum/test)
     add_subdirectory(common/test)
+    add_subdirectory(common/memory-stream/test)
+    add_subdirectory(common/executor/test)
     add_subdirectory(fs/test)
     add_subdirectory(io/test)
     add_subdirectory(net/test)
@@ -354,4 +360,5 @@ if (PHOTON_BUILD_TESTING)
     if (PHOTON_ENABLE_EXTFS)
         add_subdirectory(fs/extfs/test)
     endif ()
+    GenerateStandaloneCTestScript(${CMAKE_SOURCE_DIR} ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/CTestTestfile.cmake)
 endif ()
diff --git a/common/checksum/test/test_checksum.cpp b/common/checksum/test/test_checksum.cpp
index e6549ff4..8a3764b8 100644
--- a/common/checksum/test/test_checksum.cpp
+++ b/common/checksum/test/test_checksum.cpp
@@ -6,6 +6,7 @@
 #include <iostream>
 #include <sstream>
 #include <string>
+#include "../../../test/ci-tools.h"
 
 #ifndef DATA_DIR
 #define DATA_DIR ""
@@ -17,6 +18,7 @@
 class TestChecksum : public ::testing::Test {
     virtual void SetUp() {
         in.open(xstr(DATA_DIR) "checksum.in");
+        if (!in) in.open("checksum.in");
         ASSERT_TRUE(!!in);
         uint32_t value;
         std::string str;
@@ -70,6 +72,7 @@ TEST_F(TestChecksum, crc32c_sw) {
 
 int main(int argc, char **argv)
 {
+    if (!photon::is_using_default_engine()) return 0;
     ::testing::InitGoogleTest(&argc, argv);
     return RUN_ALL_TESTS();
 }
\ No newline at end of file
diff --git a/common/executor/test/CMakeLists.txt b/common/executor/test/CMakeLists.txt
new file mode 100644
index 00000000..303dd448
--- /dev/null
+++ b/common/executor/test/CMakeLists.txt
@@ -0,0 +1,24 @@
+add_definitions(-w)
+
+add_executable(test-executor-async test_async.cpp)
+target_link_libraries(test-executor-async PRIVATE photon_shared ${GOOGLETEST_GTEST_MAIN_LIBRARIES})
+add_test(NAME test-executor-async COMMAND $<TARGET_FILE:test-executor-async>)
+
+# add_executable(test-executor-easy test_easy.cpp ../../../third_party/easy_weak/easy_weak.cpp)
+# target_include_directories(ci-tools PRIVATE third_party/easy_weak)
+# target_link_libraries(test-executor-easy PRIVATE photon_shared ${GOOGLETEST_GTEST_MAIN_LIBRARIES})
+# add_test(NAME test-executor-easy COMMAND $<TARGET_FILE:test-executor-easy>)
+
+# add_executable(test-executor-easyexport test_easyexport.cpp ../../../third_party/easy_weak/easy_weak.cpp)
+# target_include_directories(ci-tools PRIVATE third_party/easy_weak)
+# target_link_libraries(test-executor-easyexport PRIVATE photon_shared ${GOOGLETEST_GTEST_MAIN_LIBRARIES})
+# add_test(NAME test-executor-easyexport COMMAND $<TARGET_FILE:test-executor-easyexport>)
+
+add_executable(test-executor-export_as_executor test_export_as_executor.cpp)
+target_link_libraries(test-executor-export_as_executor PRIVATE photon_shared ${GOOGLETEST_GTEST_MAIN_LIBRARIES})
+add_test(NAME test-executor-export_as_executor COMMAND $<TARGET_FILE:test-executor-export_as_executor>)
+
+add_executable(test-executor-std test_std.cpp)
+target_link_libraries(test-executor-std PRIVATE photon_shared ${GOOGLETEST_GTEST_MAIN_LIBRARIES})
+add_test(NAME test-executor-std COMMAND $<TARGET_FILE:test-executor-std>)
+
diff --git a/common/executor/test/test_async.cpp b/common/executor/test/test_async.cpp
index 6248113b..b7d6bc69 100644
--- a/common/executor/test/test_async.cpp
+++ b/common/executor/test/test_async.cpp
@@ -23,8 +23,10 @@ limitations under the License.
 #include <photon/fs/filesystem.h>
 #include <photon/fs/localfs.h>
 #include <photon/thread/thread.h>
-#include <sched.h>
-#include <immintrin.h>
+// #include <pthread.h>
+// #include <sched.h>
+#include "../../../test/ci-tools.h"
+// #include <immintrin.h>
 
 #include <chrono>
 #include <thread>
@@ -75,11 +77,7 @@ TEST(std_executor, perf) {
     for (int i = 0; i < th_num; i++) {
         ths.emplace_back([&] {
             for (int j = 0; j < app_num; j++) {
-        cpu_set_t cpuset;
-        CPU_ZERO(&cpuset);
-        CPU_SET(i, &cpuset);
-        pthread_setaffinity_np(pthread_self(),
-                                        sizeof(cpu_set_t), &cpuset);
+                photon::set_cpu_affinity(j);
                 auto start = std::chrono::high_resolution_clock::now();
                 eth.async_perform(new auto ([&] { cnt++; if (cnt % 10000 == 0) printf("%d\n", cnt);}));
                 auto end = std::chrono::high_resolution_clock::now();
diff --git a/common/executor/test/test_easyexport.cpp b/common/executor/test/test_easyexport.cpp
index 3266f395..b4eeafe0 100644
--- a/common/executor/test/test_easyexport.cpp
+++ b/common/executor/test/test_easyexport.cpp
@@ -84,7 +84,7 @@ TEST(easy_performer, test) {
     easy_atomic_set(count, 10000);
 
     std::thread([]() {
-        if (photon::init(photon::INIT_EVENT_DEFAULT, photon::INIT_IO_NONE))
+        if (ci_init_photon(photon::INIT_EVENT_DEFAULT, photon::INIT_IO_NONE))
             return -1;
         DEFER(photon::fini());
         fs::exportfs_init();
diff --git a/common/executor/test/test_export_as_executor.cpp b/common/executor/test/test_export_as_executor.cpp
index 33c1bd19..f0c66cfd 100644
--- a/common/executor/test/test_export_as_executor.cpp
+++ b/common/executor/test/test_export_as_executor.cpp
@@ -4,7 +4,6 @@
 #include <photon/common/alog.h>
 #include <photon/photon.h>
 #include <photon/thread/thread.h>
-
 #include <thread>
 
 TEST(enter_as_executor, test) {
diff --git a/common/lockfree_queue.h b/common/lockfree_queue.h
index 8bdcaac3..2d91d123 100644
--- a/common/lockfree_queue.h
+++ b/common/lockfree_queue.h
@@ -31,6 +31,8 @@ limitations under the License.
 #include <photon/common/utility.h>
 #include <photon/thread/thread.h>
 
+#define size_t uint64_t
+
 template <size_t x>
 struct Capacity_2expN {
     constexpr static size_t capacity = Capacity_2expN<(x >> 1)>::capacity << 1;
@@ -599,4 +601,7 @@ class RingChannel : public QueueType {
 };
 
 }  // namespace common
-}  // namespace photon
\ No newline at end of file
+}  // namespace photon
+
+#undef size_t
+
diff --git a/common/memory-stream/test/CMakeLists.txt b/common/memory-stream/test/CMakeLists.txt
new file mode 100644
index 00000000..5720f40c
--- /dev/null
+++ b/common/memory-stream/test/CMakeLists.txt
@@ -0,0 +1,6 @@
+add_definitions(-w)
+
+add_executable(test-memory-stream test.cpp)
+target_link_libraries(test-memory-stream PRIVATE photon_shared)
+add_test(NAME test-memory-stream COMMAND $<TARGET_FILE:test-memory-stream>)
+
diff --git a/common/memory-stream/test/test.cpp b/common/memory-stream/test/test.cpp
index 330e6544..4f006812 100644
--- a/common/memory-stream/test/test.cpp
+++ b/common/memory-stream/test/test.cpp
@@ -23,6 +23,7 @@ limitations under the License.
 #include <photon/common/utility.h>
 #include <photon/common/alog.h>
 #include <photon/common/alog-stdstring.h>
+#include "../../../test/ci-tools.h"
 
 using namespace std;
 using namespace photon;
@@ -82,6 +83,7 @@ TEST(MemoryStream, normalTest)
 
 int main(int argc, char **argv)
 {
+    if (!photon::is_using_default_engine()) return 0;
     log_output_level = ALOG_DEBUG;
     ::testing::InitGoogleTest(&argc, argv);
     photon::vcpu_init();
diff --git a/common/stream-messenger/messenger.cpp b/common/stream-messenger/messenger.cpp
deleted file mode 100644
index 293ac6e2..00000000
--- a/common/stream-messenger/messenger.cpp
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
-Copyright 2022 The Photon Authors
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-#include "messenger.h"
-#include <photon/common/alog.h>
-
-namespace StreamMessenger
-{
-    // This class turns a stream into a message channel, by adding a `Header` to the
-    // message before send it, and extracting the header immediately after recv it.
-    // The stream can be any `IStream` object, like TCP socket or UNIX domain socket.
-    class Messenger : public IMessageChannel
-    {
-    public:
-        IStream* m_stream;
-        Header m_header;       // used for recv, in case of insufficient buffer space
-        Messenger(IStream* stream)
-        {
-            m_stream = stream;
-            m_header.magic = 0;
-        }
-        virtual ssize_t send(iovector& msg) override
-        {
-            auto size = msg.sum();
-            if (size > UINT32_MAX - sizeof(Header))
-                LOG_ERRNO_RETURN(EINVAL, -1, "the message is too large to send (` > 4GB)", size);
-
-            Header h;
-            h.size = (uint32_t)size;
-            msg.push_front({&h, sizeof(h)});
-            ssize_t ret = m_stream->writev(msg.iovec(), msg.iovcnt());
-            if (ret < (ssize_t)(size + sizeof(h)))
-                LOG_ERROR_RETURN(0, -1, "failed to write to underlay stream ", m_stream);
-
-            return (ssize_t)size;
-        }
-        virtual ssize_t recv(iovector& msg) override
-        {
-            if (m_header.magic != Header::MAGIC)
-            {
-                ssize_t ret = m_stream->read(&m_header, sizeof(m_header));
-                if (ret < (ssize_t)sizeof(m_header))
-                    LOG_ERRNO_RETURN(0, -1, "failed to read the header from stream ", m_stream);
-                if (m_header.magic != Header::MAGIC)
-                    LOG_ERROR_RETURN(EBADMSG, -1, "invalid header magic in the recvd message");
-                if (m_header.version > Header::VERSION)
-                    LOG_ERROR_RETURN(EBADMSG, -1 , "invlaid header version in the recvd message");
-            }
-
-            size_t size = msg.truncate(m_header.size);
-            auto delta = (ssize_t)size - (ssize_t)m_header.size;
-            if (delta < 0)
-                LOG_ERROR_RETURN(ENOBUFS, delta, "insufficient buffer space (and allocation "
-                                 "failed), need ` more bytes", -delta);
-
-            m_header.magic = 0;   // clear the magic state
-            ssize_t ret = m_stream->readv(msg.iovec(), msg.iovcnt());
-            if (ret < (ssize_t)m_header.size)
-                LOG_ERRNO_RETURN(0, -1, "failed to readv the message from stream ", m_stream);
-
-            return m_header.size;
-        }
-        virtual uint64_t flags() override
-        {
-            return 0;
-        }
-        virtual uint64_t flags(uint64_t f) override
-        {
-            return f;
-        }
-    };
-    IMessageChannel* new_messenger(IStream* stream)
-    {
-        return new Messenger(stream);
-    }
-}
diff --git a/common/stream-messenger/messenger.h b/common/stream-messenger/messenger.h
deleted file mode 100644
index dbeffa87..00000000
--- a/common/stream-messenger/messenger.h
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
-Copyright 2022 The Photon Authors
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-#pragma once
-#include <cinttypes>
-#include <photon/common/message-channel.h>
-#include <photon/common/stream.h>
-
-// a message channel based on any `IStream` object
-namespace StreamMessenger
-{
-    // header of the stream message channel
-    struct Header
-    {
-        const static uint64_t MAGIC   = 0x4962b4d24caa439e;
-        const static uint32_t VERSION = 0;
-
-        uint64_t magic   = MAGIC;       // the header magic
-        uint32_t version = VERSION;     // version of the message
-        uint32_t size;                  // size of the payload, not including the header
-        uint64_t reserved = 0;          // padding to 24 bytes
-    };
-
-    // This class turns a stream into a message channel, by adding a `Header` to the
-    // message before send it, and extracting the header immediately after recv it.
-    // The stream can be any `IStream` object, like TCP socket or UNIX domain socket.
-    IMessageChannel* new_messenger(IStream* stream);
-}
diff --git a/common/stream-messenger/test/test.cpp b/common/stream-messenger/test/test.cpp
deleted file mode 100644
index 03d563e8..00000000
--- a/common/stream-messenger/test/test.cpp
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
-Copyright 2022 The Photon Authors
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-#include <photon/common/stream-messenger/messenger.h>
-#include <memory>
-#include <algorithm>
-#include <string>
-#include <gtest/gtest.h>
-#include <photon/thread/thread.h>
-#include <photon/common/memory-stream/memory-stream.h>
-#include <photon/common/utility.h>
-#include <photon/common/alog.h>
-using namespace std;
-using namespace photon;
-using namespace StreamMessenger;
-
-char STR[] = "abcdefghijklmnopqrstuvwxyz";
-
-void* echo_server(void* mc_)
-{
-    auto mc = (IMessageChannel*)mc_;
-
-    IOVector msg;
-    size_t blksz = 2;
-    while(true)
-    {
-        auto ret = mc->recv(msg);
-        if (ret == 0)
-        {
-            LOG_DEBUG("zero-lengthed message recvd, quit");
-            break;
-        }
-        if (ret < 0)
-        {
-            if (errno == ENOBUFS)
-            {
-                blksz *= 2;
-                msg.push_back(blksz);
-                LOG_DEBUG("adding ` bytes to the msg iovector", blksz);
-                continue;
-            }
-
-            LOG_ERROR_RETURN(0, nullptr, "failed to recv msg");
-        }
-
-        auto ret2 = mc->send(msg);
-        EXPECT_EQ(ret2, ret);
-    }
-    LOG_DEBUG("exit");
-    return nullptr;
-}
-
-TEST(StreamMessenger, DISABLED_normalTest)
-{
-    auto ds = new_duplex_memory_stream(64);
-    DEFER(delete ds);
-
-    auto mca = new_messenger(ds->endpoint_a);
-    DEFER(delete mca);
-    thread_create(&echo_server, mca);
-
-    auto mcb = new_messenger(ds->endpoint_b);
-    DEFER(delete mcb);
-
-    auto size = LEN(STR) - 1;
-    auto ret = mcb->send(STR, size);
-    EXPECT_EQ(ret, size);
-    LOG_DEBUG(ret, " bytes sent");
-    char buf[size];
-    auto ret2 = mcb->recv(buf, sizeof(buf));
-    EXPECT_EQ(ret2, sizeof(buf));
-    EXPECT_EQ(memcmp(buf, STR, size), 0);
-    LOG_DEBUG(ret, " bytes recvd and verified");
-
-    mcb->send(nullptr, 0);
-    thread_yield();
-    LOG_DEBUG("exit");
-}
-
-int main(int argc, char **argv)
-{
-    log_output_level = ALOG_DEBUG;
-    ::testing::InitGoogleTest(&argc, argv);
-    photon::vcpu_init();
-    DEFER(photon::vcpu_fini());
-    auto ret = RUN_ALL_TESTS();
-    return 0;
-}
diff --git a/common/test/CMakeLists.txt b/common/test/CMakeLists.txt
index 09497252..27896b6f 100644
--- a/common/test/CMakeLists.txt
+++ b/common/test/CMakeLists.txt
@@ -21,9 +21,9 @@ add_executable(test-constexprstr test_constexprstr.cpp)
 target_link_libraries(test-constexprstr PRIVATE photon_shared)
 add_test(NAME test-constexprstr COMMAND $<TARGET_FILE:test-constexprstr>)
 
-#add_executable(test-lockfree test_lockfree.cpp)
-#target_link_libraries(test-lockfree PRIVATE photon_shared)
-#add_test(NAME test-lockfree COMMAND $<TARGET_FILE:test-lockfree>)
+add_executable(test-lockfree test_lockfree.cpp)
+target_link_libraries(test-lockfree PRIVATE photon_shared)
+add_test(NAME test-lockfree COMMAND $<TARGET_FILE:test-lockfree>)
 
 add_executable(test-alog test_alog.cpp x.cpp)
 target_link_libraries(test-alog PRIVATE photon_shared)
diff --git a/common/test/perf_objcache.cpp b/common/test/perf_objcache.cpp
index 5db884af..5832c245 100644
--- a/common/test/perf_objcache.cpp
+++ b/common/test/perf_objcache.cpp
@@ -9,6 +9,7 @@
 #include <unordered_map>
 #include <vector>
 #include <algorithm>
+#include "../../test/ci-tools.h"
 
 #include "../expirecontainer.h"
 
diff --git a/common/test/test.cpp b/common/test/test.cpp
index 28aa6a22..96dec844 100644
--- a/common/test/test.cpp
+++ b/common/test/test.cpp
@@ -51,6 +51,9 @@ limitations under the License.
 #include <gnu/libc-version.h>
 #endif
 
+#include "../../test/ci-tools.h"
+
+
 using namespace std;
 
 char str[] = "2018/01/05 21:53:28|DEBUG| 2423423|test.cpp:254|virtual void LOGPerf_1M_memcpy_Test::TestBody():aksdjfj 234:^%$#@341234  hahah `:jksld88423CACE::::::::::::::::::::::::::::::::::::::::::::::::::::";
@@ -1170,6 +1173,7 @@ TEST(update_now, after_idle_sleep) {
 // #endif
 int main(int argc, char **argv)
 {
+    if (!photon::is_using_default_engine()) return 0;
     photon::vcpu_init();
     DEFER(photon::vcpu_fini());
     char a[100]{}, b[100]{};
diff --git a/common/test/test_alog.cpp b/common/test/test_alog.cpp
index ce5f3dd4..e723e446 100644
--- a/common/test/test_alog.cpp
+++ b/common/test/test_alog.cpp
@@ -27,6 +27,7 @@ limitations under the License.
 #include <stdint.h>
 #include <unistd.h>
 #include <fcntl.h>
+#include "../../test/ci-tools.h"
 
 class LogOutputTest : public ILogOutput {
 public:
@@ -575,6 +576,7 @@ TEST(ALOG, IPAddr) {
 
 int main(int argc, char **argv)
 {
+    if (!photon::is_using_default_engine()) return 0;
     photon::vcpu_init();
     DEFER(photon::vcpu_fini());
     ::testing::InitGoogleTest(&argc, argv);
diff --git a/common/test/test_constexprstr.cpp b/common/test/test_constexprstr.cpp
index 786c05f2..00875afd 100644
--- a/common/test/test_constexprstr.cpp
+++ b/common/test/test_constexprstr.cpp
@@ -22,6 +22,7 @@ limitations under the License.
 #include "../alog-stdstring.h"
 #include "../alog.h"
 #include "../conststr.h"
+#include "../../test/ci-tools.h"
 
 DEFINE_ENUM_STR(VERBS, verbs, UNKNOW, DELETE, GET, HEAD, POST, PUT, CONNECT,
                 OPTIONS, TRACE, COPY, LOCK, MKCOL, MOV, PROPFIND, PROPPATCH,
@@ -113,6 +114,7 @@ TEST(TString, JoinAndSplit) {
 }
 
 int main(int argc, char** argv) {
+    if (!photon::is_using_default_engine()) return 0;
     ::testing::InitGoogleTest(&argc, argv);
     int ret = RUN_ALL_TESTS();
     LOG_ERROR_RETURN(0, ret, VALUE(ret));
diff --git a/common/test/test_lockfree.cpp b/common/test/test_lockfree.cpp
index a4977267..b60091bf 100644
--- a/common/test/test_lockfree.cpp
+++ b/common/test/test_lockfree.cpp
@@ -19,13 +19,14 @@ limitations under the License.
 #include <pthread.h>
 
 #include <array>
-#include <boost/lockfree/policies.hpp>
-#include <boost/lockfree/queue.hpp>
-#include <boost/lockfree/spsc_queue.hpp>
+// #include <boost/lockfree/policies.hpp>
+// #include <boost/lockfree/queue.hpp>
+// #include <boost/lockfree/spsc_queue.hpp>
 #include <mutex>
 #include <random>
 #include <thread>
 #include <vector>
+#include "../../test/ci-tools.h"
 
 static constexpr size_t sender_num = 4;
 static constexpr size_t receiver_num = 4;
@@ -46,8 +47,8 @@ LockfreeBatchMPMCRingQueue<int, capacity> lbqueue;
 LockfreeSPSCRingQueue<int, capacity> cqueue;
 std::mutex rlock, wlock;
 
-boost::lockfree::queue<int, boost::lockfree::capacity<capacity>> bqueue;
-boost::lockfree::spsc_queue<int, boost::lockfree::capacity<capacity>> squeue;
+// boost::lockfree::queue<int, boost::lockfree::capacity<capacity>> bqueue;
+// boost::lockfree::spsc_queue<int, boost::lockfree::capacity<capacity>> squeue;
 
 struct WithLock {
     template <typename T>
@@ -75,10 +76,7 @@ int test_queue(const char *name, QType &queue) {
     auto begin = std::chrono::steady_clock::now();
     for (size_t i = 0; i < receiver_num; i++) {
         receivers.emplace_back([i, &queue] {
-            cpu_set_t cpuset;
-            CPU_ZERO(&cpuset);
-            CPU_SET(i, &cpuset);
-            pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
+            photon::set_cpu_affinity(i);
             std::chrono::nanoseconds rspent(std::chrono::nanoseconds(0));
             for (size_t x = 0; x < items_num / receiver_num; x++) {
                 int t;
@@ -101,10 +99,7 @@ int test_queue(const char *name, QType &queue) {
     }
     for (size_t i = 0; i < sender_num; i++) {
         senders.emplace_back([i, &queue] {
-            cpu_set_t cpuset;
-            CPU_ZERO(&cpuset);
-            CPU_SET(i + receiver_num, &cpuset);
-            pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
+            photon::set_cpu_affinity(i);
             std::chrono::nanoseconds wspent{std::chrono::nanoseconds(0)};
             for (size_t x = 0; x < items_num / sender_num; x++) {
                 auto tm = std::chrono::high_resolution_clock::now();
@@ -150,10 +145,7 @@ int test_queue_batch(const char *name, QType &queue) {
     auto begin = std::chrono::steady_clock::now();
     for (size_t i = 0; i < receiver_num; i++) {
         receivers.emplace_back([i, &queue] {
-            cpu_set_t cpuset;
-            CPU_ZERO(&cpuset);
-            CPU_SET(i, &cpuset);
-            pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
+            photon::set_cpu_affinity(i);
             int buffer[32];
             size_t size;
             int amount = items_num / receiver_num;
@@ -181,10 +173,7 @@ int test_queue_batch(const char *name, QType &queue) {
     }
     for (size_t i = 0; i < sender_num; i++) {
         senders.emplace_back([i, &queue] {
-            cpu_set_t cpuset;
-            CPU_ZERO(&cpuset);
-            CPU_SET(i + receiver_num, &cpuset);
-            pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
+            photon::set_cpu_affinity(i);
             std::chrono::nanoseconds wspent{std::chrono::nanoseconds(0)};
             for (size_t x = 0; x < items_num / sender_num; x++) {
                 auto tm = std::chrono::high_resolution_clock::now();
@@ -223,11 +212,12 @@ int test_queue_batch(const char *name, QType &queue) {
 }
 
 int main() {
-    test_queue<NoLock, NoLock>("BoostQueue", bqueue);
+    if (!photon::is_using_default_engine()) return 0;
+    // test_queue<NoLock, NoLock>("BoostQueue", bqueue);
     test_queue<NoLock, NoLock>("PhotonLockfreeMPMCQueue", lqueue);
     test_queue<NoLock, NoLock>("PhotonLockfreeBatchMPMCQueue", lbqueue);
     test_queue_batch<NoLock, NoLock>("PhotonLockfreeBatchMPMCQueue+Batch", lbqueue);
-    test_queue<WithLock, WithLock>("BoostSPSCQueue", squeue);
+    // test_queue<WithLock, WithLock>("BoostSPSCQueue", squeue);
     test_queue<WithLock, WithLock>("PhotonSPSCQueue", cqueue);
     test_queue_batch<WithLock, WithLock>("PhotonSPSCQueue+Batch", cqueue);
 }
diff --git a/common/test/test_objcache.cpp b/common/test/test_objcache.cpp
index 28538607..683b6067 100644
--- a/common/test/test_objcache.cpp
+++ b/common/test/test_objcache.cpp
@@ -21,11 +21,11 @@ limitations under the License.
 
 #undef private
 #undef protected
+#include <thread>
 #include <gtest/gtest.h>
 #include <photon/thread/thread.h>
 #include <photon/common/alog.h>
-
-#include <thread>
+#include "../../test/ci-tools.h"
 
 static int thread_local release_cnt = 0;
 struct ShowOnDtor {
@@ -68,8 +68,6 @@ void* objcache(void* arg) {
 }
 
 TEST(ObjectCache, release_cycle) {
-    // photon::vcpu_init();
-    // DEFER(photon::vcpu_fini());
     set_log_output_level(ALOG_INFO);
     DEFER(set_log_output_level(ALOG_DEBUG));
     ObjectCache<int, ShowOnDtor*> ocache(1000UL * 1000 * 10);
@@ -93,8 +91,6 @@ TEST(ObjectCache, release_cycle) {
 
 TEST(ObjectCache, timeout_refresh) {
     release_cnt = 0;
-    // photon::vcpu_init();
-    // DEFER(photon::vcpu_fini());
     set_log_output_level(ALOG_INFO);
     DEFER(set_log_output_level(ALOG_DEBUG));
     ObjectCache<int, ShowOnDtor*> ocache(1000UL * 1000);
@@ -129,8 +125,6 @@ void *ph_act(void *arg) {
 
 TEST(ObjectCache, ctor_may_yield_and_null) {
     release_cnt = 0;
-    // photon::vcpu_init();
-    // DEFER(photon::vcpu_fini());
     set_log_output_level(ALOG_INFO);
     DEFER(set_log_output_level(ALOG_DEBUG));
     ObjectCache<int, ShowOnDtor*> ocache(1000UL * 1000);
@@ -149,8 +143,6 @@ TEST(ObjectCache, ctor_may_yield_and_null) {
 }
 
 TEST(ObjectCache, multithread) {
-    // photon::vcpu_init();
-    // DEFER(photon::vcpu_fini());
     set_log_output_level(ALOG_INFO);
     DEFER(set_log_output_level(ALOG_DEBUG));
     ObjectCache<int, ShowOnDtor*> ocache(1000UL * 1000 * 10);
@@ -292,8 +284,6 @@ TEST(ObjectCache, borrow_with_once) {
 }
 
 TEST(ExpireContainer, expire_container) {
-    // photon::vcpu_init();
-    // DEFER(photon::vcpu_fini());
     char key[10] = "hello";
     char key2[10] = "hello";
     ExpireContainer<std::string, int, bool> expire(1000 *
@@ -334,8 +324,6 @@ TEST(ExpireContainer, refresh) {
 }
 
 TEST(ExpireList, expire_container) {
-    // photon::vcpu_init();
-    // DEFER(photon::vcpu_fini());
     char key[10] = "hello";
     ExpireList<std::string> expire(1000 * 1000);  // expire in 100ms
     expire.keep_alive(key, true);
@@ -360,6 +348,7 @@ TEST(ExpireList, expire_container) {
 }
 
 int main(int argc, char** argv) {
+    if (!photon::is_using_default_engine()) return 0;
     photon::vcpu_init();
     DEFER(photon::vcpu_fini());
     ::testing::InitGoogleTest(&argc, argv);
diff --git a/common/test/test_scalepool.cpp b/common/test/test_scalepool.cpp
index 4e260435..b20c497c 100644
--- a/common/test/test_scalepool.cpp
+++ b/common/test/test_scalepool.cpp
@@ -22,9 +22,11 @@ limitations under the License.
 #undef protected
 
 #include "../alog.h"
+#include <vector>
 #include <gtest/gtest.h>
 #include <photon/thread/thread.h>
-#include <vector>
+#include "../../test/ci-tools.h"
+
 
 TEST(IdentityPoolGC, basic) {
     auto pool = new_identity_pool<int>(128);
@@ -123,6 +125,7 @@ TEST(IOAlloc, basic) {
 }
 int main(int argc, char **argv)
 {
+    if (!photon::is_using_default_engine()) return 0;
     ::testing::InitGoogleTest(&argc, argv);
     photon::vcpu_init();
     DEFER(photon::vcpu_fini());
diff --git a/common/test/test_throttle.cpp b/common/test/test_throttle.cpp
index b8d0e6d5..c0a3b411 100644
--- a/common/test/test_throttle.cpp
+++ b/common/test/test_throttle.cpp
@@ -1,10 +1,10 @@
+#include <chrono>
 #include <gtest/gtest.h>
 #include <photon/common/alog.h>
 #include <photon/common/throttle.h>
 #include <photon/common/utility.h>
 #include <photon/photon.h>
-
-#include <chrono>
+#include "../../test/ci-tools.h"
 
 TEST(Throttle, basic) {
     // baseline
@@ -162,6 +162,7 @@ TEST(Throttle, try_consume) {
 }
 
 int main(int argc, char** argv) {
+    if (!photon::is_using_default_engine()) return 0;
     photon::init(0, 0);
     DEFER(photon::fini());
     testing::InitGoogleTest(&argc, argv);
diff --git a/fs/exportfs.cpp b/fs/exportfs.cpp
index 9563ed02..d2bf16b7 100644
--- a/fs/exportfs.cpp
+++ b/fs/exportfs.cpp
@@ -561,6 +561,7 @@ namespace fs
         ExportBase::ref = 1;
         if (thread_pool_capacity != 0) ExportBase::pool = new_thread_pool(thread_pool_capacity);
         evloop->async_run();
+        LOG_INFO(" OK");
         return 0;
     }
     int exportfs_fini()
diff --git a/fs/test/CMakeLists.txt b/fs/test/CMakeLists.txt
index fb7421b1..373dc402 100644
--- a/fs/test/CMakeLists.txt
+++ b/fs/test/CMakeLists.txt
@@ -5,7 +5,7 @@ target_link_libraries(test-fs PRIVATE photon_shared)
 add_test(NAME test-fs COMMAND $<TARGET_FILE:test-fs>)
 
 # add_executable(test-exportfs test_exportfs.cpp)
-# target_link_libraries(test-exportfs PRIVATE photon_static)
+# target_link_libraries(test-exportfs PRIVATE photon_shared)
 # add_test(NAME test-exportfs COMMAND $<TARGET_FILE:test-exportfs>)
 
 add_executable(test-filecopy test_filecopy.cpp)
diff --git a/fs/test/test.cpp b/fs/test/test.cpp
index 8e7ab6ea..665a5553 100644
--- a/fs/test/test.cpp
+++ b/fs/test/test.cpp
@@ -1369,12 +1369,7 @@ TEST(Walker, basic) {
 
 int main(int argc, char **argv){
     ::testing::InitGoogleTest(&argc, argv);
-#ifdef __linux__
-    int ev_engine = photon::INIT_EVENT_EPOLL;
-#else
-    int ev_engine = photon::INIT_EVENT_KQUEUE;
-#endif
-    if (photon::init(ev_engine, photon::INIT_IO_NONE))
+    if (photon::init(photon::INIT_EVENT_DEFAULT, photon::INIT_IO_NONE))
         return -1;
     DEFER(photon::fini());
     int ret = RUN_ALL_TESTS();
diff --git a/fs/test/test_exportfs.cpp b/fs/test/test_exportfs.cpp
index 5abfb8de..2c11abc7 100644
--- a/fs/test/test_exportfs.cpp
+++ b/fs/test/test_exportfs.cpp
@@ -29,17 +29,22 @@ limitations under the License.
 #include <thread>
 #include <utime.h>
 #include <sys/time.h>
-#include <sys/sysmacros.h>
+// #include <sys/sysmacros.h>
 
 using namespace photon;
 using namespace photon::fs;
 using namespace testing;
 
-#ifdef __linux__
-static const int event_engine = photon::INIT_EVENT_EPOLL;
-#else
-static const int event_engine = photon::INIT_EVENT_KQUEUE;
-#endif
+inline void photon_init() {
+    init(INIT_EVENT_DEFAULT, INIT_IO_NONE);
+    exportfs_init();
+}
+
+inline void photon_fini() {
+    exportfs_fini();
+    fini();
+}
+
 constexpr uint64_t magic = 150820;
 static std::atomic<int> work(0);
 
@@ -99,14 +104,6 @@ int callbackvoid(void*, AsyncResult<void>* ret) {
     }
 
 TEST(ExportFS, basic) {
-    photon::vcpu_init();
-    photon::fd_events_init(event_engine);
-    exportfs_init();
-    DEFER({
-        exportfs_fini();
-        photon::fd_events_fini();
-        photon::vcpu_fini();
-    });
     PMock::MockNullFile* mockfile = new PMock::MockNullFile();
     PMock::MockNullFileSystem* mockfs = new PMock::MockNullFileSystem();
     PMock::MockNullDIR* mockdir = new PMock::MockNullDIR();
@@ -257,20 +254,19 @@ TEST(ExportFS, init_fini_failed_situation) {
     DEFER({
         log_output = _o_output;
     });
-    auto ret = exportfs_fini();
-    EXPECT_EQ(-1, ret);
-    EXPECT_EQ(ENOSYS, errno);
-    photon::vcpu_init();
-    photon::fd_events_init(event_engine);
-    ret = exportfs_init();
-    EXPECT_EQ(0, ret);
-    ret = exportfs_init();
+
+    auto ret = exportfs_init();
     EXPECT_EQ(-1, ret);
     EXPECT_EQ(EALREADY, errno);
+
     ret = exportfs_fini();
     EXPECT_EQ(0, ret);
-    photon::fd_events_fini();
-    // photon::vcpu_fini();
+    ret = exportfs_fini();
+    EXPECT_EQ(-1, ret);
+    EXPECT_EQ(ENOSYS, errno);
+
+    ret = exportfs_init();
+    EXPECT_EQ(0, ret);
 }
 
 TEST(ExportFS, op_failed_situation) {
@@ -279,14 +275,6 @@ TEST(ExportFS, op_failed_situation) {
     DEFER({
         log_output = _o_output;
     });
-    // photon::vcpu_init();
-    photon::fd_events_init(event_engine);
-    exportfs_init();
-    DEFER({
-        exportfs_fini();
-        photon::fd_events_fini();
-        // photon::vcpu_fini();
-    });
     PMock::MockNullFile* mockfile = new PMock::MockNullFile;
     errno = 0;
     EXPECT_CALL(*mockfile, read(_, _))
@@ -308,14 +296,6 @@ TEST(ExportFS, op_failed_situation) {
 }
 
 TEST(ExportFS, xattr) {
-    photon::vcpu_init();
-    photon::fd_events_init(event_engine);
-    exportfs_init();
-    DEFER({
-        exportfs_fini();
-        photon::fd_events_fini();
-        photon::vcpu_fini();
-    });
     PMock::MockNullFile* mockfile = new PMock::MockNullFile();
     PMock::MockNullFileSystem* mockfs = new PMock::MockNullFileSystem();
     auto file = dynamic_cast<IAsyncFileXAttr*>(export_as_async_file(mockfile));
@@ -376,14 +356,8 @@ TEST(ExportFS, xattr_sync) {
     IFileSystemXAttr* fs = nullptr;
 
     std::thread th([&]{
-        photon::vcpu_init();
-        photon::fd_events_init(event_engine);
-        exportfs_init();
-        DEFER({
-            exportfs_fini();
-            photon::fd_events_fini();
-            photon::vcpu_fini();
-        });
+        photon_init();
+        DEFER(photon_fini());
         file = dynamic_cast<IFileXAttr*>(export_as_sync_file(mockfile));
         fs = dynamic_cast<IFileSystemXAttr*>(export_as_sync_fs(mockfs));
         sem.wait(1);
@@ -430,9 +404,10 @@ TEST(ExportFS, xattr_sync) {
 
 int main(int argc, char **argv)
 {
-    photon::vcpu_init();
-    DEFER(photon::vcpu_fini());
+    photon_init();
+    DEFER(photon_fini());
     ::testing::InitGoogleTest(&argc, argv);
     int ret = RUN_ALL_TESTS();
     LOG_ERROR_RETURN(0, ret, VALUE(ret));
+    return ret;
 }
diff --git a/fs/test/test_throttledfile.cpp b/fs/test/test_throttledfile.cpp
index e0839266..489d09d4 100644
--- a/fs/test/test_throttledfile.cpp
+++ b/fs/test/test_throttledfile.cpp
@@ -35,6 +35,7 @@ limitations under the License.
 #include <photon/thread/thread11.h>
 #include <photon/thread/thread.h>
 #include <photon/fs/localfs.h>
+#include "../../test/ci-tools.h"
 
 #include "mock.h"
 
@@ -433,6 +434,7 @@ TEST(ThrottledFs, concurrent){
 
 int main(int argc, char **argv)
 {
+    if (!photon::is_using_default_engine()) return 0;
     ::testing::InitGoogleTest(&argc, argv);
     int ret = RUN_ALL_TESTS();
     LOG_ERROR_RETURN(0, ret, VALUE(ret));
diff --git a/io/fd-events.h b/io/fd-events.h
index d201a5be..71fa2e4f 100644
--- a/io/fd-events.h
+++ b/io/fd-events.h
@@ -138,7 +138,7 @@ DECLARE_MASTER_AND_CASCADING_ENGINE(select);
 DECLARE_MASTER_AND_CASCADING_ENGINE(iouring);
 DECLARE_MASTER_AND_CASCADING_ENGINE(kqueue);
 
-inline int fd_events_init(int master_engine) {
+inline int fd_events_init(uint64_t master_engine) {
     switch (master_engine) {
 #ifdef __linux__
         case INIT_EVENT_EPOLL:
diff --git a/io/test/CMakeLists.txt b/io/test/CMakeLists.txt
index de102709..00a08fe8 100644
--- a/io/test/CMakeLists.txt
+++ b/io/test/CMakeLists.txt
@@ -21,6 +21,6 @@ target_link_libraries(test-syncio PRIVATE photon_shared)
 add_test(NAME test-syncio COMMAND $<TARGET_FILE:test-syncio>)
 
 add_executable(test-iouring test-iouring.cpp)
-target_link_libraries(test-iouring PRIVATE photon_static)
+target_link_libraries(test-iouring PRIVATE photon_shared)
 add_test(NAME test-iouring COMMAND $<TARGET_FILE:test-iouring>)
 endif ()
\ No newline at end of file
diff --git a/io/test/test-iouring.cpp b/io/test/test-iouring.cpp
index 464bb088..9ab8c2d7 100644
--- a/io/test/test-iouring.cpp
+++ b/io/test/test-iouring.cpp
@@ -342,7 +342,8 @@ TEST(perf, DISABLED_read) {
 class event_engine : public testing::Test {
 protected:
     void SetUp() override {
-        GTEST_ASSERT_EQ(0, photon::init(ci_ev_engine, photon::INIT_IO_NONE));
+        GTEST_ASSERT_EQ(0, photon::init(photon::INIT_EVENT_DEFAULT,
+                                        photon::INIT_IO_NONE));
 #ifdef PHOTON_URING
         engine = (ci_ev_engine == photon::INIT_EVENT_EPOLL) ? photon::new_epoll_cascading_engine()
                                                           : photon::new_iouring_cascading_engine();
@@ -621,6 +622,5 @@ int main(int argc, char** arg) {
     testing::InitGoogleTest(&argc, arg);
     testing::FLAGS_gtest_break_on_failure = true;
     gflags::ParseCommandLineFlags(&argc, &arg, true);
-    ci_parse_env();
     return RUN_ALL_TESTS();
 }
diff --git a/net/http/test/CMakeLists.txt b/net/http/test/CMakeLists.txt
index 582c4269..f60e99ba 100644
--- a/net/http/test/CMakeLists.txt
+++ b/net/http/test/CMakeLists.txt
@@ -1,13 +1,13 @@
 add_definitions(-w)
 
 add_executable(client_perf client_perf.cpp)
-target_link_libraries(client_perf PRIVATE photon_static)
+target_link_libraries(client_perf PRIVATE photon_shared)
 
 add_executable(server_perf server_perf.cpp)
 target_link_libraries(server_perf PRIVATE photon_shared)
 
 add_executable(pure_libcurl pure_libcurl.cpp)
-target_link_libraries(pure_libcurl PRIVATE photon_static)
+target_link_libraries(pure_libcurl PRIVATE photon_shared)
 
 add_executable(client_function_test client_function_test.cpp)
 target_link_libraries(client_function_test PRIVATE photon_shared)
diff --git a/net/test/CMakeLists.txt b/net/test/CMakeLists.txt
index 1d10876d..d4cbbc96 100644
--- a/net/test/CMakeLists.txt
+++ b/net/test/CMakeLists.txt
@@ -13,7 +13,7 @@ target_link_libraries(test-sockpool PRIVATE photon_shared)
 add_test(NAME test-sockpool COMMAND $<TARGET_FILE:test-sockpool>)
 
 add_executable(test-curl test_curl.cpp)
-target_link_libraries(test-curl PRIVATE photon_static)
+target_link_libraries(test-curl PRIVATE photon_shared)
 add_test(NAME test-curl COMMAND $<TARGET_FILE:test-curl>)
 
 add_executable(test-server test-server.cpp)
diff --git a/photon.cpp b/photon.cpp
index 8780969b..3d404eac 100644
--- a/photon.cpp
+++ b/photon.cpp
@@ -15,6 +15,7 @@ limitations under the License.
 */
 
 #include "photon.h"
+#include <inttypes.h>
 
 #include "io/fd-events.h"
 #include "io/signal.h"
@@ -38,21 +39,32 @@ static thread_local uint64_t g_event_engine = 0, g_io_engine = 0;
 #define INIT_IO(name, prefix)    if (INIT_IO_##name & io_engine) { if (prefix##_init() < 0) return -1; }
 #define FINI_IO(name, prefix)    if (INIT_IO_##name & g_io_engine) { prefix##_fini(); }
 
+class Shift {
+    uint8_t _n;
+public:
+    constexpr Shift(uint64_t x) : _n(__builtin_ctz(x)) { }
+    operator uint64_t() { return 1UL << _n; }
+};
+
 // Try to init master engine with the recommended order
+static const Shift recommended_order[] = {
 #if defined(__linux__)
-static const int recommended_order[] = {INIT_EVENT_EPOLL, INIT_EVENT_IOURING, INIT_EVENT_SELECT};
+    INIT_EVENT_EPOLL, INIT_EVENT_IOURING, INIT_EVENT_SELECT};
 #else   // macOS, FreeBSD ...
-static const int recommended_order[] = {INIT_EVENT_KQUEUE, INIT_EVENT_SELECT};
+    INIT_EVENT_KQUEUE, INIT_EVENT_SELECT};
 #endif
 
-int init(uint64_t event_engine, uint64_t io_engine) {
+int __photon_init(uint64_t event_engine, uint64_t io_engine) {
     if (vcpu_init() < 0)
         return -1;
 
-    if (event_engine != INIT_EVENT_NONE) {
+    const uint64_t ALL_ENGINES = INIT_EVENT_EPOLL  |
+            INIT_EVENT_IOURING | INIT_EVENT_KQUEUE |
+            INIT_EVENT_SELECT  | INIT_EVENT_IOCP;
+    if (event_engine & ALL_ENGINES) {
         bool ok = false;
-        for (auto each : recommended_order) {
-            if ((each & event_engine) && fd_events_init(each) == 0) {
+        for (auto x : recommended_order) {
+            if ((x & event_engine) && fd_events_init(x) == 0) {
                 ok = true;
                 break;
             }
@@ -84,6 +96,10 @@ int init(uint64_t event_engine, uint64_t io_engine) {
     return 0;
 }
 
+int init(uint64_t event_engine, uint64_t io_engine) {
+    return __photon_init(event_engine, io_engine);
+}
+
 int fini() {
 #ifdef __linux__
     FINI_IO(LIBAIO, libaio_wrapper)
diff --git a/rpc/test/test.cpp b/rpc/test/test.cpp
index 29f85bda..854f6495 100644
--- a/rpc/test/test.cpp
+++ b/rpc/test/test.cpp
@@ -33,7 +33,7 @@ using namespace rpc;
 class RpcTest : public testing::Test {
 public:
     void SetUp() override {
-        GTEST_ASSERT_EQ(0, photon::init(ci_ev_engine, photon::INIT_IO_NONE));
+        GTEST_ASSERT_EQ(0, photon::init(photon::INIT_EVENT_DEFAULT, photon::INIT_IO_NONE));
     }
     void TearDown() override {
         photon::fini();
@@ -481,7 +481,6 @@ TEST_F(RpcTest, passive_shutdown) {
 
 int main(int argc, char** arg)
 {
-    ci_parse_env();
     ::testing::InitGoogleTest(&argc, arg);
     return RUN_ALL_TESTS();
 }
diff --git a/test/ci-tools.cpp b/test/ci-tools.cpp
index 265b4ac3..7243fd25 100644
--- a/test/ci-tools.cpp
+++ b/test/ci-tools.cpp
@@ -1,26 +1,100 @@
-#include "ci-tools.h"
+// #include "ci-tools.h"
 #include <cstdlib>
 #include <cstring>
-#include "../photon.h"
+#include <cstdio>
+#include <assert.h>
+#include <photon/common/estring.h>
+#include <photon/common/alog.h>
+#include <photon/common/alog-stdstring.h>
+#include <photon/photon.h>
 
-#if __linux__
-uint64_t ci_ev_engine = photon::INIT_EVENT_EPOLL;
-#else
-uint64_t ci_ev_engine = photon::INIT_EVENT_KQUEUE;
-#endif
 
-uint64_t ci_ev_engine_with_signal = ci_ev_engine | photon::INIT_EVENT_SIGNAL;
+namespace photon {
 
-void ci_parse_env() {
-    const char* ev_engine = getenv("PHOTON_CI_EV_ENGINE");
-    if (!ev_engine)
+static uint64_t _engine = 0;
+static const char* _engine_name = nullptr;
+
+__attribute__((constructor))
+static void parse_env_eng() {
+    _engine_name = getenv("PHOTON_CI_EV_ENGINE");
+    if (!_engine_name)
         return;
-    if (strcmp(ev_engine, "epoll") == 0) {
-        ci_ev_engine = photon::INIT_EVENT_EPOLL;
-    } else if (strcmp(ev_engine, "io_uring") == 0) {
-        ci_ev_engine = photon::INIT_EVENT_IOURING;
-    } else if (strcmp(ev_engine, "kqueue") == 0) {
-        ci_ev_engine = photon::INIT_EVENT_KQUEUE;
+    if (strcmp(_engine_name, "epoll") == 0) {
+        _engine = photon::INIT_EVENT_EPOLL;
+    } else if (strcmp(_engine_name, "io_uring") == 0) {
+        _engine = photon::INIT_EVENT_IOURING;
+    } else if (strcmp(_engine_name, "kqueue") == 0) {
+        _engine = photon::INIT_EVENT_KQUEUE;
+    } else {
+        printf("invalid event engine: %s\n", _engine_name);
+        _engine_name = nullptr;
+        exit(-1);
+    }
+}
+
+static estring_view get_engine_name(uint64_t eng) {
+    switch(eng) {
+        case INIT_EVENT_EPOLL:      return "epoll";
+        case INIT_EVENT_IOURING:    return "io_uring";
+        case INIT_EVENT_KQUEUE:     return "kqueue";
+        case INIT_EVENT_SELECT:     return "select";
+        case INIT_EVENT_IOCP:       return "iocp";
+    }
+    return {};
+}
+
+static estring get_engine_names(uint64_t engs) {
+    estring names;
+    if (!engs) {
+        names = "none";
+    } else {
+        for (uint64_t i = 0; i < 64; ++i) {
+            auto name = get_engine_name(engs & (1UL << i));
+            if (name.size()) names.appends(name, ", ");
+        }
+        if (names.size() > 2)
+            names.resize(names.size() - 2);
+    }
+    return names;
+}
+
+int __photon_init(uint64_t event_engine, uint64_t io_engine);
+
+// this function is supposed to be linked statically to test
+// cases, so as to override the real one in libphoton.so/dylib
+int init(uint64_t event_engine, uint64_t io_engine) {
+    LOG_INFO("enter CI overriden photon::init()");
+    const uint64_t all_engines = INIT_EVENT_EPOLL  |
+            INIT_EVENT_IOURING | INIT_EVENT_KQUEUE |
+            INIT_EVENT_SELECT  | INIT_EVENT_IOCP;
+    auto arg_specified = event_engine & all_engines;
+    LOG_INFO("argument specified: ` (`)", get_engine_names(arg_specified), arg_specified);
+    LOG_INFO("environment specified: '`'", _engine_name);
+    if (_engine && arg_specified) {
+        event_engine &= ~all_engines;
+        event_engine |= _engine;
+        LOG_INFO("event engine overridden to: ", get_engine_names(event_engine & all_engines));
     }
-    ci_ev_engine_with_signal = ci_ev_engine | photon::INIT_EVENT_SIGNAL;
-}
\ No newline at end of file
+    return __photon_init(event_engine, io_engine);
+}
+
+bool is_using_default_engine() {
+#ifdef __linux__
+    const uint64_t default_eng = INIT_EVENT_EPOLL;
+#else
+    const uint64_t default_eng = INIT_EVENT_KQUEUE;
+#endif
+    return !_engine || _engine == default_eng;
+}
+
+void set_cpu_affinity(int i) {
+#ifdef __linux__
+    cpu_set_t cpuset;
+    CPU_ZERO(&cpuset);
+    CPU_SET(i, &cpuset);
+    pthread_setaffinity_np(pthread_self(),
+                sizeof(cpu_set_t), &cpuset);
+#endif
+}
+
+}
diff --git a/test/ci-tools.h b/test/ci-tools.h
index f0cbedd8..f3a0af30 100644
--- a/test/ci-tools.h
+++ b/test/ci-tools.h
@@ -1,7 +1,9 @@
 #pragma once
-#include <cstdint>
 
-extern uint64_t ci_ev_engine;
-extern uint64_t ci_ev_engine_with_signal;
+namespace photon {
 
-void ci_parse_env();
\ No newline at end of file
+bool is_using_default_engine();
+
+void set_cpu_affinity(int i);
+
+}
diff --git a/thread/test/CMakeLists.txt b/thread/test/CMakeLists.txt
index 44634a93..0dff72ff 100644
--- a/thread/test/CMakeLists.txt
+++ b/thread/test/CMakeLists.txt
@@ -5,9 +5,13 @@ target_link_libraries(perf_usleepdefer_semaphore PRIVATE photon_shared)
 add_test(NAME perf_usleepdefer_semaphore COMMAND $<TARGET_FILE:perf_usleepdefer_semaphore>)
 
 add_executable(test-thread test.cpp x.cpp)
-target_link_libraries(test-thread PRIVATE photon_static)
+target_link_libraries(test-thread PRIVATE photon_shared)
 add_test(NAME test-thread COMMAND $<TARGET_FILE:test-thread>)
 
+add_executable(test-pool test-pool.cpp x.cpp)
+target_link_libraries(test-pool PRIVATE photon_shared)
+add_test(NAME test-pool COMMAND $<TARGET_FILE:test-pool>)
+
 add_executable(test-std-compat test-std-compat.cpp)
 target_link_libraries(test-std-compat PRIVATE photon_shared)
 add_test(NAME test-std-compat COMMAND $<TARGET_FILE:test-std-compat>)
@@ -33,5 +37,5 @@ target_link_libraries(test-lib-data PRIVATE photon_shared)
 add_test(NAME test-lib-data COMMAND $<TARGET_FILE:test-lib-data>)
 
 add_executable(test-multi-vcpu-locking test-multi-vcpu-locking.cpp)
-target_link_libraries(test-multi-vcpu-locking PRIVATE photon_static)
+target_link_libraries(test-multi-vcpu-locking PRIVATE photon_shared)
 add_test(NAME test-multi-vcpu-locking COMMAND $<TARGET_FILE:test-multi-vcpu-locking>)
\ No newline at end of file
diff --git a/thread/test/test-multi-vcpu-locking.cpp b/thread/test/test-multi-vcpu-locking.cpp
index aefafe0c..bfd50ddf 100644
--- a/thread/test/test-multi-vcpu-locking.cpp
+++ b/thread/test/test-multi-vcpu-locking.cpp
@@ -60,7 +60,7 @@ static void myThread(int tid) {
 }
 
 TEST(multi_vcpu_locking, long_time_acquisition_should_abort) {
-    int ret = photon::init(ci_ev_engine, photon::INIT_IO_NONE);
+    int ret = photon::init(photon::INIT_EVENT_DEFAULT, photon::INIT_IO_NONE);
     GTEST_ASSERT_EQ(0, ret);
     DEFER(photon::fini());
 
@@ -90,6 +90,5 @@ TEST(multi_vcpu_locking, long_time_acquisition_should_abort) {
 
 int main(int argc, char** argv) {
     ::testing::InitGoogleTest(&argc, argv);
-    ci_parse_env();
     return RUN_ALL_TESTS();
 }
\ No newline at end of file
diff --git a/thread/test/test-pool.cpp b/thread/test/test-pool.cpp
new file mode 100644
index 00000000..2472592f
--- /dev/null
+++ b/thread/test/test-pool.cpp
@@ -0,0 +1,297 @@
+#include <inttypes.h>
+#include <errno.h>
+#include <math.h>
+#include <string>
+#include <stdlib.h>
+#include <queue>
+#include <algorithm>
+#include <sys/time.h>
+#include <gflags/gflags.h>
+#include <gtest/gtest.h>
+#include <photon/common/alog.h>
+#include <photon/thread/thread-pool.h>
+#include <photon/thread/thread11.h>
+#include <photon/thread/workerpool.h>
+#include "../../test/ci-tools.h"
+
+DEFINE_int32(ths_total, 100, "total threads when testing threadpool.");
+
+using namespace std;
+using namespace photon;
+
+void *func1(void *)
+{
+    thread_sleep(rand()%5);
+    return nullptr;
+}
+
+TEST(ThreadPool, test)
+{
+    ThreadPool<64> pool(64*1024);
+    vector<TPControl*> ths;
+    ths.resize(FLAGS_ths_total);
+    for (int i = 0; i<FLAGS_ths_total; i++)
+        ths[i] = pool.thread_create_ex(&::func1, nullptr, true);
+    // LOG_INFO("----------");
+    for (int i = 0; i<FLAGS_ths_total; i++) {
+        LOG_DEBUG("wait thread: `", ths[i]->th);
+        pool.join(ths[i]);
+    }
+    // LOG_INFO("???????????????");
+}
+
+TEST(ThreadPool, migrate) {
+    WorkPool wp(4, 0, 0, -1);
+    ThreadPool<64> pool(64 * 1024);
+    vector<TPControl*> ths;
+    ths.resize(FLAGS_ths_total);
+    for (int i = 0; i < FLAGS_ths_total; i++) {
+        ths[i] = pool.thread_create_ex(&::func1, nullptr, true);
+        wp.thread_migrate(ths[i]->th);
+    }
+    LOG_INFO("----------");
+    for (int i = 0; i < FLAGS_ths_total; i++) {
+        LOG_DEBUG("wait thread: `", ths[i]->th);
+        pool.join(ths[i]);
+    }
+    LOG_INFO("???????????????");
+}
+
+TEST(ThreadPool, multithread) {
+    WorkPool wp(4, 0, 0, -1);
+    ThreadPool<64> pool(64 * 1024);
+    vector<TPControl*> ths;
+    ths.resize(FLAGS_ths_total);
+    for (int i = 0; i < FLAGS_ths_total; i++) {
+        wp.call(
+            [&] { ths[i] = pool.thread_create_ex(&::func1, nullptr, true); });
+    }
+    for (int i = 0; i < FLAGS_ths_total; i++) {
+        wp.call([&] {
+            pool.join(ths[i]);
+        });
+    }
+}
+
+int jobwork(WorkPool* pool, int i) {
+    LOG_INFO("LAUNCH");
+    int ret = 0;
+    pool->call(
+        [&ret](int i) {
+            LOG_INFO("START");
+            this_thread::sleep_for(std::chrono::seconds(1));
+            LOG_INFO("FINISH");
+            ret = i;
+        },
+        i);
+    LOG_INFO("RETURN");
+    EXPECT_EQ(i, ret);
+    return 0;
+}
+
+TEST(workpool, work) {
+    std::unique_ptr<WorkPool> pool(new WorkPool(2));
+
+    std::vector<join_handle*> jhs;
+    auto start = std::chrono::system_clock::now();
+    for (int i = 0; i < 4; i++) {
+        jhs.emplace_back(thread_enable_join(
+            thread_create11(&jobwork, pool.get(), i)));
+    }
+    for (auto& j : jhs) {
+        thread_join(j);
+    }
+    auto duration = std::chrono::system_clock::now() - start;
+    EXPECT_GE(duration, std::chrono::seconds(2));
+    EXPECT_LE(duration, std::chrono::seconds(3));
+}
+
+TEST(workpool, async_work_capture) {
+    std::unique_ptr<WorkPool> pool(new WorkPool(2, 0, 0, 0));
+
+    semaphore sem;
+    int flag[10] = {0};
+    auto start = std::chrono::system_clock::now();
+    for (int i = 0; i < 10; i++) {
+        pool->async_call(new auto([&sem, i, &flag]{
+            EXPECT_FALSE(flag[i]);
+            flag[i] = true;
+            auto x = i;
+            LOG_INFO(x);
+            thread_usleep(2000 * 1000);
+            EXPECT_EQ(x, i);
+            EXPECT_TRUE(flag[i]);
+            sem.signal(1);
+        }));
+    }
+    auto duration = std::chrono::system_clock::now() - start;
+    EXPECT_GE(duration, std::chrono::seconds(0));
+    EXPECT_LE(duration, std::chrono::seconds(1));
+    sem.wait(10);
+    LOG_INFO("DONE");
+}
+
+struct CopyMoveRecord{
+    size_t copy = 0;
+    size_t move = 0;
+    CopyMoveRecord() {
+    }
+    ~CopyMoveRecord() {
+    }
+    CopyMoveRecord(const CopyMoveRecord& rhs) {
+        copy = rhs.copy + 1;
+        move = rhs.move;
+        LOG_INFO("COPY ", this);
+    }
+    CopyMoveRecord(CopyMoveRecord&& rhs) {
+        copy = rhs.copy;
+        move = rhs.move + 1;
+        LOG_INFO("MOVE ", this);
+    }
+    CopyMoveRecord& operator=(const CopyMoveRecord& rhs) {
+        copy = rhs.copy + 1;
+        move = rhs.move;
+        LOG_INFO("COPY ASSIGN ", this);
+        return *this;
+    }
+    CopyMoveRecord& operator=(CopyMoveRecord&& rhs) {
+        copy = rhs.copy;
+        move = rhs.move + 1;
+        LOG_INFO("MOVE ASSIGN", this);
+        return *this;
+    }
+};
+
+TEST(workpool, async_work_lambda) {
+    std::unique_ptr<WorkPool> pool(new WorkPool(2));
+
+    std::vector<join_handle*> jhs;
+    auto start = std::chrono::system_clock::now();
+    for (int i = 0; i < 4; i++) {
+        CopyMoveRecord *r = new CopyMoveRecord();
+        pool->async_call(
+            new auto ([i, r]() {
+                LOG_INFO("START ", VALUE(__cplusplus), VALUE(r->copy),
+                         VALUE(r->move));
+                EXPECT_EQ(0, r->copy);
+                this_thread::sleep_for(std::chrono::seconds(1));
+                LOG_INFO("FINISH");
+                delete r;
+            }));
+    }
+    auto duration = std::chrono::system_clock::now() - start;
+    EXPECT_GE(duration, std::chrono::seconds(0));
+    EXPECT_LE(duration, std::chrono::seconds(1));
+    thread_sleep(3);
+    LOG_INFO("DONE");
+}
+
+
+TEST(workpool, async_work_lambda_threadcreate) {
+    std::unique_ptr<WorkPool> pool(new WorkPool(1, 0, 0, 0));
+
+    std::vector<join_handle*> jhs;
+    auto start = std::chrono::system_clock::now();
+    semaphore sem;
+    for (int i = 0; i < 4; i++) {
+        CopyMoveRecord *r = new CopyMoveRecord();
+        pool->async_call(
+            new auto ([&sem, i, r]() {
+                LOG_INFO("START ", VALUE(__cplusplus), VALUE(r->copy),
+                         VALUE(r->move));
+                EXPECT_EQ(0, r->copy);
+                thread_sleep(1);
+                sem.signal(1);
+                LOG_INFO("FINISH");
+                delete r;
+            }));
+    }
+    auto duration = std::chrono::system_clock::now() - start;
+    EXPECT_GE(duration, std::chrono::seconds(0));
+    EXPECT_LE(duration, std::chrono::seconds(1));
+    sem.wait(4);
+    duration = std::chrono::system_clock::now() - start;
+    EXPECT_GE(duration, std::chrono::seconds(1));
+    EXPECT_LE(duration, std::chrono::seconds(3));
+    LOG_INFO("DONE");
+}
+
+TEST(workpool, async_work_lambda_threadpool) {
+    std::unique_ptr<WorkPool> pool(new WorkPool(1, 0, 0, 4));
+
+    std::vector<join_handle*> jhs;
+    auto start = std::chrono::system_clock::now();
+    semaphore sem;
+    for (int i = 0; i < 4; i++) {
+        CopyMoveRecord *r = new CopyMoveRecord();
+        pool->async_call(
+            new auto ([&sem, i, r]() {
+                LOG_INFO("START ", VALUE(__cplusplus), VALUE(r->copy),
+                         VALUE(r->move));
+                EXPECT_EQ(0, r->copy);
+                thread_sleep(1);
+                sem.signal(1);
+                LOG_INFO("FINISH");
+                delete r;
+            }));
+    }
+    auto duration = std::chrono::system_clock::now() - start;
+    EXPECT_GE(duration, std::chrono::seconds(0));
+    EXPECT_LE(duration, std::chrono::seconds(1));
+    sem.wait(4);
+    duration = std::chrono::system_clock::now() - start;
+    LOG_INFO(VALUE(duration.count()));
+    EXPECT_GE(duration, std::chrono::seconds(1));
+    EXPECT_LE(duration, std::chrono::seconds(2));
+    LOG_INFO("DONE");
+}
+
+TEST(workpool, async_work_lambda_threadpool_append) {
+    std::unique_ptr<WorkPool> pool(new WorkPool(0, 0, 0, 0));
+
+    for (int i=0;i<4;i++) {
+        std::thread([&]{
+            vcpu_init();
+            DEFER(vcpu_fini());
+            pool->join_current_vcpu_into_workpool();
+        }).detach();
+    }
+
+    std::vector<join_handle*> jhs;
+    auto start = std::chrono::system_clock::now();
+    semaphore sem;
+    for (int i = 0; i < 4; i++) {
+        CopyMoveRecord *r = new CopyMoveRecord();
+        pool->async_call(
+            new auto ([&sem, i, r]() {
+                LOG_INFO("START ", VALUE(__cplusplus), VALUE(r->copy),
+                         VALUE(r->move));
+                EXPECT_EQ(0, r->copy);
+                thread_sleep(1);
+                sem.signal(1);
+                LOG_INFO("FINISH");
+                delete r;
+            }));
+    }
+    auto duration = std::chrono::system_clock::now() - start;
+    EXPECT_GE(duration, std::chrono::seconds(0));
+    EXPECT_LE(duration, std::chrono::seconds(1));
+    sem.wait(4);
+    duration = std::chrono::system_clock::now() - start;
+    EXPECT_GE(duration, std::chrono::seconds(1));
+    EXPECT_LE(duration, std::chrono::seconds(2));
+    LOG_INFO("DONE");
+}
+
+int main(int argc, char** arg)
+{
+    if (!is_using_default_engine()) return 0;
+    ::testing::InitGoogleTest(&argc, arg);
+    gflags::ParseCommandLineFlags(&argc, &arg, true);
+    default_audit_logger.log_output = log_output_stdout;
+    vcpu_init();
+    DEFER(vcpu_fini());
+    set_log_output_level(ALOG_DEBUG);
+    return RUN_ALL_TESTS();
+}
+
diff --git a/thread/test/test.cpp b/thread/test/test.cpp
index 4fd2a171..5485b82f 100644
--- a/thread/test/test.cpp
+++ b/thread/test/test.cpp
@@ -24,19 +24,17 @@ limitations under the License.
 #include <sys/time.h>
 #include <gflags/gflags.h>
 #include <gtest/gtest.h>
+#include <photon/common/alog-audit.h>
 
 #define private public
-
 #include "../thread.cpp"
 #include "../thread11.h"
-#include "../thread-pool.h"
-#include "../workerpool.h"
-#include <photon/common/alog-audit.h>
+#include "../../test/ci-tools.h"
+
 
 using namespace std;
 using namespace photon;
 
-DEFINE_int32(ths_total, 100, "total threads when testing threadpool.");
 DEFINE_int32(vcpus, 1, "total # of vCPUs");
 
 thread_local semaphore aSem(4);
@@ -709,60 +707,6 @@ TEST(Sleep, sleep_only_thread) {    //Sleep_sleep_only_thread_Test::TestBody
     EXPECT_LE(dt, 3UL*1024*1024);
 }
 
-void *func1(void *)
-{
-    photon::thread_sleep(rand()%5);
-    return nullptr;
-}
-
-TEST(ThreadPool, test)
-{
-    ThreadPool<64> pool(64*1024);
-    vector<TPControl*> ths;
-    ths.resize(FLAGS_ths_total);
-    for (int i = 0; i<FLAGS_ths_total; i++)
-        ths[i] = pool.thread_create_ex(&::func1, nullptr, true);
-    // LOG_INFO("----------");
-    for (int i = 0; i<FLAGS_ths_total; i++) {
-        LOG_DEBUG("wait thread: `", ths[i]->th);
-        pool.join(ths[i]);
-    }
-    // LOG_INFO("???????????????");
-}
-
-TEST(ThreadPool, migrate) {
-    WorkPool wp(4, 0, 0, -1);
-    ThreadPool<64> pool(64 * 1024);
-    vector<TPControl*> ths;
-    ths.resize(FLAGS_ths_total);
-    for (int i = 0; i < FLAGS_ths_total; i++) {
-        ths[i] = pool.thread_create_ex(&::func1, nullptr, true);
-        wp.thread_migrate(ths[i]->th);
-    }
-    LOG_INFO("----------");
-    for (int i = 0; i < FLAGS_ths_total; i++) {
-        LOG_DEBUG("wait thread: `", ths[i]->th);
-        pool.join(ths[i]);
-    }
-    LOG_INFO("???????????????");
-}
-
-TEST(ThreadPool, multithread) {
-    WorkPool wp(4, 0, 0, -1);
-    ThreadPool<64> pool(64 * 1024);
-    vector<TPControl*> ths;
-    ths.resize(FLAGS_ths_total);
-    for (int i = 0; i < FLAGS_ths_total; i++) {
-        wp.call(
-            [&] { ths[i] = pool.thread_create_ex(&::func1, nullptr, true); });
-    }
-    for (int i = 0; i < FLAGS_ths_total; i++) {
-        wp.call([&] {
-            pool.join(ths[i]);
-        });
-    }
-}
-
 thread_local uint64_t rw_count;
 thread_local bool writing = false;
 thread_local photon::rwlock rwl;
@@ -862,7 +806,7 @@ void run_all_tests(uint32_t i)
     RUN_TEST(thread11, test);
     RUN_TEST(Semaphore, basic);
     RUN_TEST(Semaphore, heavy);
-    RUN_TEST(ThreadPool, test);
+    // RUN_TEST(ThreadPool, test);
     RUN_TEST(RWLock, checklock);
     RUN_TEST(RWLock, interrupt);
     wait_for_completion(i);
@@ -1250,216 +1194,6 @@ TEST(mutex, timeout_is_zero) {
     LOG_INFO("Meet ` lock timeout, all work finished", cnt.load());
 }
 
-int jobwork(WorkPool* pool, int i) {
-    LOG_INFO("LAUNCH");
-    int ret = 0;
-    pool->call(
-        [&ret](int i) {
-            LOG_INFO("START");
-            this_thread::sleep_for(std::chrono::seconds(1));
-            LOG_INFO("FINISH");
-            ret = i;
-        },
-        i);
-    LOG_INFO("RETURN");
-    EXPECT_EQ(i, ret);
-    return 0;
-}
-
-TEST(workpool, work) {
-    std::unique_ptr<WorkPool> pool(new WorkPool(2));
-
-    std::vector<photon::join_handle*> jhs;
-    auto start = std::chrono::system_clock::now();
-    for (int i = 0; i < 4; i++) {
-        jhs.emplace_back(photon::thread_enable_join(
-            photon::thread_create11(&jobwork, pool.get(), i)));
-    }
-    for (auto& j : jhs) {
-        photon::thread_join(j);
-    }
-    auto duration = std::chrono::system_clock::now() - start;
-    EXPECT_GE(duration, std::chrono::seconds(2));
-    EXPECT_LE(duration, std::chrono::seconds(3));
-}
-
-TEST(workpool, async_work_capture) {
-    std::unique_ptr<WorkPool> pool(new WorkPool(2, 0, 0, 0));
-
-    photon::semaphore sem;
-    int flag[10] = {0};
-    auto start = std::chrono::system_clock::now();
-    for (int i = 0; i < 10; i++) {
-        pool->async_call(new auto([&sem, i, &flag]{
-            EXPECT_FALSE(flag[i]);
-            flag[i] = true;
-            auto x = i;
-            LOG_INFO(x);
-            photon::thread_usleep(2000 * 1000);
-            EXPECT_EQ(x, i);
-            EXPECT_TRUE(flag[i]);
-            sem.signal(1);
-        }));
-    }
-    auto duration = std::chrono::system_clock::now() - start;
-    EXPECT_GE(duration, std::chrono::seconds(0));
-    EXPECT_LE(duration, std::chrono::seconds(1));
-    sem.wait(10);
-    LOG_INFO("DONE");
-}
-
-struct CopyMoveRecord{
-    size_t copy = 0;
-    size_t move = 0;
-    CopyMoveRecord() {
-    }
-    ~CopyMoveRecord() {
-    }
-    CopyMoveRecord(const CopyMoveRecord& rhs) {
-        copy = rhs.copy + 1;
-        move = rhs.move;
-        LOG_INFO("COPY ", this);
-    }
-    CopyMoveRecord(CopyMoveRecord&& rhs) {
-        copy = rhs.copy;
-        move = rhs.move + 1;
-        LOG_INFO("MOVE ", this);
-    }
-    CopyMoveRecord& operator=(const CopyMoveRecord& rhs) {
-        copy = rhs.copy + 1;
-        move = rhs.move;
-        LOG_INFO("COPY ASSIGN ", this);
-        return *this;
-    }
-    CopyMoveRecord& operator=(CopyMoveRecord&& rhs) {
-        copy = rhs.copy;
-        move = rhs.move + 1;
-        LOG_INFO("MOVE ASSIGN", this);
-        return *this;
-    }
-};
-
-TEST(workpool, async_work_lambda) {
-    std::unique_ptr<WorkPool> pool(new WorkPool(2));
-
-    std::vector<photon::join_handle*> jhs;
-    auto start = std::chrono::system_clock::now();
-    for (int i = 0; i < 4; i++) {
-        CopyMoveRecord *r = new CopyMoveRecord();
-        pool->async_call(
-            new auto ([i, r]() {
-                LOG_INFO("START ", VALUE(__cplusplus), VALUE(r->copy),
-                         VALUE(r->move));
-                EXPECT_EQ(0, r->copy);
-                this_thread::sleep_for(std::chrono::seconds(1));
-                LOG_INFO("FINISH");
-                delete r;
-            }));
-    }
-    auto duration = std::chrono::system_clock::now() - start;
-    EXPECT_GE(duration, std::chrono::seconds(0));
-    EXPECT_LE(duration, std::chrono::seconds(1));
-    photon::thread_sleep(3);
-    LOG_INFO("DONE");
-}
-
-
-TEST(workpool, async_work_lambda_threadcreate) {
-    std::unique_ptr<WorkPool> pool(new WorkPool(1, 0, 0, 0));
-
-    std::vector<photon::join_handle*> jhs;
-    auto start = std::chrono::system_clock::now();
-    photon::semaphore sem;
-    for (int i = 0; i < 4; i++) {
-        CopyMoveRecord *r = new CopyMoveRecord();
-        pool->async_call(
-            new auto ([&sem, i, r]() {
-                LOG_INFO("START ", VALUE(__cplusplus), VALUE(r->copy),
-                         VALUE(r->move));
-                EXPECT_EQ(0, r->copy);
-                photon::thread_sleep(1);
-                sem.signal(1);
-                LOG_INFO("FINISH");
-                delete r;
-            }));
-    }
-    auto duration = std::chrono::system_clock::now() - start;
-    EXPECT_GE(duration, std::chrono::seconds(0));
-    EXPECT_LE(duration, std::chrono::seconds(1));
-    sem.wait(4);
-    duration = std::chrono::system_clock::now() - start;
-    EXPECT_GE(duration, std::chrono::seconds(1));
-    EXPECT_LE(duration, std::chrono::seconds(3));
-    LOG_INFO("DONE");
-}
-
-TEST(workpool, async_work_lambda_threadpool) {
-    std::unique_ptr<WorkPool> pool(new WorkPool(1, 0, 0, 4));
-
-    std::vector<photon::join_handle*> jhs;
-    auto start = std::chrono::system_clock::now();
-    photon::semaphore sem;
-    for (int i = 0; i < 4; i++) {
-        CopyMoveRecord *r = new CopyMoveRecord();
-        pool->async_call(
-            new auto ([&sem, i, r]() {
-                LOG_INFO("START ", VALUE(__cplusplus), VALUE(r->copy),
-                         VALUE(r->move));
-                EXPECT_EQ(0, r->copy);
-                photon::thread_sleep(1);
-                sem.signal(1);
-                LOG_INFO("FINISH");
-                delete r;
-            }));
-    }
-    auto duration = std::chrono::system_clock::now() - start;
-    EXPECT_GE(duration, std::chrono::seconds(0));
-    EXPECT_LE(duration, std::chrono::seconds(1));
-    sem.wait(4);
-    duration = std::chrono::system_clock::now() - start;
-    LOG_INFO(VALUE(duration.count()));
-    EXPECT_GE(duration, std::chrono::seconds(1));
-    EXPECT_LE(duration, std::chrono::seconds(2));
-    LOG_INFO("DONE");
-}
-
-TEST(workpool, async_work_lambda_threadpool_append) {
-    std::unique_ptr<WorkPool> pool(new WorkPool(0, 0, 0, 0));
-
-    for (int i=0;i<4;i++) {
-        std::thread([&]{
-            photon::vcpu_init();
-            DEFER(photon::vcpu_fini());
-            pool->join_current_vcpu_into_workpool();
-        }).detach();
-    }
-
-    std::vector<photon::join_handle*> jhs;
-    auto start = std::chrono::system_clock::now();
-    photon::semaphore sem;
-    for (int i = 0; i < 4; i++) {
-        CopyMoveRecord *r = new CopyMoveRecord();
-        pool->async_call(
-            new auto ([&sem, i, r]() {
-                LOG_INFO("START ", VALUE(__cplusplus), VALUE(r->copy),
-                         VALUE(r->move));
-                EXPECT_EQ(0, r->copy);
-                photon::thread_sleep(1);
-                sem.signal(1);
-                LOG_INFO("FINISH");
-                delete r;
-            }));
-    }
-    auto duration = std::chrono::system_clock::now() - start;
-    EXPECT_GE(duration, std::chrono::seconds(0));
-    EXPECT_LE(duration, std::chrono::seconds(1));
-    sem.wait(4);
-    duration = std::chrono::system_clock::now() - start;
-    EXPECT_GE(duration, std::chrono::seconds(1));
-    EXPECT_LE(duration, std::chrono::seconds(2));
-    LOG_INFO("DONE");
-}
-
 #if defined(_WIN64)
 #define SAVE_REG(R) register uint64_t R asm(#R); volatile uint64_t saved_##R = R;
 #define CHECK_REG(R) asm volatile ("" : "=m"(saved_##R)); if (saved_##R != R) puts(#R " differs after context switch!");
@@ -1892,6 +1626,7 @@ TEST(intrusive_list, split) {
 
 int main(int argc, char** arg)
 {
+    if (!photon::is_using_default_engine()) return 0;
     ::testing::InitGoogleTest(&argc, arg);
     gflags::ParseCommandLineFlags(&argc, &arg, true);
     default_audit_logger.log_output = log_output_stdout;
diff --git a/thread/thread.cpp b/thread/thread.cpp
index e6e2379d..de1e6b1a 100644
--- a/thread/thread.cpp
+++ b/thread/thread.cpp
@@ -268,8 +268,9 @@ namespace photon
 
         void init_main_thread_stack() {
 #ifdef __APPLE__
-            stack_size = pthread_get_stacksize_np(pthread_self());
-            stackful_alloc_top = (char*) pthread_get_stackaddr_np(pthread_self());
+            auto self = pthread_self();
+            stack_size = pthread_get_stacksize_np(self);
+            stackful_alloc_top = (char*) pthread_get_stackaddr_np(self);
 #elif defined(_WIN64)
             ULONG_PTR stack_low, stack_high;
             GetCurrentThreadStackLimits(&stack_low, &stack_high);

From fa7a4383173b85811d4d2a3e0cd368f01998c77f Mon Sep 17 00:00:00 2001
From: Coldwings <coldwings@me.com>
Date: Tue, 11 Jun 2024 17:45:31 +0800
Subject: [PATCH 76/76] CI use images that pre-installed all deps (#505)

---
 .github/workflows/ci.linux.arm.yml | 22 ++--------------------
 1 file changed, 2 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/ci.linux.arm.yml b/.github/workflows/ci.linux.arm.yml
index fbe35f61..6a806f49 100644
--- a/.github/workflows/ci.linux.arm.yml
+++ b/.github/workflows/ci.linux.arm.yml
@@ -11,7 +11,7 @@ jobs:
     runs-on: [self-hosted, Linux, ARM64]
 
     container:
-      image: dokken/centos-stream-8:sha-40294ce
+      image: ghcr.io/coldwings/photon-ut-base:latest
       options: --cpus 4
 
     steps:
@@ -23,15 +23,6 @@ jobs:
 
       - uses: actions/checkout@v3
 
-      - name: Install Dependencies
-        run: |
-          dnf install -y git gcc-c++ cmake 'dnf-command(config-manager)'
-          dnf install -y gcc-toolset-9-gcc-c++
-          dnf install -y openssl-devel libcurl-devel libaio-devel
-          dnf install -y epel-release
-          dnf config-manager --set-enabled powertools
-          dnf install -y gtest-devel gmock-devel gflags-devel fuse-devel libgsasl-devel e2fsprogs-devel
-
       - name: Build
         run: |
           source /opt/rh/gcc-toolset-9/enable
@@ -48,7 +39,7 @@ jobs:
     runs-on: [self-hosted, Linux, ARM64]
 
     container:
-      image: dokken/centos-stream-8:sha-40294ce
+      image: ghcr.io/coldwings/photon-ut-base:latest
       options: --cpus 4
 
     steps:
@@ -60,15 +51,6 @@ jobs:
 
       - uses: actions/checkout@v3
 
-      - name: Install Dependencies
-        run: |
-          dnf install -y git gcc-c++ cmake 'dnf-command(config-manager)'
-          dnf install -y gcc-toolset-9-gcc-c++
-          dnf install -y openssl-devel libcurl-devel libaio-devel
-          dnf install -y epel-release
-          dnf config-manager --set-enabled powertools
-          dnf install -y gtest-devel gmock-devel gflags-devel fuse-devel libgsasl-devel e2fsprogs-devel
-
       - name: Build
         run: |
           source /opt/rh/gcc-toolset-9/enable