Merge branch 'main' into manual_pr_0.7_main

.github/workflows/ci.linux.arm.yml

@@ -26,8 +26,13 @@ jobs:
       - name: Build
         run: |
           source /opt/rh/gcc-toolset-9/enable
-          cmake -B build -D CMAKE_BUILD_TYPE=MinSizeRel -D PHOTON_BUILD_TESTING=ON \
-            -D PHOTON_ENABLE_SASL=ON -D PHOTON_ENABLE_FUSE=ON -D PHOTON_ENABLE_EXTFS=ON
+          cmake -B build \
+            -D CMAKE_BUILD_TYPE=MinSizeRel \
+            -D PHOTON_ENABLE_ECOSYSTEM=ON \
+            -D PHOTON_BUILD_TESTING=ON \
+            -D PHOTON_ENABLE_SASL=ON \
+            -D PHOTON_ENABLE_FUSE=ON \
+            -D PHOTON_ENABLE_EXTFS=ON
           cmake --build build -j $(nproc) -- VERBOSE=1
 
       - name: Test
@@ -54,8 +59,13 @@ jobs:
       - name: Build
         run: |
           source /opt/rh/gcc-toolset-9/enable
-          cmake -B build -D CMAKE_BUILD_TYPE=Debug -D PHOTON_BUILD_TESTING=ON \
-            -D PHOTON_ENABLE_SASL=ON -D PHOTON_ENABLE_FUSE=ON -D PHOTON_ENABLE_EXTFS=ON
+          cmake -B build \
+            -D CMAKE_BUILD_TYPE=Debug \
+            -D PHOTON_ENABLE_ECOSYSTEM=ON \
+            -D PHOTON_BUILD_TESTING=ON \
+            -D PHOTON_ENABLE_SASL=ON \
+            -D PHOTON_ENABLE_FUSE=ON \
+            -D PHOTON_ENABLE_EXTFS=ON
           cmake --build build -j $(nproc) -- VERBOSE=1
 
       - name: Test

.github/workflows/ci.linux.x86-64.yml

@@ -15,6 +15,7 @@ jobs:
         run: |
           rm -fr build
           cmake -B build -D CMAKE_BUILD_TYPE=MinSizeRel   \
+                         -D PHOTON_ENABLE_ECOSYSTEM=ON    \
                          -D PHOTON_BUILD_TESTING=ON       \
                          -D PHOTON_ENABLE_SASL=ON         \
                          -D PHOTON_ENABLE_FUSE=ON         \
@@ -62,6 +63,8 @@ jobs:
           ctest -E test-lockfree --timeout 3600 -V
           export PHOTON_CI_EV_ENGINE=io_uring
           ctest -E test-lockfree --timeout 3600 -V
+          export PHOTON_CI_EV_ENGINE=epoll_ng
+          ctest -E test-lockfree --timeout 3600 -V
 
   gcc921:
     needs: gcc850
@@ -103,6 +106,8 @@ jobs:
           ctest -E test-lockfree --timeout 3600 -V
           export PHOTON_CI_EV_ENGINE=io_uring
           ctest -E test-lockfree --timeout 3600 -V
+          export PHOTON_CI_EV_ENGINE=epoll_ng
+          ctest -E test-lockfree --timeout 3600 -V
 
   gcc1031:
     needs: gcc921
@@ -144,6 +149,8 @@ jobs:
           ctest -E test-lockfree --timeout 3600 -V
           export PHOTON_CI_EV_ENGINE=io_uring
           ctest -E test-lockfree --timeout 3600 -V
+          export PHOTON_CI_EV_ENGINE=epoll_ng
+          ctest -E test-lockfree --timeout 3600 -V
 
   gcc1121:
     needs: gcc1031
@@ -152,7 +159,7 @@ jobs:
       - name: Build1121
         run: |
           source /opt/rh/gcc-toolset-10/enable
-          cmake --build build -j --clean-first -- VERBOSE=1
+          cmake --build build -j $(nproc) --clean-first -- VERBOSE=1
           ln -f common/checksum/test/checksum.in build/output/
           tar -c --use-compress-program=zstdmt -f output1121.tzs build/output/
       - name: Upload1121
@@ -185,6 +192,8 @@ jobs:
           ctest -E test-lockfree --timeout 3600 -V
           export PHOTON_CI_EV_ENGINE=io_uring
           ctest -E test-lockfree --timeout 3600 -V
+          export PHOTON_CI_EV_ENGINE=epoll_ng
+          ctest -E test-lockfree --timeout 3600 -V
 
   gcc1211:
     needs: gcc1121
@@ -226,5 +235,7 @@ jobs:
           ctest -E test-lockfree --timeout 3600 -V
           export PHOTON_CI_EV_ENGINE=io_uring
           ctest -E test-lockfree --timeout 3600 -V
+          export PHOTON_CI_EV_ENGINE=epoll_ng
+          ctest -E test-lockfree --timeout 3600 -V
 
 

.github/workflows/ci.macos.arm.yml

@@ -26,9 +26,13 @@ jobs:
 
       - name: Build
         run: |
-          cmake -B ${{github.workspace}}/build -D PHOTON_BUILD_TESTING=ON -D CMAKE_BUILD_TYPE=Release \
-            -D PHOTON_ENABLE_SASL=ON -DOPENSSL_ROOT_DIR=/usr/local/opt/openssl@3
-          cmake --build ${{github.workspace}}/build -j $(nproc)
+          cmake -B ${{github.workspace}}/build \
+            -D PHOTON_ENABLE_ECOSYSTEM=ON \
+            -D PHOTON_BUILD_TESTING=ON \
+            -D CMAKE_BUILD_TYPE=MinSizeRel \
+            -D PHOTON_ENABLE_SASL=ON \
+            -D OPENSSL_ROOT_DIR=/opt/homebrew/Cellar/[email protected]/1.1.1w
+          cmake --build ${{github.workspace}}/build -j $(sysctl -n hw.logicalcpu)
 
       - name: Test
         working-directory: ${{github.workspace}}/build

.github/workflows/ci.macos.x86.yml

@@ -26,9 +26,13 @@ jobs:
 
     - name: Build
       run: |
-        cmake -B ${{github.workspace}}/build -D PHOTON_BUILD_TESTING=ON -D CMAKE_BUILD_TYPE=Release \
-          -D PHOTON_ENABLE_SASL=ON -DOPENSSL_ROOT_DIR=/usr/local/opt/openssl@3
-        cmake --build ${{github.workspace}}/build -j $(nproc)
+        cmake -B ${{github.workspace}}/build \
+          -D PHOTON_ENABLE_ECOSYSTEM=ON \
+          -D PHOTON_BUILD_TESTING=ON \
+          -D CMAKE_BUILD_TYPE=MinSizeRel \
+          -D PHOTON_ENABLE_SASL=ON \
+          -D OPENSSL_ROOT_DIR=/usr/local/opt/openssl@3
+        cmake --build ${{github.workspace}}/build -j $(sysctl -n hw.logicalcpu)
 
     - name: Test
       working-directory: ${{github.workspace}}/build

.gitignore

@@ -12,6 +12,8 @@
 /.clang_complete
 /.ccls*
 /.cquery*
+/.cache
+/compile_commands.json
 
 # alimake
 /.dep_create/

CMake/Findopenssl.cmake

@@ -1,7 +1,25 @@
-find_path(OPENSSL_INCLUDE_DIRS openssl/ssl.h openssl/crypto.h)
+if (NOT APPLE)
+    find_path(OPENSSL_INCLUDE_DIRS openssl/ssl.h openssl/crypto.h)
+    find_library(OPENSSL_SSL_LIBRARIES ssl)
+    find_library(OPENSSL_CRYPTO_LIBRARIES crypto)
 
-find_library(OPENSSL_SSL_LIBRARIES ssl)
-find_library(OPENSSL_CRYPTO_LIBRARIES crypto)
+else ()
+    find_path(OPENSSL_INCLUDE_DIRS
+            NAMES openssl/ssl.h openssl/crypto.h
+            PATHS ${OPENSSL_ROOT_DIR}/include
+            NO_DEFAULT_PATH
+    )
+    find_library(OPENSSL_SSL_LIBRARIES
+            NAMES ssl
+            PATHS ${OPENSSL_ROOT_DIR}/lib
+            NO_DEFAULT_PATH
+    )
+    find_library(OPENSSL_CRYPTO_LIBRARIES
+            NAMES crypto
+            PATHS ${OPENSSL_ROOT_DIR}/lib
+            NO_DEFAULT_PATH
+    )
+endif ()
 
 set(OPENSSL_LIBRARIES ${OPENSSL_SSL_LIBRARIES} ${OPENSSL_CRYPTO_LIBRARIES})
 

CMake/build-from-src.cmake

@@ -87,7 +87,7 @@ function(build_from_src [dep])
                 BUILD_IN_SOURCE ON
                 CONFIGURE_COMMAND ./config -fPIC --prefix=${BINARY_DIR} --openssldir=${BINARY_DIR} shared
                 BUILD_COMMAND make -j 1  # https://github.com/openssl/openssl/issues/5762#issuecomment-376622684
-                INSTALL_COMMAND make install
+                INSTALL_COMMAND make -j 1 install
                 LOG_CONFIGURE ON
                 LOG_BUILD ON
                 LOG_INSTALL ON

CMakeLists.txt

@@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.14 FATAL_ERROR)
 
 project(
         photon
-        VERSION 0.6
+        VERSION 0.7
         LANGUAGES C CXX ASM
 )
 
@@ -18,12 +18,14 @@ find_package(PkgConfig REQUIRED)
 # Options
 set(PHOTON_CXX_STANDARD "14" CACHE STRING "C++ standard")
 option(PHOTON_BUILD_TESTING "enable build testing" OFF)
+option(PHOTON_BUILD_WITH_ASAN "build with asan" OFF)
 option(PHOTON_ENABLE_URING "enable io_uring function" OFF)
 option(PHOTON_ENABLE_FUSE "enable fuse function" OFF)
 option(PHOTON_ENABLE_SASL "enable sasl" OFF)
 option(PHOTON_ENABLE_MIMIC_VDSO "enable mimic vdso" OFF)
 option(PHOTON_ENABLE_FSTACK_DPDK "Use f-stack + DPDK as the event engine" OFF)
 option(PHOTON_ENABLE_EXTFS "enable extfs" OFF)
+option(PHOTON_ENABLE_ECOSYSTEM "enable ecosystem" OFF)
 
 option(PHOTON_BUILD_DEPENDENCIES "" OFF)
 set(PHOTON_AIO_SOURCE "https://pagure.io/libaio/archive/libaio-0.3.113/libaio-0.3.113.tar.gz" CACHE STRING "")
@@ -37,16 +39,31 @@ set(PHOTON_FSTACK_SOURCE "" CACHE STRING "")
 set(PHOTON_E2FS_SOURCE "" CACHE STRING "")
 set(PHOTON_GFLAGS_SOURCE "https://github.com/gflags/gflags/archive/refs/tags/v2.2.2.tar.gz" CACHE STRING "")
 set(PHOTON_GOOGLETEST_SOURCE "https://github.com/google/googletest/archive/refs/tags/release-1.12.1.tar.gz" CACHE STRING "")
+set(PHOTON_RAPIDJSON_GIT "https://github.com/Tencent/rapidjson.git" CACHE STRING "")
+set(PHOTON_RAPIDXML_SOURCE "https://sourceforge.net/projects/rapidxml/files/rapidxml/rapidxml%201.13/rapidxml-1.13.zip/download" CACHE STRING "")
+set(PHOTON_RAPIDYAML_SOURCE "https://github.com/biojppm/rapidyaml/releases/download/v0.5.0/rapidyaml-0.5.0.hpp" CACHE STRING "")
+set(PHOTON_CPP_REDIS_SOURCE "https://github.com/cpp-redis/cpp_redis/archive/refs/tags/4.3.1.tar.gz" CACHE STRING "")
 
-# Get CPU arch and number
+# Get CPU arch
 execute_process(COMMAND uname -m OUTPUT_VARIABLE ARCH OUTPUT_STRIP_TRAILING_WHITESPACE)
 if (NOT (${ARCH} STREQUAL x86_64) AND NOT (${ARCH} STREQUAL aarch64) AND NOT (${ARCH} STREQUAL arm64))
     message(FATAL_ERROR "Unknown CPU architecture ${ARCH}")
 endif ()
-ProcessorCount(NumCPU)
 
-# Global compiler options, only effective within this project
-add_compile_options(-Wall -Werror -Wno-error=pragmas)
+# Global compile options, only effective within this project
+set(global_compile_options -Wall -Wno-error=pragmas)
+if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 8.0)
+    # Hint: -faligned-new is enabled by default after -std=c++17
+    list(APPEND global_compile_options -Werror -faligned-new)
+endif ()
+add_compile_options(${global_compile_options})
+
+if (PHOTON_BUILD_WITH_ASAN)
+    if ((NOT CMAKE_BUILD_TYPE STREQUAL "Debug") OR (NOT CMAKE_SYSTEM_NAME STREQUAL "Linux"))
+        message(FATAL_ERROR "Wrong environment")
+    endif ()
+    add_link_options(-fsanitize=address -static-libasan)
+endif ()
 
 set(CMAKE_CXX_STANDARD ${PHOTON_CXX_STANDARD})
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
@@ -59,7 +76,6 @@ set(CMAKE_CXX_FLAGS_MINSIZEREL "-O2")    # Only for CI test
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 set(CMAKE_BUILD_RPATH_USE_ORIGIN ON)
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
-set(CMAKE_BUILD_PARALLEL_LEVEL ${NumCPU})
 
 if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-packed-bitfield-compat")
@@ -87,12 +103,6 @@ if (NOT CMAKE_BUILD_TYPE)
     set(CMAKE_BUILD_TYPE Release)
 endif ()
 
-# If ccache exists, use it to speed up compiling
-find_program(CCACHE_FOUND ccache)
-if(CCACHE_FOUND)
-    set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache)
-endif(CCACHE_FOUND)
-
 # CMake dirs
 list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/CMake)
 set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/output)
@@ -152,6 +162,10 @@ endforeach ()
 
 add_subdirectory(third_party)
 
+if (PHOTON_ENABLE_ECOSYSTEM)
+    add_subdirectory(ecosystem)
+endif ()
+
 # Compile photon objects
 file(GLOB PHOTON_SRC
         photon.cpp
@@ -181,7 +195,7 @@ file(GLOB PHOTON_SRC
 if (APPLE)
     list(APPEND PHOTON_SRC io/kqueue.cpp)
 else ()
-    list(APPEND PHOTON_SRC io/aio-wrapper.cpp io/epoll.cpp)
+    list(APPEND PHOTON_SRC io/aio-wrapper.cpp io/epoll.cpp io/epoll-ng.cpp)
     if (PHOTON_ENABLE_URING)
         list(APPEND PHOTON_SRC io/iouring-wrapper.cpp)
     endif ()
@@ -199,10 +213,19 @@ if (PHOTON_ENABLE_EXTFS)
     file(GLOB EXTFS_SRC fs/extfs/*.cpp)
     list(APPEND PHOTON_SRC ${EXTFS_SRC})
 endif ()
+if (PHOTON_ENABLE_ECOSYSTEM)
+    file(GLOB ECOSYSTEM_SRC ecosystem/*.cpp)
+    list(APPEND PHOTON_SRC ${ECOSYSTEM_SRC})
+endif ()
 
 # An object library compiles source files but does not archive or link their object files.
 add_library(photon_obj OBJECT ${PHOTON_SRC})
-target_include_directories(photon_obj PRIVATE include)
+if (PHOTON_ENABLE_ECOSYSTEM)
+    target_link_libraries(photon_obj PRIVATE ecosystem_deps)
+endif ()
+target_include_directories(photon_obj PRIVATE include ${OPENSSL_INCLUDE_DIRS} ${AIO_INCLUDE_DIRS}
+        ${ZLIB_INCLUDE_DIRS} ${CURL_INCLUDE_DIRS}
+)
 
 target_compile_definitions(photon_obj PRIVATE _FILE_OFFSET_BITS=64 FUSE_USE_VERSION=29)
 if (PHOTON_ENABLE_URING)
@@ -218,10 +241,6 @@ endif()
 if (PHOTON_ENABLE_EXTFS)
     target_include_directories(photon_obj PRIVATE ${E2FS_INCLUDE_DIRS})
 endif()
-if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 8.0)
-    # This option is enabled by default after -std=c++17
-    target_compile_options(photon_obj PRIVATE -faligned-new)
-endif()
 
 if (actually_built)
     add_dependencies(photon_obj ${actually_built})
@@ -357,6 +376,9 @@ if (PHOTON_BUILD_TESTING)
     add_subdirectory(rpc/test)
     add_subdirectory(thread/test)
     add_subdirectory(net/security-context/test)
+    if (PHOTON_ENABLE_ECOSYSTEM)
+        add_subdirectory(ecosystem/test)
+    endif ()
     if (PHOTON_ENABLE_EXTFS)
         add_subdirectory(fs/extfs/test)
     endif ()

README.md

@@ -2,11 +2,13 @@
 
 [![CI](https://github.com/alibaba/PhotonLibOS/actions/workflows/ci.linux.x86.yml/badge.svg)](https://github.com/alibaba/PhotonLibOS/actions/workflows/ci.linux.x86.yml)
 [![CI](https://github.com/alibaba/PhotonLibOS/actions/workflows/ci.linux.arm.yml/badge.svg)](https://github.com/alibaba/PhotonLibOS/actions/workflows/ci.linux.arm.yml)
-[![CI](https://github.com/alibaba/PhotonLibOS/actions/workflows/ci.macos.yml/badge.svg)](https://github.com/alibaba/PhotonLibOS/actions/workflows/ci.macos.yml)
+[![CI](https://github.com/alibaba/PhotonLibOS/actions/workflows/ci.macos.yml/badge.svg)](https://github.com/alibaba/PhotonLibOS/actions/workflows/ci.macos.x86.yml)
+[![CI](https://github.com/alibaba/PhotonLibOS/actions/workflows/ci.macos.yml/badge.svg)](https://github.com/alibaba/PhotonLibOS/actions/workflows/ci.macos.arm.yml)
 
 [PhotonlibOS.github.io](https://photonlibos.github.io)
 
 ## What's New
+* Feb 2024，[中文文档](https://photonlibos.github.io/cn/docs/category/introduction)在官网上线了
 * Since 0.7, Photon will use release branches to enhance the reliability of software delivery. Bugfix will be merged into a stable release at first, then to higher release versions, and finally main.
 * Since version 0.6, Photon can run with a userspace TCP/IP stack on top of `DPDK`.
 [En](https://developer.aliyun.com/article/1208512) / [中文](https://developer.aliyun.com/article/1208390).

common/PMF.h

@@ -24,6 +24,8 @@ struct pmf_map
     T* obj; // may be adjusted for virtual function call
 };
 
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Warray-bounds"
 template<typename PF, typename T, typename MF>
 inline auto __get_mfa__(T* obj, MF f)
     -> pmf_map<PF, T>
@@ -65,6 +67,7 @@ inline auto __get_mfa__(T* obj, MF f)
     auto addr = pmf.get_function_address((void*&)obj);
     return pmf_map<PF, T>{(PF)addr, obj};
 }
+#pragma GCC diagnostic pop
 
 template<typename T, typename R, typename...ARGS>
 inline auto get_member_function_address(T* obj, R (T::*f)(ARGS...))