From 8261b9307ba21561bb70e8391d2c827657833ded Mon Sep 17 00:00:00 2001 From: Huiba Li Date: Mon, 3 Jun 2024 17:18:11 +0800 Subject: [PATCH] new CI workflow for linux.x86 (#494) new CI workflow for linux.x86 --- .github/workflows/ci.linux.arm.yml | 8 +- .github/workflows/ci.linux.x86-64.yml | 230 ++++++++++++++ .github/workflows/ci.linux.x86.yml | 119 ------- .github/workflows/ci.macos.arm.yml | 18 +- .../{ci.macos.yml => ci.macos.x86.yml} | 14 +- CMake/Findgoogletest.cmake | 1 + CMake/build-from-src.cmake | 2 +- CMake/generate-ctest-packed-script.cmake | 17 + CMakeLists.txt | 21 +- common/checksum/test/test_checksum.cpp | 3 + common/executor/test/CMakeLists.txt | 24 ++ common/executor/test/test_async.cpp | 12 +- common/executor/test/test_easyexport.cpp | 2 +- .../executor/test/test_export_as_executor.cpp | 1 - common/lockfree_queue.h | 7 +- common/memory-stream/test/CMakeLists.txt | 6 + common/memory-stream/test/test.cpp | 2 + common/stream-messenger/messenger.cpp | 89 ------ common/stream-messenger/messenger.h | 41 --- common/stream-messenger/test/test.cpp | 101 ------ common/test/CMakeLists.txt | 6 +- common/test/perf_objcache.cpp | 1 + common/test/test.cpp | 4 + common/test/test_alog.cpp | 2 + common/test/test_constexprstr.cpp | 2 + common/test/test_lockfree.cpp | 36 +-- common/test/test_objcache.cpp | 17 +- common/test/test_scalepool.cpp | 5 +- common/test/test_throttle.cpp | 5 +- fs/exportfs.cpp | 1 + fs/test/CMakeLists.txt | 2 +- fs/test/test.cpp | 7 +- fs/test/test_exportfs.cpp | 75 ++--- fs/test/test_throttledfile.cpp | 2 + io/fd-events.h | 2 +- io/test/CMakeLists.txt | 2 +- io/test/test-iouring.cpp | 4 +- net/http/test/CMakeLists.txt | 4 +- net/test/CMakeLists.txt | 2 +- photon.cpp | 28 +- rpc/test/test.cpp | 3 +- test/ci-tools.cpp | 112 +++++-- test/ci-tools.h | 10 +- thread/test/CMakeLists.txt | 8 +- thread/test/test-multi-vcpu-locking.cpp | 3 +- thread/test/test-pool.cpp | 297 ++++++++++++++++++ thread/test/test.cpp | 275 +--------------- thread/thread.cpp | 5 +- 48 files changed, 836 insertions(+), 802 deletions(-) create mode 100644 .github/workflows/ci.linux.x86-64.yml delete mode 100644 .github/workflows/ci.linux.x86.yml rename .github/workflows/{ci.macos.yml => ci.macos.x86.yml} (74%) create mode 100644 CMake/generate-ctest-packed-script.cmake create mode 100644 common/executor/test/CMakeLists.txt create mode 100644 common/memory-stream/test/CMakeLists.txt delete mode 100644 common/stream-messenger/messenger.cpp delete mode 100644 common/stream-messenger/messenger.h delete mode 100644 common/stream-messenger/test/test.cpp create mode 100644 thread/test/test-pool.cpp diff --git a/.github/workflows/ci.linux.arm.yml b/.github/workflows/ci.linux.arm.yml index 5418a3e3..fbe35f61 100644 --- a/.github/workflows/ci.linux.arm.yml +++ b/.github/workflows/ci.linux.arm.yml @@ -37,12 +37,12 @@ jobs: source /opt/rh/gcc-toolset-9/enable cmake -B build -D CMAKE_BUILD_TYPE=MinSizeRel -D PHOTON_BUILD_TESTING=ON \ -D PHOTON_ENABLE_SASL=ON -D PHOTON_ENABLE_FUSE=ON -D PHOTON_ENABLE_EXTFS=ON - cmake --build build -j -- VERBOSE=1 + cmake --build build -j $(nproc) -- VERBOSE=1 - name: Test run: | cd build - ctest --timeout 3600 -V + ctest -E test-lockfree --timeout 3600 -V centos8-gcc921-epoll-debug: runs-on: [self-hosted, Linux, ARM64] @@ -74,9 +74,9 @@ jobs: source /opt/rh/gcc-toolset-9/enable cmake -B build -D CMAKE_BUILD_TYPE=Debug -D PHOTON_BUILD_TESTING=ON \ -D PHOTON_ENABLE_SASL=ON -D PHOTON_ENABLE_FUSE=ON -D PHOTON_ENABLE_EXTFS=ON - cmake --build build -j -- VERBOSE=1 + cmake --build build -j $(nproc) -- VERBOSE=1 - name: Test run: | cd build - ctest --timeout 3600 -V + ctest -E test-lockfree --timeout 3600 -V diff --git a/.github/workflows/ci.linux.x86-64.yml b/.github/workflows/ci.linux.x86-64.yml new file mode 100644 index 00000000..453dff50 --- /dev/null +++ b/.github/workflows/ci.linux.x86-64.yml @@ -0,0 +1,230 @@ +name: Linux x86 (new) + +on: + push: + branches: [ "main", "release/*" ] + pull_request: + branches: [ "main", "release/*" ] + +jobs: + gcc850: + runs-on: [self-hosted, compiler] + steps: + - uses: actions/checkout@v3 + - name: Build850 + run: | + rm -fr build + cmake -B build -D CMAKE_BUILD_TYPE=MinSizeRel \ + -D PHOTON_BUILD_TESTING=ON \ + -D PHOTON_ENABLE_SASL=ON \ + -D PHOTON_ENABLE_FUSE=ON \ + -D PHOTON_ENABLE_URING=ON \ + -D PHOTON_ENABLE_EXTFS=ON \ + # -D PHOTON_BUILD_DEPENDENCIES=ON \ + # -D PHOTON_AIO_SOURCE="" \ + # -D PHOTON_ZLIB_SOURCE="" \ + # -D PHOTON_CURL_SOURCE="" \ + # -D PHOTON_OPENSSL_SOURCE="" \ + # -D PHOTON_GFLAGS_SOURCE="" \ + # -D PHOTON_GOOGLETEST_SOURCE="" \ + # -D PHOTON_URING_SOURCE=https://github.com/axboe/liburing/archive/refs/tags/liburing-2.3.tar.gz + + cmake --build build -j $(nproc) --clean-first -- VERBOSE=1 + ln -f common/checksum/test/checksum.in build/output/ + tar -c -h --use-compress-program=zstdmt -f output850.tzs build/output/ + - name: Upload850 + uses: actions/upload-artifact@v4 + with: + name: output850 + path: output850.tzs + retention-days: 5 + compression-level: 0 + + test850: + needs: gcc850 + runs-on: ubuntu-latest + container: + image: ghcr.io/coldwings/photon-ut-base:latest + options: --cpus 4 + steps: + - uses: szenius/set-timezone@v1.2 + with: + timezoneLinux: "Asia/Shanghai" + timezoneMacos: "Asia/Shanghai" + timezoneWindows: "China Standard Time" + - uses: actions/download-artifact@v4 + with: + name: output850 + - name: test + run: | + tar -x --use-compress-program=zstdmt -f output850.tzs + cd build/output/ + ctest -E test-lockfree --timeout 3600 -V + export PHOTON_CI_EV_ENGINE=io_uring + ctest -E test-lockfree --timeout 3600 -V + + gcc921: + needs: gcc850 + runs-on: [self-hosted, compiler] + steps: + - name: Build921 + run: | + source /opt/rh/gcc-toolset-9/enable + cmake --build build -j $(nproc) --clean-first -- VERBOSE=1 + ln -f common/checksum/test/checksum.in build/output/ + tar -c --use-compress-program=zstdmt -f output921.tzs build/output/ + - name: Upload921 + uses: actions/upload-artifact@v4 + with: + name: output921 + path: output921.tzs + retention-days: 5 + compression-level: 0 + + test921: + needs: gcc921 + runs-on: ubuntu-latest + container: + image: ghcr.io/coldwings/photon-ut-base:latest + options: --cpus 4 + steps: + - uses: szenius/set-timezone@v1.2 + with: + timezoneLinux: "Asia/Shanghai" + timezoneMacos: "Asia/Shanghai" + timezoneWindows: "China Standard Time" + - uses: actions/download-artifact@v4 + with: + name: output921 + - name: test + run: | + tar -x --use-compress-program=zstdmt -f output921.tzs + cd build/output/ + ctest -E test-lockfree --timeout 3600 -V + export PHOTON_CI_EV_ENGINE=io_uring + ctest -E test-lockfree --timeout 3600 -V + + gcc1031: + needs: gcc921 + runs-on: [self-hosted, compiler] + steps: + - name: Build1031 + run: | + source /opt/rh/gcc-toolset-10/enable + cmake --build build -j $(nproc) --clean-first -- VERBOSE=1 + ln -f common/checksum/test/checksum.in build/output/ + tar -c --use-compress-program=zstdmt -f output1031.tzs build/output/ + - name: Upload1031 + uses: actions/upload-artifact@v4 + with: + name: output1031 + path: output1031.tzs + retention-days: 5 + compression-level: 0 + + test1031: + needs: gcc1031 + runs-on: ubuntu-latest + container: + image: ghcr.io/coldwings/photon-ut-base:latest + options: --cpus 4 + steps: + - uses: szenius/set-timezone@v1.2 + with: + timezoneLinux: "Asia/Shanghai" + timezoneMacos: "Asia/Shanghai" + timezoneWindows: "China Standard Time" + - uses: actions/download-artifact@v4 + with: + name: output1031 + - name: test + run: | + tar -x --use-compress-program=zstdmt -f output1031.tzs + cd build/output/ + ctest -E test-lockfree --timeout 3600 -V + export PHOTON_CI_EV_ENGINE=io_uring + ctest -E test-lockfree --timeout 3600 -V + + gcc1121: + needs: gcc1031 + runs-on: [self-hosted, compiler] + steps: + - name: Build1121 + run: | + source /opt/rh/gcc-toolset-10/enable + cmake --build build -j --clean-first -- VERBOSE=1 + ln -f common/checksum/test/checksum.in build/output/ + tar -c --use-compress-program=zstdmt -f output1121.tzs build/output/ + - name: Upload1121 + uses: actions/upload-artifact@v4 + with: + name: output1121 + path: output1121.tzs + retention-days: 5 + compression-level: 0 + + test1121: + needs: gcc1121 + runs-on: ubuntu-latest + container: + image: ghcr.io/coldwings/photon-ut-base:latest + options: --cpus 4 + steps: + - uses: szenius/set-timezone@v1.2 + with: + timezoneLinux: "Asia/Shanghai" + timezoneMacos: "Asia/Shanghai" + timezoneWindows: "China Standard Time" + - uses: actions/download-artifact@v4 + with: + name: output1121 + - name: test + run: | + tar -x --use-compress-program=zstdmt -f output1121.tzs + cd build/output/ + ctest -E test-lockfree --timeout 3600 -V + export PHOTON_CI_EV_ENGINE=io_uring + ctest -E test-lockfree --timeout 3600 -V + + gcc1211: + needs: gcc1121 + runs-on: [self-hosted, compiler] + steps: + - name: Build1211 + run: | + source /opt/rh/gcc-toolset-10/enable + cmake --build build -j $(nproc) --clean-first -- VERBOSE=1 + ln -f common/checksum/test/checksum.in build/output/ + tar -c --use-compress-program=zstdmt -f output1211.tzs build/output/ + - name: Upload1211 + uses: actions/upload-artifact@v4 + with: + name: output1211 + path: output1211.tzs + retention-days: 5 + compression-level: 0 + + test1211: + needs: gcc1211 + runs-on: ubuntu-latest + container: + image: ghcr.io/coldwings/photon-ut-base:latest + options: --cpus 4 + steps: + - uses: szenius/set-timezone@v1.2 + with: + timezoneLinux: "Asia/Shanghai" + timezoneMacos: "Asia/Shanghai" + timezoneWindows: "China Standard Time" + - uses: actions/download-artifact@v4 + with: + name: output1211 + - name: test + run: | + tar -x --use-compress-program=zstdmt -f output1211.tzs + cd build/output/ + ctest -E test-lockfree --timeout 3600 -V + export PHOTON_CI_EV_ENGINE=io_uring + ctest -E test-lockfree --timeout 3600 -V + + diff --git a/.github/workflows/ci.linux.x86.yml b/.github/workflows/ci.linux.x86.yml deleted file mode 100644 index b0101a2b..00000000 --- a/.github/workflows/ci.linux.x86.yml +++ /dev/null @@ -1,119 +0,0 @@ -name: Linux x86 - -on: - push: - branches: [ "main", "release/*" ] - pull_request: - branches: [ "main", "release/*" ] - -jobs: - centos8-gcc921-epoll-release: - runs-on: ubuntu-latest - - container: - image: dokken/centos-stream-8:sha-40294ce - options: --cpus 4 - - steps: - - uses: szenius/set-timezone@v1.2 - with: - timezoneLinux: "Asia/Shanghai" - timezoneMacos: "Asia/Shanghai" - timezoneWindows: "China Standard Time" - - - uses: actions/checkout@v3 - - - name: Install Dependencies - run: | - dnf install -y git gcc-c++ cmake 'dnf-command(config-manager)' - dnf install -y gcc-toolset-9-gcc-c++ - dnf install -y openssl-devel libcurl-devel libaio-devel - dnf install -y epel-release - dnf config-manager --set-enabled powertools - dnf install -y gtest-devel gmock-devel gflags-devel fuse-devel libgsasl-devel e2fsprogs-devel - - - name: Build - run: | - source /opt/rh/gcc-toolset-9/enable - cmake -B build -D CMAKE_BUILD_TYPE=MinSizeRel -D PHOTON_BUILD_TESTING=ON \ - -D PHOTON_ENABLE_SASL=ON -D PHOTON_ENABLE_FUSE=ON -D PHOTON_ENABLE_EXTFS=ON - cmake --build build -j -- VERBOSE=1 - - - name: Test - run: | - cd build - ctest --timeout 3600 -V - - centos8-gcc921-epoll-debug: - runs-on: ubuntu-latest - - container: - image: dokken/centos-stream-8:sha-40294ce - options: --cpus 4 - - steps: - - uses: szenius/set-timezone@v1.2 - with: - timezoneLinux: "Asia/Shanghai" - timezoneMacos: "Asia/Shanghai" - timezoneWindows: "China Standard Time" - - - uses: actions/checkout@v3 - - - name: Install Dependencies - run: | - dnf install -y git gcc-c++ cmake 'dnf-command(config-manager)' - dnf install -y gcc-toolset-9-gcc-c++ - dnf install -y openssl-devel libcurl-devel libaio-devel - dnf install -y epel-release - dnf config-manager --set-enabled powertools - dnf install -y gtest-devel gmock-devel gflags-devel fuse-devel libgsasl-devel e2fsprogs-devel - - - name: Build - run: | - source /opt/rh/gcc-toolset-9/enable - cmake -B build -D CMAKE_BUILD_TYPE=Debug -D PHOTON_BUILD_TESTING=ON \ - -D PHOTON_ENABLE_SASL=ON -D PHOTON_ENABLE_FUSE=ON -D PHOTON_ENABLE_EXTFS=ON - cmake --build build -j -- VERBOSE=1 - - - name: Test - run: | - cd build - ctest --timeout 3600 -V - - centos8-gcc921-iouring-release: - runs-on: [self-hosted, Linux, X64, io_uring] - - container: - image: dokken/centos-stream-8:sha-40294ce - options: --cpus 4 - - steps: - - uses: szenius/set-timezone@v1.2 - with: - timezoneLinux: "Asia/Shanghai" - timezoneMacos: "Asia/Shanghai" - timezoneWindows: "China Standard Time" - - - uses: actions/checkout@v3 - - - name: Install Dependencies - run: | - dnf install -y git gcc-c++ cmake - dnf install -y gcc-toolset-9-gcc-c++ - dnf install -y autoconf automake libtool - - - name: Build - run: | - source /opt/rh/gcc-toolset-9/enable - cmake -B build -D CMAKE_BUILD_TYPE=MinSizeRel \ - -D PHOTON_BUILD_DEPENDENCIES=ON \ - -D PHOTON_BUILD_TESTING=ON \ - -D PHOTON_ENABLE_URING=ON - cmake --build build -j - - - name: Test - run: | - cd build - export PHOTON_CI_EV_ENGINE=io_uring - ctest --timeout 3600 -V diff --git a/.github/workflows/ci.macos.arm.yml b/.github/workflows/ci.macos.arm.yml index 77ad0230..a8084fbd 100644 --- a/.github/workflows/ci.macos.arm.yml +++ b/.github/workflows/ci.macos.arm.yml @@ -7,15 +7,15 @@ on: branches: [ "main", "release/*" ] jobs: - macOS-clang-release: - runs-on: [self-hosted, macOS, ARM64] + macOS14-arm: + runs-on: macos-14 steps: -# - uses: szenius/set-timezone@v1.2 -# with: -# timezoneLinux: "Asia/Shanghai" -# timezoneMacos: "Asia/Shanghai" -# timezoneWindows: "China Standard Time" + - uses: szenius/set-timezone@v1.2 + with: + timezoneLinux: "Asia/Shanghai" + timezoneMacos: "Asia/Shanghai" + timezoneWindows: "China Standard Time" - uses: actions/checkout@v3 @@ -28,8 +28,8 @@ jobs: run: | cmake -B ${{github.workspace}}/build -D PHOTON_BUILD_TESTING=ON -D CMAKE_BUILD_TYPE=Release \ -D PHOTON_ENABLE_SASL=ON -DOPENSSL_ROOT_DIR=/usr/local/opt/openssl@3 - cmake --build ${{github.workspace}}/build -j + cmake --build ${{github.workspace}}/build -j $(nproc) - name: Test working-directory: ${{github.workspace}}/build - run: ctest --timeout 3600 -V + run: ctest -E test-lockfree --timeout 3600 -V diff --git a/.github/workflows/ci.macos.yml b/.github/workflows/ci.macos.x86.yml similarity index 74% rename from .github/workflows/ci.macos.yml rename to .github/workflows/ci.macos.x86.yml index 2ecfae9e..4ca1ef90 100644 --- a/.github/workflows/ci.macos.yml +++ b/.github/workflows/ci.macos.x86.yml @@ -7,8 +7,8 @@ on: branches: [ "main", "release/*" ] jobs: - macOS-12-Monterey-release: - runs-on: macos-12 + macOS13-x86: + runs-on: macos-13 steps: - uses: szenius/set-timezone@v1.2 @@ -20,17 +20,17 @@ jobs: - uses: actions/checkout@v3 - name: Install Dependencies - shell: bash + shell: bash run: | - brew install cmake openssl gflags googletest gsasl + brew install cmake openssl gflags googletest gsasl - name: Build run: | cmake -B ${{github.workspace}}/build -D PHOTON_BUILD_TESTING=ON -D CMAKE_BUILD_TYPE=Release \ -D PHOTON_ENABLE_SASL=ON -DOPENSSL_ROOT_DIR=/usr/local/opt/openssl@3 - cmake --build ${{github.workspace}}/build -j + cmake --build ${{github.workspace}}/build -j $(nproc) - name: Test working-directory: ${{github.workspace}}/build - run: ctest --timeout 3600 -V - + run: ctest -E test-lockfree --timeout 3600 -V + diff --git a/CMake/Findgoogletest.cmake b/CMake/Findgoogletest.cmake index 0de8d20f..e1e82570 100644 --- a/CMake/Findgoogletest.cmake +++ b/CMake/Findgoogletest.cmake @@ -1,6 +1,7 @@ find_path(GOOGLETEST_INCLUDE_DIRS gtest/gtest.h gmock/gmock.h) find_library(GOOGLETEST_GTEST_LIBRARIES gtest) +find_library(GOOGLETEST_GTEST_MAIN_LIBRARIES gtest_main) find_library(GOOGLETEST_GMOCK_LIBRARIES gmock) set(GOOGLETEST_LIBRARIES diff --git a/CMake/build-from-src.cmake b/CMake/build-from-src.cmake index caeb2598..c7bc59d7 100644 --- a/CMake/build-from-src.cmake +++ b/CMake/build-from-src.cmake @@ -43,7 +43,7 @@ function(build_from_src [dep]) UPDATE_DISCONNECTED ON BUILD_IN_SOURCE ON CONFIGURE_COMMAND ./configure --prefix=${BINARY_DIR} - BUILD_COMMAND sh -c "V=1 CFLAGS=\"-fPIC -g -O3 -Wall -Wextra -fno-stack-protector\" $(MAKE) -C src" + BUILD_COMMAND sh -c "V=1 CFLAGS=\"-fPIC -O3 -Wall -Wextra -fno-stack-protector\" $(MAKE) -C src" INSTALL_COMMAND $(MAKE) install ) set(URING_INCLUDE_DIRS ${BINARY_DIR}/include PARENT_SCOPE) diff --git a/CMake/generate-ctest-packed-script.cmake b/CMake/generate-ctest-packed-script.cmake new file mode 100644 index 00000000..ab0d6e8e --- /dev/null +++ b/CMake/generate-ctest-packed-script.cmake @@ -0,0 +1,17 @@ +function(__GenerateStandaloneCTestScript DIR FILEPATH) + get_property(TGTS DIRECTORY "${DIR}" PROPERTY TESTS) + foreach(TGT IN LISTS TGTS) + file(APPEND ${FILEPATH} "add_test(${TGT} \"./${TGT}\")\n") + endforeach() + + get_property(SUBDIRS DIRECTORY "${DIR}" PROPERTY SUBDIRECTORIES) + foreach(SUBDIR IN LISTS SUBDIRS) + __GenerateStandaloneCTestScript("${SUBDIR}" ${FILEPATH}) + endforeach() +endfunction() + + +function(GenerateStandaloneCTestScript DIR FILEPATH) + file(WRITE ${FILEPATH} "") + __GenerateStandaloneCTestScript(${DIR} ${FILEPATH}) +endfunction() diff --git a/CMakeLists.txt b/CMakeLists.txt index 63794d65..7c844c89 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -55,7 +55,7 @@ set(CMAKE_CXX_EXTENSIONS OFF) set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g") set(CMAKE_CXX_FLAGS_RELEASE "-O2 -DNDEBUG") set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -DNDEBUG -g") -set(CMAKE_CXX_FLAGS_MINSIZEREL "-O2 -g") # Only for CI test +set(CMAKE_CXX_FLAGS_MINSIZEREL "-O2") # Only for CI test set(CMAKE_EXPORT_COMPILE_COMMANDS ON) set(CMAKE_BUILD_RPATH_USE_ORIGIN ON) set(CMAKE_POSITION_INDEPENDENT_CODE ON) @@ -159,7 +159,6 @@ file(GLOB PHOTON_SRC common/checksum/*.cpp common/executor/*.cpp common/memory-stream/*.cpp - common/stream-messenger/*.cpp fs/aligned-file.cpp fs/async_filesystem.cpp fs/exportfs.cpp @@ -203,9 +202,8 @@ endif () # An object library compiles source files but does not archive or link their object files. add_library(photon_obj OBJECT ${PHOTON_SRC}) -target_include_directories(photon_obj PRIVATE include ${OPENSSL_INCLUDE_DIRS} ${AIO_INCLUDE_DIRS} - ${ZLIB_INCLUDE_DIRS} ${CURL_INCLUDE_DIRS} -) +target_include_directories(photon_obj PRIVATE include) + target_compile_definitions(photon_obj PRIVATE _FILE_OFFSET_BITS=64 FUSE_USE_VERSION=29) if (PHOTON_ENABLE_URING) target_include_directories(photon_obj PRIVATE ${URING_INCLUDE_DIRS}) @@ -334,16 +332,24 @@ endif () # Build test cases if (PHOTON_BUILD_TESTING) + include_directories(photon_static ${GFLAGS_INCLUDE_DIRS} ${GOOGLETEST_INCLUDE_DIRS}) + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/examples-output) + link_libraries(${GFLAGS_LIBRARIES}) + add_subdirectory(examples) + include(CTest) + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/output) + include(generate-ctest-packed-script) add_library(ci-tools STATIC test/ci-tools.cpp) + target_include_directories(ci-tools PRIVATE include) - include_directories(photon_static ${GFLAGS_INCLUDE_DIRS} ${GOOGLETEST_INCLUDE_DIRS}) link_libraries(${GFLAGS_LIBRARIES} ${GOOGLETEST_LIBRARIES} ci-tools) - add_subdirectory(examples) add_subdirectory(common/checksum/test) add_subdirectory(common/test) + add_subdirectory(common/memory-stream/test) + add_subdirectory(common/executor/test) add_subdirectory(fs/test) add_subdirectory(io/test) add_subdirectory(net/test) @@ -354,4 +360,5 @@ if (PHOTON_BUILD_TESTING) if (PHOTON_ENABLE_EXTFS) add_subdirectory(fs/extfs/test) endif () + GenerateStandaloneCTestScript(${CMAKE_SOURCE_DIR} ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/CTestTestfile.cmake) endif () diff --git a/common/checksum/test/test_checksum.cpp b/common/checksum/test/test_checksum.cpp index e6549ff4..8a3764b8 100644 --- a/common/checksum/test/test_checksum.cpp +++ b/common/checksum/test/test_checksum.cpp @@ -6,6 +6,7 @@ #include #include #include +#include "../../../test/ci-tools.h" #ifndef DATA_DIR #define DATA_DIR "" @@ -17,6 +18,7 @@ class TestChecksum : public ::testing::Test { virtual void SetUp() { in.open(xstr(DATA_DIR) "checksum.in"); + if (!in) in.open("checksum.in"); ASSERT_TRUE(!!in); uint32_t value; std::string str; @@ -70,6 +72,7 @@ TEST_F(TestChecksum, crc32c_sw) { int main(int argc, char **argv) { + if (!photon::is_using_default_engine()) return 0; ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } \ No newline at end of file diff --git a/common/executor/test/CMakeLists.txt b/common/executor/test/CMakeLists.txt new file mode 100644 index 00000000..303dd448 --- /dev/null +++ b/common/executor/test/CMakeLists.txt @@ -0,0 +1,24 @@ +add_definitions(-w) + +add_executable(test-executor-async test_async.cpp) +target_link_libraries(test-executor-async PRIVATE photon_shared ${GOOGLETEST_GTEST_MAIN_LIBRARIES}) +add_test(NAME test-executor-async COMMAND $) + +# add_executable(test-executor-easy test_easy.cpp ../../../third_party/easy_weak/easy_weak.cpp) +# target_include_directories(ci-tools PRIVATE third_party/easy_weak) +# target_link_libraries(test-executor-easy PRIVATE photon_shared ${GOOGLETEST_GTEST_MAIN_LIBRARIES}) +# add_test(NAME test-executor-easy COMMAND $) + +# add_executable(test-executor-easyexport test_easyexport.cpp ../../../third_party/easy_weak/easy_weak.cpp) +# target_include_directories(ci-tools PRIVATE third_party/easy_weak) +# target_link_libraries(test-executor-easyexport PRIVATE photon_shared ${GOOGLETEST_GTEST_MAIN_LIBRARIES}) +# add_test(NAME test-executor-easyexport COMMAND $) + +add_executable(test-executor-export_as_executor test_export_as_executor.cpp) +target_link_libraries(test-executor-export_as_executor PRIVATE photon_shared ${GOOGLETEST_GTEST_MAIN_LIBRARIES}) +add_test(NAME test-executor-export_as_executor COMMAND $) + +add_executable(test-executor-std test_std.cpp) +target_link_libraries(test-executor-std PRIVATE photon_shared ${GOOGLETEST_GTEST_MAIN_LIBRARIES}) +add_test(NAME test-executor-std COMMAND $) + diff --git a/common/executor/test/test_async.cpp b/common/executor/test/test_async.cpp index 6248113b..b7d6bc69 100644 --- a/common/executor/test/test_async.cpp +++ b/common/executor/test/test_async.cpp @@ -23,8 +23,10 @@ limitations under the License. #include #include #include -#include -#include +// #include +// #include +#include "../../../test/ci-tools.h" +// #include #include #include @@ -75,11 +77,7 @@ TEST(std_executor, perf) { for (int i = 0; i < th_num; i++) { ths.emplace_back([&] { for (int j = 0; j < app_num; j++) { - cpu_set_t cpuset; - CPU_ZERO(&cpuset); - CPU_SET(i, &cpuset); - pthread_setaffinity_np(pthread_self(), - sizeof(cpu_set_t), &cpuset); + photon::set_cpu_affinity(j); auto start = std::chrono::high_resolution_clock::now(); eth.async_perform(new auto ([&] { cnt++; if (cnt % 10000 == 0) printf("%d\n", cnt);})); auto end = std::chrono::high_resolution_clock::now(); diff --git a/common/executor/test/test_easyexport.cpp b/common/executor/test/test_easyexport.cpp index 3266f395..b4eeafe0 100644 --- a/common/executor/test/test_easyexport.cpp +++ b/common/executor/test/test_easyexport.cpp @@ -84,7 +84,7 @@ TEST(easy_performer, test) { easy_atomic_set(count, 10000); std::thread([]() { - if (photon::init(photon::INIT_EVENT_DEFAULT, photon::INIT_IO_NONE)) + if (ci_init_photon(photon::INIT_EVENT_DEFAULT, photon::INIT_IO_NONE)) return -1; DEFER(photon::fini()); fs::exportfs_init(); diff --git a/common/executor/test/test_export_as_executor.cpp b/common/executor/test/test_export_as_executor.cpp index 33c1bd19..f0c66cfd 100644 --- a/common/executor/test/test_export_as_executor.cpp +++ b/common/executor/test/test_export_as_executor.cpp @@ -4,7 +4,6 @@ #include #include #include - #include TEST(enter_as_executor, test) { diff --git a/common/lockfree_queue.h b/common/lockfree_queue.h index 8bdcaac3..2d91d123 100644 --- a/common/lockfree_queue.h +++ b/common/lockfree_queue.h @@ -31,6 +31,8 @@ limitations under the License. #include #include +#define size_t uint64_t + template struct Capacity_2expN { constexpr static size_t capacity = Capacity_2expN<(x >> 1)>::capacity << 1; @@ -599,4 +601,7 @@ class RingChannel : public QueueType { }; } // namespace common -} // namespace photon \ No newline at end of file +} // namespace photon + +#undef size_t + diff --git a/common/memory-stream/test/CMakeLists.txt b/common/memory-stream/test/CMakeLists.txt new file mode 100644 index 00000000..5720f40c --- /dev/null +++ b/common/memory-stream/test/CMakeLists.txt @@ -0,0 +1,6 @@ +add_definitions(-w) + +add_executable(test-memory-stream test.cpp) +target_link_libraries(test-memory-stream PRIVATE photon_shared) +add_test(NAME test-memory-stream COMMAND $) + diff --git a/common/memory-stream/test/test.cpp b/common/memory-stream/test/test.cpp index 330e6544..4f006812 100644 --- a/common/memory-stream/test/test.cpp +++ b/common/memory-stream/test/test.cpp @@ -23,6 +23,7 @@ limitations under the License. #include #include #include +#include "../../../test/ci-tools.h" using namespace std; using namespace photon; @@ -82,6 +83,7 @@ TEST(MemoryStream, normalTest) int main(int argc, char **argv) { + if (!photon::is_using_default_engine()) return 0; log_output_level = ALOG_DEBUG; ::testing::InitGoogleTest(&argc, argv); photon::vcpu_init(); diff --git a/common/stream-messenger/messenger.cpp b/common/stream-messenger/messenger.cpp deleted file mode 100644 index 293ac6e2..00000000 --- a/common/stream-messenger/messenger.cpp +++ /dev/null @@ -1,89 +0,0 @@ -/* -Copyright 2022 The Photon Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -#include "messenger.h" -#include - -namespace StreamMessenger -{ - // This class turns a stream into a message channel, by adding a `Header` to the - // message before send it, and extracting the header immediately after recv it. - // The stream can be any `IStream` object, like TCP socket or UNIX domain socket. - class Messenger : public IMessageChannel - { - public: - IStream* m_stream; - Header m_header; // used for recv, in case of insufficient buffer space - Messenger(IStream* stream) - { - m_stream = stream; - m_header.magic = 0; - } - virtual ssize_t send(iovector& msg) override - { - auto size = msg.sum(); - if (size > UINT32_MAX - sizeof(Header)) - LOG_ERRNO_RETURN(EINVAL, -1, "the message is too large to send (` > 4GB)", size); - - Header h; - h.size = (uint32_t)size; - msg.push_front({&h, sizeof(h)}); - ssize_t ret = m_stream->writev(msg.iovec(), msg.iovcnt()); - if (ret < (ssize_t)(size + sizeof(h))) - LOG_ERROR_RETURN(0, -1, "failed to write to underlay stream ", m_stream); - - return (ssize_t)size; - } - virtual ssize_t recv(iovector& msg) override - { - if (m_header.magic != Header::MAGIC) - { - ssize_t ret = m_stream->read(&m_header, sizeof(m_header)); - if (ret < (ssize_t)sizeof(m_header)) - LOG_ERRNO_RETURN(0, -1, "failed to read the header from stream ", m_stream); - if (m_header.magic != Header::MAGIC) - LOG_ERROR_RETURN(EBADMSG, -1, "invalid header magic in the recvd message"); - if (m_header.version > Header::VERSION) - LOG_ERROR_RETURN(EBADMSG, -1 , "invlaid header version in the recvd message"); - } - - size_t size = msg.truncate(m_header.size); - auto delta = (ssize_t)size - (ssize_t)m_header.size; - if (delta < 0) - LOG_ERROR_RETURN(ENOBUFS, delta, "insufficient buffer space (and allocation " - "failed), need ` more bytes", -delta); - - m_header.magic = 0; // clear the magic state - ssize_t ret = m_stream->readv(msg.iovec(), msg.iovcnt()); - if (ret < (ssize_t)m_header.size) - LOG_ERRNO_RETURN(0, -1, "failed to readv the message from stream ", m_stream); - - return m_header.size; - } - virtual uint64_t flags() override - { - return 0; - } - virtual uint64_t flags(uint64_t f) override - { - return f; - } - }; - IMessageChannel* new_messenger(IStream* stream) - { - return new Messenger(stream); - } -} diff --git a/common/stream-messenger/messenger.h b/common/stream-messenger/messenger.h deleted file mode 100644 index dbeffa87..00000000 --- a/common/stream-messenger/messenger.h +++ /dev/null @@ -1,41 +0,0 @@ -/* -Copyright 2022 The Photon Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -#pragma once -#include -#include -#include - -// a message channel based on any `IStream` object -namespace StreamMessenger -{ - // header of the stream message channel - struct Header - { - const static uint64_t MAGIC = 0x4962b4d24caa439e; - const static uint32_t VERSION = 0; - - uint64_t magic = MAGIC; // the header magic - uint32_t version = VERSION; // version of the message - uint32_t size; // size of the payload, not including the header - uint64_t reserved = 0; // padding to 24 bytes - }; - - // This class turns a stream into a message channel, by adding a `Header` to the - // message before send it, and extracting the header immediately after recv it. - // The stream can be any `IStream` object, like TCP socket or UNIX domain socket. - IMessageChannel* new_messenger(IStream* stream); -} diff --git a/common/stream-messenger/test/test.cpp b/common/stream-messenger/test/test.cpp deleted file mode 100644 index 03d563e8..00000000 --- a/common/stream-messenger/test/test.cpp +++ /dev/null @@ -1,101 +0,0 @@ -/* -Copyright 2022 The Photon Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -using namespace std; -using namespace photon; -using namespace StreamMessenger; - -char STR[] = "abcdefghijklmnopqrstuvwxyz"; - -void* echo_server(void* mc_) -{ - auto mc = (IMessageChannel*)mc_; - - IOVector msg; - size_t blksz = 2; - while(true) - { - auto ret = mc->recv(msg); - if (ret == 0) - { - LOG_DEBUG("zero-lengthed message recvd, quit"); - break; - } - if (ret < 0) - { - if (errno == ENOBUFS) - { - blksz *= 2; - msg.push_back(blksz); - LOG_DEBUG("adding ` bytes to the msg iovector", blksz); - continue; - } - - LOG_ERROR_RETURN(0, nullptr, "failed to recv msg"); - } - - auto ret2 = mc->send(msg); - EXPECT_EQ(ret2, ret); - } - LOG_DEBUG("exit"); - return nullptr; -} - -TEST(StreamMessenger, DISABLED_normalTest) -{ - auto ds = new_duplex_memory_stream(64); - DEFER(delete ds); - - auto mca = new_messenger(ds->endpoint_a); - DEFER(delete mca); - thread_create(&echo_server, mca); - - auto mcb = new_messenger(ds->endpoint_b); - DEFER(delete mcb); - - auto size = LEN(STR) - 1; - auto ret = mcb->send(STR, size); - EXPECT_EQ(ret, size); - LOG_DEBUG(ret, " bytes sent"); - char buf[size]; - auto ret2 = mcb->recv(buf, sizeof(buf)); - EXPECT_EQ(ret2, sizeof(buf)); - EXPECT_EQ(memcmp(buf, STR, size), 0); - LOG_DEBUG(ret, " bytes recvd and verified"); - - mcb->send(nullptr, 0); - thread_yield(); - LOG_DEBUG("exit"); -} - -int main(int argc, char **argv) -{ - log_output_level = ALOG_DEBUG; - ::testing::InitGoogleTest(&argc, argv); - photon::vcpu_init(); - DEFER(photon::vcpu_fini()); - auto ret = RUN_ALL_TESTS(); - return 0; -} diff --git a/common/test/CMakeLists.txt b/common/test/CMakeLists.txt index 09497252..27896b6f 100644 --- a/common/test/CMakeLists.txt +++ b/common/test/CMakeLists.txt @@ -21,9 +21,9 @@ add_executable(test-constexprstr test_constexprstr.cpp) target_link_libraries(test-constexprstr PRIVATE photon_shared) add_test(NAME test-constexprstr COMMAND $) -#add_executable(test-lockfree test_lockfree.cpp) -#target_link_libraries(test-lockfree PRIVATE photon_shared) -#add_test(NAME test-lockfree COMMAND $) +add_executable(test-lockfree test_lockfree.cpp) +target_link_libraries(test-lockfree PRIVATE photon_shared) +add_test(NAME test-lockfree COMMAND $) add_executable(test-alog test_alog.cpp x.cpp) target_link_libraries(test-alog PRIVATE photon_shared) diff --git a/common/test/perf_objcache.cpp b/common/test/perf_objcache.cpp index 5db884af..5832c245 100644 --- a/common/test/perf_objcache.cpp +++ b/common/test/perf_objcache.cpp @@ -9,6 +9,7 @@ #include #include #include +#include "../../test/ci-tools.h" #include "../expirecontainer.h" diff --git a/common/test/test.cpp b/common/test/test.cpp index 28aa6a22..96dec844 100644 --- a/common/test/test.cpp +++ b/common/test/test.cpp @@ -51,6 +51,9 @@ limitations under the License. #include #endif +#include "../../test/ci-tools.h" + + using namespace std; char str[] = "2018/01/05 21:53:28|DEBUG| 2423423|test.cpp:254|virtual void LOGPerf_1M_memcpy_Test::TestBody():aksdjfj 234:^%$#@341234 hahah `:jksld88423CACE::::::::::::::::::::::::::::::::::::::::::::::::::::"; @@ -1170,6 +1173,7 @@ TEST(update_now, after_idle_sleep) { // #endif int main(int argc, char **argv) { + if (!photon::is_using_default_engine()) return 0; photon::vcpu_init(); DEFER(photon::vcpu_fini()); char a[100]{}, b[100]{}; diff --git a/common/test/test_alog.cpp b/common/test/test_alog.cpp index ce5f3dd4..e723e446 100644 --- a/common/test/test_alog.cpp +++ b/common/test/test_alog.cpp @@ -27,6 +27,7 @@ limitations under the License. #include #include #include +#include "../../test/ci-tools.h" class LogOutputTest : public ILogOutput { public: @@ -575,6 +576,7 @@ TEST(ALOG, IPAddr) { int main(int argc, char **argv) { + if (!photon::is_using_default_engine()) return 0; photon::vcpu_init(); DEFER(photon::vcpu_fini()); ::testing::InitGoogleTest(&argc, argv); diff --git a/common/test/test_constexprstr.cpp b/common/test/test_constexprstr.cpp index 786c05f2..00875afd 100644 --- a/common/test/test_constexprstr.cpp +++ b/common/test/test_constexprstr.cpp @@ -22,6 +22,7 @@ limitations under the License. #include "../alog-stdstring.h" #include "../alog.h" #include "../conststr.h" +#include "../../test/ci-tools.h" DEFINE_ENUM_STR(VERBS, verbs, UNKNOW, DELETE, GET, HEAD, POST, PUT, CONNECT, OPTIONS, TRACE, COPY, LOCK, MKCOL, MOV, PROPFIND, PROPPATCH, @@ -113,6 +114,7 @@ TEST(TString, JoinAndSplit) { } int main(int argc, char** argv) { + if (!photon::is_using_default_engine()) return 0; ::testing::InitGoogleTest(&argc, argv); int ret = RUN_ALL_TESTS(); LOG_ERROR_RETURN(0, ret, VALUE(ret)); diff --git a/common/test/test_lockfree.cpp b/common/test/test_lockfree.cpp index a4977267..b60091bf 100644 --- a/common/test/test_lockfree.cpp +++ b/common/test/test_lockfree.cpp @@ -19,13 +19,14 @@ limitations under the License. #include #include -#include -#include -#include +// #include +// #include +// #include #include #include #include #include +#include "../../test/ci-tools.h" static constexpr size_t sender_num = 4; static constexpr size_t receiver_num = 4; @@ -46,8 +47,8 @@ LockfreeBatchMPMCRingQueue lbqueue; LockfreeSPSCRingQueue cqueue; std::mutex rlock, wlock; -boost::lockfree::queue> bqueue; -boost::lockfree::spsc_queue> squeue; +// boost::lockfree::queue> bqueue; +// boost::lockfree::spsc_queue> squeue; struct WithLock { template @@ -75,10 +76,7 @@ int test_queue(const char *name, QType &queue) { auto begin = std::chrono::steady_clock::now(); for (size_t i = 0; i < receiver_num; i++) { receivers.emplace_back([i, &queue] { - cpu_set_t cpuset; - CPU_ZERO(&cpuset); - CPU_SET(i, &cpuset); - pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset); + photon::set_cpu_affinity(i); std::chrono::nanoseconds rspent(std::chrono::nanoseconds(0)); for (size_t x = 0; x < items_num / receiver_num; x++) { int t; @@ -101,10 +99,7 @@ int test_queue(const char *name, QType &queue) { } for (size_t i = 0; i < sender_num; i++) { senders.emplace_back([i, &queue] { - cpu_set_t cpuset; - CPU_ZERO(&cpuset); - CPU_SET(i + receiver_num, &cpuset); - pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset); + photon::set_cpu_affinity(i); std::chrono::nanoseconds wspent{std::chrono::nanoseconds(0)}; for (size_t x = 0; x < items_num / sender_num; x++) { auto tm = std::chrono::high_resolution_clock::now(); @@ -150,10 +145,7 @@ int test_queue_batch(const char *name, QType &queue) { auto begin = std::chrono::steady_clock::now(); for (size_t i = 0; i < receiver_num; i++) { receivers.emplace_back([i, &queue] { - cpu_set_t cpuset; - CPU_ZERO(&cpuset); - CPU_SET(i, &cpuset); - pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset); + photon::set_cpu_affinity(i); int buffer[32]; size_t size; int amount = items_num / receiver_num; @@ -181,10 +173,7 @@ int test_queue_batch(const char *name, QType &queue) { } for (size_t i = 0; i < sender_num; i++) { senders.emplace_back([i, &queue] { - cpu_set_t cpuset; - CPU_ZERO(&cpuset); - CPU_SET(i + receiver_num, &cpuset); - pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset); + photon::set_cpu_affinity(i); std::chrono::nanoseconds wspent{std::chrono::nanoseconds(0)}; for (size_t x = 0; x < items_num / sender_num; x++) { auto tm = std::chrono::high_resolution_clock::now(); @@ -223,11 +212,12 @@ int test_queue_batch(const char *name, QType &queue) { } int main() { - test_queue("BoostQueue", bqueue); + if (!photon::is_using_default_engine()) return 0; + // test_queue("BoostQueue", bqueue); test_queue("PhotonLockfreeMPMCQueue", lqueue); test_queue("PhotonLockfreeBatchMPMCQueue", lbqueue); test_queue_batch("PhotonLockfreeBatchMPMCQueue+Batch", lbqueue); - test_queue("BoostSPSCQueue", squeue); + // test_queue("BoostSPSCQueue", squeue); test_queue("PhotonSPSCQueue", cqueue); test_queue_batch("PhotonSPSCQueue+Batch", cqueue); } diff --git a/common/test/test_objcache.cpp b/common/test/test_objcache.cpp index 28538607..683b6067 100644 --- a/common/test/test_objcache.cpp +++ b/common/test/test_objcache.cpp @@ -21,11 +21,11 @@ limitations under the License. #undef private #undef protected +#include #include #include #include - -#include +#include "../../test/ci-tools.h" static int thread_local release_cnt = 0; struct ShowOnDtor { @@ -68,8 +68,6 @@ void* objcache(void* arg) { } TEST(ObjectCache, release_cycle) { - // photon::vcpu_init(); - // DEFER(photon::vcpu_fini()); set_log_output_level(ALOG_INFO); DEFER(set_log_output_level(ALOG_DEBUG)); ObjectCache ocache(1000UL * 1000 * 10); @@ -93,8 +91,6 @@ TEST(ObjectCache, release_cycle) { TEST(ObjectCache, timeout_refresh) { release_cnt = 0; - // photon::vcpu_init(); - // DEFER(photon::vcpu_fini()); set_log_output_level(ALOG_INFO); DEFER(set_log_output_level(ALOG_DEBUG)); ObjectCache ocache(1000UL * 1000); @@ -129,8 +125,6 @@ void *ph_act(void *arg) { TEST(ObjectCache, ctor_may_yield_and_null) { release_cnt = 0; - // photon::vcpu_init(); - // DEFER(photon::vcpu_fini()); set_log_output_level(ALOG_INFO); DEFER(set_log_output_level(ALOG_DEBUG)); ObjectCache ocache(1000UL * 1000); @@ -149,8 +143,6 @@ TEST(ObjectCache, ctor_may_yield_and_null) { } TEST(ObjectCache, multithread) { - // photon::vcpu_init(); - // DEFER(photon::vcpu_fini()); set_log_output_level(ALOG_INFO); DEFER(set_log_output_level(ALOG_DEBUG)); ObjectCache ocache(1000UL * 1000 * 10); @@ -292,8 +284,6 @@ TEST(ObjectCache, borrow_with_once) { } TEST(ExpireContainer, expire_container) { - // photon::vcpu_init(); - // DEFER(photon::vcpu_fini()); char key[10] = "hello"; char key2[10] = "hello"; ExpireContainer expire(1000 * @@ -334,8 +324,6 @@ TEST(ExpireContainer, refresh) { } TEST(ExpireList, expire_container) { - // photon::vcpu_init(); - // DEFER(photon::vcpu_fini()); char key[10] = "hello"; ExpireList expire(1000 * 1000); // expire in 100ms expire.keep_alive(key, true); @@ -360,6 +348,7 @@ TEST(ExpireList, expire_container) { } int main(int argc, char** argv) { + if (!photon::is_using_default_engine()) return 0; photon::vcpu_init(); DEFER(photon::vcpu_fini()); ::testing::InitGoogleTest(&argc, argv); diff --git a/common/test/test_scalepool.cpp b/common/test/test_scalepool.cpp index 4e260435..b20c497c 100644 --- a/common/test/test_scalepool.cpp +++ b/common/test/test_scalepool.cpp @@ -22,9 +22,11 @@ limitations under the License. #undef protected #include "../alog.h" +#include #include #include -#include +#include "../../test/ci-tools.h" + TEST(IdentityPoolGC, basic) { auto pool = new_identity_pool(128); @@ -123,6 +125,7 @@ TEST(IOAlloc, basic) { } int main(int argc, char **argv) { + if (!photon::is_using_default_engine()) return 0; ::testing::InitGoogleTest(&argc, argv); photon::vcpu_init(); DEFER(photon::vcpu_fini()); diff --git a/common/test/test_throttle.cpp b/common/test/test_throttle.cpp index b8d0e6d5..c0a3b411 100644 --- a/common/test/test_throttle.cpp +++ b/common/test/test_throttle.cpp @@ -1,10 +1,10 @@ +#include #include #include #include #include #include - -#include +#include "../../test/ci-tools.h" TEST(Throttle, basic) { // baseline @@ -162,6 +162,7 @@ TEST(Throttle, try_consume) { } int main(int argc, char** argv) { + if (!photon::is_using_default_engine()) return 0; photon::init(0, 0); DEFER(photon::fini()); testing::InitGoogleTest(&argc, argv); diff --git a/fs/exportfs.cpp b/fs/exportfs.cpp index 9563ed02..d2bf16b7 100644 --- a/fs/exportfs.cpp +++ b/fs/exportfs.cpp @@ -561,6 +561,7 @@ namespace fs ExportBase::ref = 1; if (thread_pool_capacity != 0) ExportBase::pool = new_thread_pool(thread_pool_capacity); evloop->async_run(); + LOG_INFO(" OK"); return 0; } int exportfs_fini() diff --git a/fs/test/CMakeLists.txt b/fs/test/CMakeLists.txt index fb7421b1..373dc402 100644 --- a/fs/test/CMakeLists.txt +++ b/fs/test/CMakeLists.txt @@ -5,7 +5,7 @@ target_link_libraries(test-fs PRIVATE photon_shared) add_test(NAME test-fs COMMAND $) # add_executable(test-exportfs test_exportfs.cpp) -# target_link_libraries(test-exportfs PRIVATE photon_static) +# target_link_libraries(test-exportfs PRIVATE photon_shared) # add_test(NAME test-exportfs COMMAND $) add_executable(test-filecopy test_filecopy.cpp) diff --git a/fs/test/test.cpp b/fs/test/test.cpp index 8e7ab6ea..665a5553 100644 --- a/fs/test/test.cpp +++ b/fs/test/test.cpp @@ -1369,12 +1369,7 @@ TEST(Walker, basic) { int main(int argc, char **argv){ ::testing::InitGoogleTest(&argc, argv); -#ifdef __linux__ - int ev_engine = photon::INIT_EVENT_EPOLL; -#else - int ev_engine = photon::INIT_EVENT_KQUEUE; -#endif - if (photon::init(ev_engine, photon::INIT_IO_NONE)) + if (photon::init(photon::INIT_EVENT_DEFAULT, photon::INIT_IO_NONE)) return -1; DEFER(photon::fini()); int ret = RUN_ALL_TESTS(); diff --git a/fs/test/test_exportfs.cpp b/fs/test/test_exportfs.cpp index 5abfb8de..2c11abc7 100644 --- a/fs/test/test_exportfs.cpp +++ b/fs/test/test_exportfs.cpp @@ -29,17 +29,22 @@ limitations under the License. #include #include #include -#include +// #include using namespace photon; using namespace photon::fs; using namespace testing; -#ifdef __linux__ -static const int event_engine = photon::INIT_EVENT_EPOLL; -#else -static const int event_engine = photon::INIT_EVENT_KQUEUE; -#endif +inline void photon_init() { + init(INIT_EVENT_DEFAULT, INIT_IO_NONE); + exportfs_init(); +} + +inline void photon_fini() { + exportfs_fini(); + fini(); +} + constexpr uint64_t magic = 150820; static std::atomic work(0); @@ -99,14 +104,6 @@ int callbackvoid(void*, AsyncResult* ret) { } TEST(ExportFS, basic) { - photon::vcpu_init(); - photon::fd_events_init(event_engine); - exportfs_init(); - DEFER({ - exportfs_fini(); - photon::fd_events_fini(); - photon::vcpu_fini(); - }); PMock::MockNullFile* mockfile = new PMock::MockNullFile(); PMock::MockNullFileSystem* mockfs = new PMock::MockNullFileSystem(); PMock::MockNullDIR* mockdir = new PMock::MockNullDIR(); @@ -257,20 +254,19 @@ TEST(ExportFS, init_fini_failed_situation) { DEFER({ log_output = _o_output; }); - auto ret = exportfs_fini(); - EXPECT_EQ(-1, ret); - EXPECT_EQ(ENOSYS, errno); - photon::vcpu_init(); - photon::fd_events_init(event_engine); - ret = exportfs_init(); - EXPECT_EQ(0, ret); - ret = exportfs_init(); + + auto ret = exportfs_init(); EXPECT_EQ(-1, ret); EXPECT_EQ(EALREADY, errno); + ret = exportfs_fini(); EXPECT_EQ(0, ret); - photon::fd_events_fini(); - // photon::vcpu_fini(); + ret = exportfs_fini(); + EXPECT_EQ(-1, ret); + EXPECT_EQ(ENOSYS, errno); + + ret = exportfs_init(); + EXPECT_EQ(0, ret); } TEST(ExportFS, op_failed_situation) { @@ -279,14 +275,6 @@ TEST(ExportFS, op_failed_situation) { DEFER({ log_output = _o_output; }); - // photon::vcpu_init(); - photon::fd_events_init(event_engine); - exportfs_init(); - DEFER({ - exportfs_fini(); - photon::fd_events_fini(); - // photon::vcpu_fini(); - }); PMock::MockNullFile* mockfile = new PMock::MockNullFile; errno = 0; EXPECT_CALL(*mockfile, read(_, _)) @@ -308,14 +296,6 @@ TEST(ExportFS, op_failed_situation) { } TEST(ExportFS, xattr) { - photon::vcpu_init(); - photon::fd_events_init(event_engine); - exportfs_init(); - DEFER({ - exportfs_fini(); - photon::fd_events_fini(); - photon::vcpu_fini(); - }); PMock::MockNullFile* mockfile = new PMock::MockNullFile(); PMock::MockNullFileSystem* mockfs = new PMock::MockNullFileSystem(); auto file = dynamic_cast(export_as_async_file(mockfile)); @@ -376,14 +356,8 @@ TEST(ExportFS, xattr_sync) { IFileSystemXAttr* fs = nullptr; std::thread th([&]{ - photon::vcpu_init(); - photon::fd_events_init(event_engine); - exportfs_init(); - DEFER({ - exportfs_fini(); - photon::fd_events_fini(); - photon::vcpu_fini(); - }); + photon_init(); + DEFER(photon_fini()); file = dynamic_cast(export_as_sync_file(mockfile)); fs = dynamic_cast(export_as_sync_fs(mockfs)); sem.wait(1); @@ -430,9 +404,10 @@ TEST(ExportFS, xattr_sync) { int main(int argc, char **argv) { - photon::vcpu_init(); - DEFER(photon::vcpu_fini()); + photon_init(); + DEFER(photon_fini()); ::testing::InitGoogleTest(&argc, argv); int ret = RUN_ALL_TESTS(); LOG_ERROR_RETURN(0, ret, VALUE(ret)); + return ret; } diff --git a/fs/test/test_throttledfile.cpp b/fs/test/test_throttledfile.cpp index e0839266..489d09d4 100644 --- a/fs/test/test_throttledfile.cpp +++ b/fs/test/test_throttledfile.cpp @@ -35,6 +35,7 @@ limitations under the License. #include #include #include +#include "../../test/ci-tools.h" #include "mock.h" @@ -433,6 +434,7 @@ TEST(ThrottledFs, concurrent){ int main(int argc, char **argv) { + if (!photon::is_using_default_engine()) return 0; ::testing::InitGoogleTest(&argc, argv); int ret = RUN_ALL_TESTS(); LOG_ERROR_RETURN(0, ret, VALUE(ret)); diff --git a/io/fd-events.h b/io/fd-events.h index d201a5be..71fa2e4f 100644 --- a/io/fd-events.h +++ b/io/fd-events.h @@ -138,7 +138,7 @@ DECLARE_MASTER_AND_CASCADING_ENGINE(select); DECLARE_MASTER_AND_CASCADING_ENGINE(iouring); DECLARE_MASTER_AND_CASCADING_ENGINE(kqueue); -inline int fd_events_init(int master_engine) { +inline int fd_events_init(uint64_t master_engine) { switch (master_engine) { #ifdef __linux__ case INIT_EVENT_EPOLL: diff --git a/io/test/CMakeLists.txt b/io/test/CMakeLists.txt index de102709..00a08fe8 100644 --- a/io/test/CMakeLists.txt +++ b/io/test/CMakeLists.txt @@ -21,6 +21,6 @@ target_link_libraries(test-syncio PRIVATE photon_shared) add_test(NAME test-syncio COMMAND $) add_executable(test-iouring test-iouring.cpp) -target_link_libraries(test-iouring PRIVATE photon_static) +target_link_libraries(test-iouring PRIVATE photon_shared) add_test(NAME test-iouring COMMAND $) endif () \ No newline at end of file diff --git a/io/test/test-iouring.cpp b/io/test/test-iouring.cpp index 464bb088..9ab8c2d7 100644 --- a/io/test/test-iouring.cpp +++ b/io/test/test-iouring.cpp @@ -342,7 +342,8 @@ TEST(perf, DISABLED_read) { class event_engine : public testing::Test { protected: void SetUp() override { - GTEST_ASSERT_EQ(0, photon::init(ci_ev_engine, photon::INIT_IO_NONE)); + GTEST_ASSERT_EQ(0, photon::init(photon::INIT_EVENT_DEFAULT, + photon::INIT_IO_NONE)); #ifdef PHOTON_URING engine = (ci_ev_engine == photon::INIT_EVENT_EPOLL) ? photon::new_epoll_cascading_engine() : photon::new_iouring_cascading_engine(); @@ -621,6 +622,5 @@ int main(int argc, char** arg) { testing::InitGoogleTest(&argc, arg); testing::FLAGS_gtest_break_on_failure = true; gflags::ParseCommandLineFlags(&argc, &arg, true); - ci_parse_env(); return RUN_ALL_TESTS(); } diff --git a/net/http/test/CMakeLists.txt b/net/http/test/CMakeLists.txt index 582c4269..f60e99ba 100644 --- a/net/http/test/CMakeLists.txt +++ b/net/http/test/CMakeLists.txt @@ -1,13 +1,13 @@ add_definitions(-w) add_executable(client_perf client_perf.cpp) -target_link_libraries(client_perf PRIVATE photon_static) +target_link_libraries(client_perf PRIVATE photon_shared) add_executable(server_perf server_perf.cpp) target_link_libraries(server_perf PRIVATE photon_shared) add_executable(pure_libcurl pure_libcurl.cpp) -target_link_libraries(pure_libcurl PRIVATE photon_static) +target_link_libraries(pure_libcurl PRIVATE photon_shared) add_executable(client_function_test client_function_test.cpp) target_link_libraries(client_function_test PRIVATE photon_shared) diff --git a/net/test/CMakeLists.txt b/net/test/CMakeLists.txt index 1d10876d..d4cbbc96 100644 --- a/net/test/CMakeLists.txt +++ b/net/test/CMakeLists.txt @@ -13,7 +13,7 @@ target_link_libraries(test-sockpool PRIVATE photon_shared) add_test(NAME test-sockpool COMMAND $) add_executable(test-curl test_curl.cpp) -target_link_libraries(test-curl PRIVATE photon_static) +target_link_libraries(test-curl PRIVATE photon_shared) add_test(NAME test-curl COMMAND $) add_executable(test-server test-server.cpp) diff --git a/photon.cpp b/photon.cpp index 8780969b..3d404eac 100644 --- a/photon.cpp +++ b/photon.cpp @@ -15,6 +15,7 @@ limitations under the License. */ #include "photon.h" +#include #include "io/fd-events.h" #include "io/signal.h" @@ -38,21 +39,32 @@ static thread_local uint64_t g_event_engine = 0, g_io_engine = 0; #define INIT_IO(name, prefix) if (INIT_IO_##name & io_engine) { if (prefix##_init() < 0) return -1; } #define FINI_IO(name, prefix) if (INIT_IO_##name & g_io_engine) { prefix##_fini(); } +class Shift { + uint8_t _n; +public: + constexpr Shift(uint64_t x) : _n(__builtin_ctz(x)) { } + operator uint64_t() { return 1UL << _n; } +}; + // Try to init master engine with the recommended order +static const Shift recommended_order[] = { #if defined(__linux__) -static const int recommended_order[] = {INIT_EVENT_EPOLL, INIT_EVENT_IOURING, INIT_EVENT_SELECT}; + INIT_EVENT_EPOLL, INIT_EVENT_IOURING, INIT_EVENT_SELECT}; #else // macOS, FreeBSD ... -static const int recommended_order[] = {INIT_EVENT_KQUEUE, INIT_EVENT_SELECT}; + INIT_EVENT_KQUEUE, INIT_EVENT_SELECT}; #endif -int init(uint64_t event_engine, uint64_t io_engine) { +int __photon_init(uint64_t event_engine, uint64_t io_engine) { if (vcpu_init() < 0) return -1; - if (event_engine != INIT_EVENT_NONE) { + const uint64_t ALL_ENGINES = INIT_EVENT_EPOLL | + INIT_EVENT_IOURING | INIT_EVENT_KQUEUE | + INIT_EVENT_SELECT | INIT_EVENT_IOCP; + if (event_engine & ALL_ENGINES) { bool ok = false; - for (auto each : recommended_order) { - if ((each & event_engine) && fd_events_init(each) == 0) { + for (auto x : recommended_order) { + if ((x & event_engine) && fd_events_init(x) == 0) { ok = true; break; } @@ -84,6 +96,10 @@ int init(uint64_t event_engine, uint64_t io_engine) { return 0; } +int init(uint64_t event_engine, uint64_t io_engine) { + return __photon_init(event_engine, io_engine); +} + int fini() { #ifdef __linux__ FINI_IO(LIBAIO, libaio_wrapper) diff --git a/rpc/test/test.cpp b/rpc/test/test.cpp index 29f85bda..854f6495 100644 --- a/rpc/test/test.cpp +++ b/rpc/test/test.cpp @@ -33,7 +33,7 @@ using namespace rpc; class RpcTest : public testing::Test { public: void SetUp() override { - GTEST_ASSERT_EQ(0, photon::init(ci_ev_engine, photon::INIT_IO_NONE)); + GTEST_ASSERT_EQ(0, photon::init(photon::INIT_EVENT_DEFAULT, photon::INIT_IO_NONE)); } void TearDown() override { photon::fini(); @@ -481,7 +481,6 @@ TEST_F(RpcTest, passive_shutdown) { int main(int argc, char** arg) { - ci_parse_env(); ::testing::InitGoogleTest(&argc, arg); return RUN_ALL_TESTS(); } diff --git a/test/ci-tools.cpp b/test/ci-tools.cpp index 265b4ac3..7243fd25 100644 --- a/test/ci-tools.cpp +++ b/test/ci-tools.cpp @@ -1,26 +1,100 @@ -#include "ci-tools.h" +// #include "ci-tools.h" #include #include -#include "../photon.h" +#include +#include +#include +#include +#include +#include -#if __linux__ -uint64_t ci_ev_engine = photon::INIT_EVENT_EPOLL; -#else -uint64_t ci_ev_engine = photon::INIT_EVENT_KQUEUE; -#endif -uint64_t ci_ev_engine_with_signal = ci_ev_engine | photon::INIT_EVENT_SIGNAL; +namespace photon { -void ci_parse_env() { - const char* ev_engine = getenv("PHOTON_CI_EV_ENGINE"); - if (!ev_engine) +static uint64_t _engine = 0; +static const char* _engine_name = nullptr; + +__attribute__((constructor)) +static void parse_env_eng() { + _engine_name = getenv("PHOTON_CI_EV_ENGINE"); + if (!_engine_name) return; - if (strcmp(ev_engine, "epoll") == 0) { - ci_ev_engine = photon::INIT_EVENT_EPOLL; - } else if (strcmp(ev_engine, "io_uring") == 0) { - ci_ev_engine = photon::INIT_EVENT_IOURING; - } else if (strcmp(ev_engine, "kqueue") == 0) { - ci_ev_engine = photon::INIT_EVENT_KQUEUE; + if (strcmp(_engine_name, "epoll") == 0) { + _engine = photon::INIT_EVENT_EPOLL; + } else if (strcmp(_engine_name, "io_uring") == 0) { + _engine = photon::INIT_EVENT_IOURING; + } else if (strcmp(_engine_name, "kqueue") == 0) { + _engine = photon::INIT_EVENT_KQUEUE; + } else { + printf("invalid event engine: %s\n", _engine_name); + _engine_name = nullptr; + exit(-1); + } +} + +static estring_view get_engine_name(uint64_t eng) { + switch(eng) { + case INIT_EVENT_EPOLL: return "epoll"; + case INIT_EVENT_IOURING: return "io_uring"; + case INIT_EVENT_KQUEUE: return "kqueue"; + case INIT_EVENT_SELECT: return "select"; + case INIT_EVENT_IOCP: return "iocp"; + } + return {}; +} + +static estring get_engine_names(uint64_t engs) { + estring names; + if (!engs) { + names = "none"; + } else { + for (uint64_t i = 0; i < 64; ++i) { + auto name = get_engine_name(engs & (1UL << i)); + if (name.size()) names.appends(name, ", "); + } + if (names.size() > 2) + names.resize(names.size() - 2); + } + return names; +} + +int __photon_init(uint64_t event_engine, uint64_t io_engine); + +// this function is supposed to be linked statically to test +// cases, so as to override the real one in libphoton.so/dylib +int init(uint64_t event_engine, uint64_t io_engine) { + LOG_INFO("enter CI overriden photon::init()"); + const uint64_t all_engines = INIT_EVENT_EPOLL | + INIT_EVENT_IOURING | INIT_EVENT_KQUEUE | + INIT_EVENT_SELECT | INIT_EVENT_IOCP; + auto arg_specified = event_engine & all_engines; + LOG_INFO("argument specified: ` (`)", get_engine_names(arg_specified), arg_specified); + LOG_INFO("environment specified: '`'", _engine_name); + if (_engine && arg_specified) { + event_engine &= ~all_engines; + event_engine |= _engine; + LOG_INFO("event engine overridden to: ", get_engine_names(event_engine & all_engines)); } - ci_ev_engine_with_signal = ci_ev_engine | photon::INIT_EVENT_SIGNAL; -} \ No newline at end of file + return __photon_init(event_engine, io_engine); +} + +bool is_using_default_engine() { +#ifdef __linux__ + const uint64_t default_eng = INIT_EVENT_EPOLL; +#else + const uint64_t default_eng = INIT_EVENT_KQUEUE; +#endif + return !_engine || _engine == default_eng; +} + +void set_cpu_affinity(int i) { +#ifdef __linux__ + cpu_set_t cpuset; + CPU_ZERO(&cpuset); + CPU_SET(i, &cpuset); + pthread_setaffinity_np(pthread_self(), + sizeof(cpu_set_t), &cpuset); +#endif +} + +} diff --git a/test/ci-tools.h b/test/ci-tools.h index f0cbedd8..f3a0af30 100644 --- a/test/ci-tools.h +++ b/test/ci-tools.h @@ -1,7 +1,9 @@ #pragma once -#include -extern uint64_t ci_ev_engine; -extern uint64_t ci_ev_engine_with_signal; +namespace photon { -void ci_parse_env(); \ No newline at end of file +bool is_using_default_engine(); + +void set_cpu_affinity(int i); + +} diff --git a/thread/test/CMakeLists.txt b/thread/test/CMakeLists.txt index 44634a93..0dff72ff 100644 --- a/thread/test/CMakeLists.txt +++ b/thread/test/CMakeLists.txt @@ -5,9 +5,13 @@ target_link_libraries(perf_usleepdefer_semaphore PRIVATE photon_shared) add_test(NAME perf_usleepdefer_semaphore COMMAND $) add_executable(test-thread test.cpp x.cpp) -target_link_libraries(test-thread PRIVATE photon_static) +target_link_libraries(test-thread PRIVATE photon_shared) add_test(NAME test-thread COMMAND $) +add_executable(test-pool test-pool.cpp x.cpp) +target_link_libraries(test-pool PRIVATE photon_shared) +add_test(NAME test-pool COMMAND $) + add_executable(test-std-compat test-std-compat.cpp) target_link_libraries(test-std-compat PRIVATE photon_shared) add_test(NAME test-std-compat COMMAND $) @@ -33,5 +37,5 @@ target_link_libraries(test-lib-data PRIVATE photon_shared) add_test(NAME test-lib-data COMMAND $) add_executable(test-multi-vcpu-locking test-multi-vcpu-locking.cpp) -target_link_libraries(test-multi-vcpu-locking PRIVATE photon_static) +target_link_libraries(test-multi-vcpu-locking PRIVATE photon_shared) add_test(NAME test-multi-vcpu-locking COMMAND $) \ No newline at end of file diff --git a/thread/test/test-multi-vcpu-locking.cpp b/thread/test/test-multi-vcpu-locking.cpp index aefafe0c..bfd50ddf 100644 --- a/thread/test/test-multi-vcpu-locking.cpp +++ b/thread/test/test-multi-vcpu-locking.cpp @@ -60,7 +60,7 @@ static void myThread(int tid) { } TEST(multi_vcpu_locking, long_time_acquisition_should_abort) { - int ret = photon::init(ci_ev_engine, photon::INIT_IO_NONE); + int ret = photon::init(photon::INIT_EVENT_DEFAULT, photon::INIT_IO_NONE); GTEST_ASSERT_EQ(0, ret); DEFER(photon::fini()); @@ -90,6 +90,5 @@ TEST(multi_vcpu_locking, long_time_acquisition_should_abort) { int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); - ci_parse_env(); return RUN_ALL_TESTS(); } \ No newline at end of file diff --git a/thread/test/test-pool.cpp b/thread/test/test-pool.cpp new file mode 100644 index 00000000..2472592f --- /dev/null +++ b/thread/test/test-pool.cpp @@ -0,0 +1,297 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../test/ci-tools.h" + +DEFINE_int32(ths_total, 100, "total threads when testing threadpool."); + +using namespace std; +using namespace photon; + +void *func1(void *) +{ + thread_sleep(rand()%5); + return nullptr; +} + +TEST(ThreadPool, test) +{ + ThreadPool<64> pool(64*1024); + vector ths; + ths.resize(FLAGS_ths_total); + for (int i = 0; ith); + pool.join(ths[i]); + } + // LOG_INFO("???????????????"); +} + +TEST(ThreadPool, migrate) { + WorkPool wp(4, 0, 0, -1); + ThreadPool<64> pool(64 * 1024); + vector ths; + ths.resize(FLAGS_ths_total); + for (int i = 0; i < FLAGS_ths_total; i++) { + ths[i] = pool.thread_create_ex(&::func1, nullptr, true); + wp.thread_migrate(ths[i]->th); + } + LOG_INFO("----------"); + for (int i = 0; i < FLAGS_ths_total; i++) { + LOG_DEBUG("wait thread: `", ths[i]->th); + pool.join(ths[i]); + } + LOG_INFO("???????????????"); +} + +TEST(ThreadPool, multithread) { + WorkPool wp(4, 0, 0, -1); + ThreadPool<64> pool(64 * 1024); + vector ths; + ths.resize(FLAGS_ths_total); + for (int i = 0; i < FLAGS_ths_total; i++) { + wp.call( + [&] { ths[i] = pool.thread_create_ex(&::func1, nullptr, true); }); + } + for (int i = 0; i < FLAGS_ths_total; i++) { + wp.call([&] { + pool.join(ths[i]); + }); + } +} + +int jobwork(WorkPool* pool, int i) { + LOG_INFO("LAUNCH"); + int ret = 0; + pool->call( + [&ret](int i) { + LOG_INFO("START"); + this_thread::sleep_for(std::chrono::seconds(1)); + LOG_INFO("FINISH"); + ret = i; + }, + i); + LOG_INFO("RETURN"); + EXPECT_EQ(i, ret); + return 0; +} + +TEST(workpool, work) { + std::unique_ptr pool(new WorkPool(2)); + + std::vector jhs; + auto start = std::chrono::system_clock::now(); + for (int i = 0; i < 4; i++) { + jhs.emplace_back(thread_enable_join( + thread_create11(&jobwork, pool.get(), i))); + } + for (auto& j : jhs) { + thread_join(j); + } + auto duration = std::chrono::system_clock::now() - start; + EXPECT_GE(duration, std::chrono::seconds(2)); + EXPECT_LE(duration, std::chrono::seconds(3)); +} + +TEST(workpool, async_work_capture) { + std::unique_ptr pool(new WorkPool(2, 0, 0, 0)); + + semaphore sem; + int flag[10] = {0}; + auto start = std::chrono::system_clock::now(); + for (int i = 0; i < 10; i++) { + pool->async_call(new auto([&sem, i, &flag]{ + EXPECT_FALSE(flag[i]); + flag[i] = true; + auto x = i; + LOG_INFO(x); + thread_usleep(2000 * 1000); + EXPECT_EQ(x, i); + EXPECT_TRUE(flag[i]); + sem.signal(1); + })); + } + auto duration = std::chrono::system_clock::now() - start; + EXPECT_GE(duration, std::chrono::seconds(0)); + EXPECT_LE(duration, std::chrono::seconds(1)); + sem.wait(10); + LOG_INFO("DONE"); +} + +struct CopyMoveRecord{ + size_t copy = 0; + size_t move = 0; + CopyMoveRecord() { + } + ~CopyMoveRecord() { + } + CopyMoveRecord(const CopyMoveRecord& rhs) { + copy = rhs.copy + 1; + move = rhs.move; + LOG_INFO("COPY ", this); + } + CopyMoveRecord(CopyMoveRecord&& rhs) { + copy = rhs.copy; + move = rhs.move + 1; + LOG_INFO("MOVE ", this); + } + CopyMoveRecord& operator=(const CopyMoveRecord& rhs) { + copy = rhs.copy + 1; + move = rhs.move; + LOG_INFO("COPY ASSIGN ", this); + return *this; + } + CopyMoveRecord& operator=(CopyMoveRecord&& rhs) { + copy = rhs.copy; + move = rhs.move + 1; + LOG_INFO("MOVE ASSIGN", this); + return *this; + } +}; + +TEST(workpool, async_work_lambda) { + std::unique_ptr pool(new WorkPool(2)); + + std::vector jhs; + auto start = std::chrono::system_clock::now(); + for (int i = 0; i < 4; i++) { + CopyMoveRecord *r = new CopyMoveRecord(); + pool->async_call( + new auto ([i, r]() { + LOG_INFO("START ", VALUE(__cplusplus), VALUE(r->copy), + VALUE(r->move)); + EXPECT_EQ(0, r->copy); + this_thread::sleep_for(std::chrono::seconds(1)); + LOG_INFO("FINISH"); + delete r; + })); + } + auto duration = std::chrono::system_clock::now() - start; + EXPECT_GE(duration, std::chrono::seconds(0)); + EXPECT_LE(duration, std::chrono::seconds(1)); + thread_sleep(3); + LOG_INFO("DONE"); +} + + +TEST(workpool, async_work_lambda_threadcreate) { + std::unique_ptr pool(new WorkPool(1, 0, 0, 0)); + + std::vector jhs; + auto start = std::chrono::system_clock::now(); + semaphore sem; + for (int i = 0; i < 4; i++) { + CopyMoveRecord *r = new CopyMoveRecord(); + pool->async_call( + new auto ([&sem, i, r]() { + LOG_INFO("START ", VALUE(__cplusplus), VALUE(r->copy), + VALUE(r->move)); + EXPECT_EQ(0, r->copy); + thread_sleep(1); + sem.signal(1); + LOG_INFO("FINISH"); + delete r; + })); + } + auto duration = std::chrono::system_clock::now() - start; + EXPECT_GE(duration, std::chrono::seconds(0)); + EXPECT_LE(duration, std::chrono::seconds(1)); + sem.wait(4); + duration = std::chrono::system_clock::now() - start; + EXPECT_GE(duration, std::chrono::seconds(1)); + EXPECT_LE(duration, std::chrono::seconds(3)); + LOG_INFO("DONE"); +} + +TEST(workpool, async_work_lambda_threadpool) { + std::unique_ptr pool(new WorkPool(1, 0, 0, 4)); + + std::vector jhs; + auto start = std::chrono::system_clock::now(); + semaphore sem; + for (int i = 0; i < 4; i++) { + CopyMoveRecord *r = new CopyMoveRecord(); + pool->async_call( + new auto ([&sem, i, r]() { + LOG_INFO("START ", VALUE(__cplusplus), VALUE(r->copy), + VALUE(r->move)); + EXPECT_EQ(0, r->copy); + thread_sleep(1); + sem.signal(1); + LOG_INFO("FINISH"); + delete r; + })); + } + auto duration = std::chrono::system_clock::now() - start; + EXPECT_GE(duration, std::chrono::seconds(0)); + EXPECT_LE(duration, std::chrono::seconds(1)); + sem.wait(4); + duration = std::chrono::system_clock::now() - start; + LOG_INFO(VALUE(duration.count())); + EXPECT_GE(duration, std::chrono::seconds(1)); + EXPECT_LE(duration, std::chrono::seconds(2)); + LOG_INFO("DONE"); +} + +TEST(workpool, async_work_lambda_threadpool_append) { + std::unique_ptr pool(new WorkPool(0, 0, 0, 0)); + + for (int i=0;i<4;i++) { + std::thread([&]{ + vcpu_init(); + DEFER(vcpu_fini()); + pool->join_current_vcpu_into_workpool(); + }).detach(); + } + + std::vector jhs; + auto start = std::chrono::system_clock::now(); + semaphore sem; + for (int i = 0; i < 4; i++) { + CopyMoveRecord *r = new CopyMoveRecord(); + pool->async_call( + new auto ([&sem, i, r]() { + LOG_INFO("START ", VALUE(__cplusplus), VALUE(r->copy), + VALUE(r->move)); + EXPECT_EQ(0, r->copy); + thread_sleep(1); + sem.signal(1); + LOG_INFO("FINISH"); + delete r; + })); + } + auto duration = std::chrono::system_clock::now() - start; + EXPECT_GE(duration, std::chrono::seconds(0)); + EXPECT_LE(duration, std::chrono::seconds(1)); + sem.wait(4); + duration = std::chrono::system_clock::now() - start; + EXPECT_GE(duration, std::chrono::seconds(1)); + EXPECT_LE(duration, std::chrono::seconds(2)); + LOG_INFO("DONE"); +} + +int main(int argc, char** arg) +{ + if (!is_using_default_engine()) return 0; + ::testing::InitGoogleTest(&argc, arg); + gflags::ParseCommandLineFlags(&argc, &arg, true); + default_audit_logger.log_output = log_output_stdout; + vcpu_init(); + DEFER(vcpu_fini()); + set_log_output_level(ALOG_DEBUG); + return RUN_ALL_TESTS(); +} + diff --git a/thread/test/test.cpp b/thread/test/test.cpp index 4fd2a171..5485b82f 100644 --- a/thread/test/test.cpp +++ b/thread/test/test.cpp @@ -24,19 +24,17 @@ limitations under the License. #include #include #include +#include #define private public - #include "../thread.cpp" #include "../thread11.h" -#include "../thread-pool.h" -#include "../workerpool.h" -#include +#include "../../test/ci-tools.h" + using namespace std; using namespace photon; -DEFINE_int32(ths_total, 100, "total threads when testing threadpool."); DEFINE_int32(vcpus, 1, "total # of vCPUs"); thread_local semaphore aSem(4); @@ -709,60 +707,6 @@ TEST(Sleep, sleep_only_thread) { //Sleep_sleep_only_thread_Test::TestBody EXPECT_LE(dt, 3UL*1024*1024); } -void *func1(void *) -{ - photon::thread_sleep(rand()%5); - return nullptr; -} - -TEST(ThreadPool, test) -{ - ThreadPool<64> pool(64*1024); - vector ths; - ths.resize(FLAGS_ths_total); - for (int i = 0; ith); - pool.join(ths[i]); - } - // LOG_INFO("???????????????"); -} - -TEST(ThreadPool, migrate) { - WorkPool wp(4, 0, 0, -1); - ThreadPool<64> pool(64 * 1024); - vector ths; - ths.resize(FLAGS_ths_total); - for (int i = 0; i < FLAGS_ths_total; i++) { - ths[i] = pool.thread_create_ex(&::func1, nullptr, true); - wp.thread_migrate(ths[i]->th); - } - LOG_INFO("----------"); - for (int i = 0; i < FLAGS_ths_total; i++) { - LOG_DEBUG("wait thread: `", ths[i]->th); - pool.join(ths[i]); - } - LOG_INFO("???????????????"); -} - -TEST(ThreadPool, multithread) { - WorkPool wp(4, 0, 0, -1); - ThreadPool<64> pool(64 * 1024); - vector ths; - ths.resize(FLAGS_ths_total); - for (int i = 0; i < FLAGS_ths_total; i++) { - wp.call( - [&] { ths[i] = pool.thread_create_ex(&::func1, nullptr, true); }); - } - for (int i = 0; i < FLAGS_ths_total; i++) { - wp.call([&] { - pool.join(ths[i]); - }); - } -} - thread_local uint64_t rw_count; thread_local bool writing = false; thread_local photon::rwlock rwl; @@ -862,7 +806,7 @@ void run_all_tests(uint32_t i) RUN_TEST(thread11, test); RUN_TEST(Semaphore, basic); RUN_TEST(Semaphore, heavy); - RUN_TEST(ThreadPool, test); + // RUN_TEST(ThreadPool, test); RUN_TEST(RWLock, checklock); RUN_TEST(RWLock, interrupt); wait_for_completion(i); @@ -1250,216 +1194,6 @@ TEST(mutex, timeout_is_zero) { LOG_INFO("Meet ` lock timeout, all work finished", cnt.load()); } -int jobwork(WorkPool* pool, int i) { - LOG_INFO("LAUNCH"); - int ret = 0; - pool->call( - [&ret](int i) { - LOG_INFO("START"); - this_thread::sleep_for(std::chrono::seconds(1)); - LOG_INFO("FINISH"); - ret = i; - }, - i); - LOG_INFO("RETURN"); - EXPECT_EQ(i, ret); - return 0; -} - -TEST(workpool, work) { - std::unique_ptr pool(new WorkPool(2)); - - std::vector jhs; - auto start = std::chrono::system_clock::now(); - for (int i = 0; i < 4; i++) { - jhs.emplace_back(photon::thread_enable_join( - photon::thread_create11(&jobwork, pool.get(), i))); - } - for (auto& j : jhs) { - photon::thread_join(j); - } - auto duration = std::chrono::system_clock::now() - start; - EXPECT_GE(duration, std::chrono::seconds(2)); - EXPECT_LE(duration, std::chrono::seconds(3)); -} - -TEST(workpool, async_work_capture) { - std::unique_ptr pool(new WorkPool(2, 0, 0, 0)); - - photon::semaphore sem; - int flag[10] = {0}; - auto start = std::chrono::system_clock::now(); - for (int i = 0; i < 10; i++) { - pool->async_call(new auto([&sem, i, &flag]{ - EXPECT_FALSE(flag[i]); - flag[i] = true; - auto x = i; - LOG_INFO(x); - photon::thread_usleep(2000 * 1000); - EXPECT_EQ(x, i); - EXPECT_TRUE(flag[i]); - sem.signal(1); - })); - } - auto duration = std::chrono::system_clock::now() - start; - EXPECT_GE(duration, std::chrono::seconds(0)); - EXPECT_LE(duration, std::chrono::seconds(1)); - sem.wait(10); - LOG_INFO("DONE"); -} - -struct CopyMoveRecord{ - size_t copy = 0; - size_t move = 0; - CopyMoveRecord() { - } - ~CopyMoveRecord() { - } - CopyMoveRecord(const CopyMoveRecord& rhs) { - copy = rhs.copy + 1; - move = rhs.move; - LOG_INFO("COPY ", this); - } - CopyMoveRecord(CopyMoveRecord&& rhs) { - copy = rhs.copy; - move = rhs.move + 1; - LOG_INFO("MOVE ", this); - } - CopyMoveRecord& operator=(const CopyMoveRecord& rhs) { - copy = rhs.copy + 1; - move = rhs.move; - LOG_INFO("COPY ASSIGN ", this); - return *this; - } - CopyMoveRecord& operator=(CopyMoveRecord&& rhs) { - copy = rhs.copy; - move = rhs.move + 1; - LOG_INFO("MOVE ASSIGN", this); - return *this; - } -}; - -TEST(workpool, async_work_lambda) { - std::unique_ptr pool(new WorkPool(2)); - - std::vector jhs; - auto start = std::chrono::system_clock::now(); - for (int i = 0; i < 4; i++) { - CopyMoveRecord *r = new CopyMoveRecord(); - pool->async_call( - new auto ([i, r]() { - LOG_INFO("START ", VALUE(__cplusplus), VALUE(r->copy), - VALUE(r->move)); - EXPECT_EQ(0, r->copy); - this_thread::sleep_for(std::chrono::seconds(1)); - LOG_INFO("FINISH"); - delete r; - })); - } - auto duration = std::chrono::system_clock::now() - start; - EXPECT_GE(duration, std::chrono::seconds(0)); - EXPECT_LE(duration, std::chrono::seconds(1)); - photon::thread_sleep(3); - LOG_INFO("DONE"); -} - - -TEST(workpool, async_work_lambda_threadcreate) { - std::unique_ptr pool(new WorkPool(1, 0, 0, 0)); - - std::vector jhs; - auto start = std::chrono::system_clock::now(); - photon::semaphore sem; - for (int i = 0; i < 4; i++) { - CopyMoveRecord *r = new CopyMoveRecord(); - pool->async_call( - new auto ([&sem, i, r]() { - LOG_INFO("START ", VALUE(__cplusplus), VALUE(r->copy), - VALUE(r->move)); - EXPECT_EQ(0, r->copy); - photon::thread_sleep(1); - sem.signal(1); - LOG_INFO("FINISH"); - delete r; - })); - } - auto duration = std::chrono::system_clock::now() - start; - EXPECT_GE(duration, std::chrono::seconds(0)); - EXPECT_LE(duration, std::chrono::seconds(1)); - sem.wait(4); - duration = std::chrono::system_clock::now() - start; - EXPECT_GE(duration, std::chrono::seconds(1)); - EXPECT_LE(duration, std::chrono::seconds(3)); - LOG_INFO("DONE"); -} - -TEST(workpool, async_work_lambda_threadpool) { - std::unique_ptr pool(new WorkPool(1, 0, 0, 4)); - - std::vector jhs; - auto start = std::chrono::system_clock::now(); - photon::semaphore sem; - for (int i = 0; i < 4; i++) { - CopyMoveRecord *r = new CopyMoveRecord(); - pool->async_call( - new auto ([&sem, i, r]() { - LOG_INFO("START ", VALUE(__cplusplus), VALUE(r->copy), - VALUE(r->move)); - EXPECT_EQ(0, r->copy); - photon::thread_sleep(1); - sem.signal(1); - LOG_INFO("FINISH"); - delete r; - })); - } - auto duration = std::chrono::system_clock::now() - start; - EXPECT_GE(duration, std::chrono::seconds(0)); - EXPECT_LE(duration, std::chrono::seconds(1)); - sem.wait(4); - duration = std::chrono::system_clock::now() - start; - LOG_INFO(VALUE(duration.count())); - EXPECT_GE(duration, std::chrono::seconds(1)); - EXPECT_LE(duration, std::chrono::seconds(2)); - LOG_INFO("DONE"); -} - -TEST(workpool, async_work_lambda_threadpool_append) { - std::unique_ptr pool(new WorkPool(0, 0, 0, 0)); - - for (int i=0;i<4;i++) { - std::thread([&]{ - photon::vcpu_init(); - DEFER(photon::vcpu_fini()); - pool->join_current_vcpu_into_workpool(); - }).detach(); - } - - std::vector jhs; - auto start = std::chrono::system_clock::now(); - photon::semaphore sem; - for (int i = 0; i < 4; i++) { - CopyMoveRecord *r = new CopyMoveRecord(); - pool->async_call( - new auto ([&sem, i, r]() { - LOG_INFO("START ", VALUE(__cplusplus), VALUE(r->copy), - VALUE(r->move)); - EXPECT_EQ(0, r->copy); - photon::thread_sleep(1); - sem.signal(1); - LOG_INFO("FINISH"); - delete r; - })); - } - auto duration = std::chrono::system_clock::now() - start; - EXPECT_GE(duration, std::chrono::seconds(0)); - EXPECT_LE(duration, std::chrono::seconds(1)); - sem.wait(4); - duration = std::chrono::system_clock::now() - start; - EXPECT_GE(duration, std::chrono::seconds(1)); - EXPECT_LE(duration, std::chrono::seconds(2)); - LOG_INFO("DONE"); -} - #if defined(_WIN64) #define SAVE_REG(R) register uint64_t R asm(#R); volatile uint64_t saved_##R = R; #define CHECK_REG(R) asm volatile ("" : "=m"(saved_##R)); if (saved_##R != R) puts(#R " differs after context switch!"); @@ -1892,6 +1626,7 @@ TEST(intrusive_list, split) { int main(int argc, char** arg) { + if (!photon::is_using_default_engine()) return 0; ::testing::InitGoogleTest(&argc, arg); gflags::ParseCommandLineFlags(&argc, &arg, true); default_audit_logger.log_output = log_output_stdout; diff --git a/thread/thread.cpp b/thread/thread.cpp index e6e2379d..de1e6b1a 100644 --- a/thread/thread.cpp +++ b/thread/thread.cpp @@ -268,8 +268,9 @@ namespace photon void init_main_thread_stack() { #ifdef __APPLE__ - stack_size = pthread_get_stacksize_np(pthread_self()); - stackful_alloc_top = (char*) pthread_get_stackaddr_np(pthread_self()); + auto self = pthread_self(); + stack_size = pthread_get_stacksize_np(self); + stackful_alloc_top = (char*) pthread_get_stackaddr_np(self); #elif defined(_WIN64) ULONG_PTR stack_low, stack_high; GetCurrentThreadStackLimits(&stack_low, &stack_high);