diff --git a/.bazelrc b/.bazelrc new file mode 100644 index 000000000..e72297f86 --- /dev/null +++ b/.bazelrc @@ -0,0 +1,52 @@ +# Must be first. Enables build:windows, build:linux, build:macos, build:freebsd, build:openbsd +build --enable_platform_specific_config +############################################################################### +# On Windows, provide: BAZEL_SH, and BAZEL_LLVM (if using clang-cl) +# On all platforms, provide: PYTHON3_BIN_PATH=python +############################################################################### +build --action_env=PATH +# For --compilation_mode=dbg, consider enabling checks in the standard library as well (below). +build --compilation_mode=opt +# FIXME(lingxuan.zlx) TEST CASE: test wide string crash since cxx abi off. +build --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0" +# Using C++ 20 on all platforms. +build:linux --cxxopt="-std=c++20" +build:macos --cxxopt="-std=c++20" +build:clang-cl --cxxopt="-std=c++20" +build:gcc-cl --cxxopt="-std=c++20" +build:gcc-cl --cxxopt="-fcoroutines" +build:msvc-cl --cxxopt="/std:c++20" +build:windows --cxxopt="/std:c++20" +# This workaround is needed to prevent Bazel from compiling the same file twice (once PIC and once not). +build:linux --force_pic +build:macos --force_pic +build:clang-cl --compiler=clang-cl +build:msvc-cl --compiler=msvc-cl +# `LC_ALL` and `LANG` is needed for cpp worker tests, because they will call "ray start". +# If we don't add them, python's `click` library will raise an error. +build --action_env=LC_ALL +build --action_env=LANG +# Allow C++ worker tests to execute "ray start" with the correct version of Python. +build --action_env=VIRTUAL_ENV +build --action_env=PYENV_VIRTUAL_ENV +build --action_env=PYENV_VERSION +build --action_env=PYENV_SHELL +# This is needed for some core tests to run correctly +build:windows --enable_runfiles +build:linux --per_file_copt="-\\.(asm|S)$@-Werror" +build:macos --per_file_copt="-\\.(asm|S)$@-Werror" +build:clang-cl --per_file_copt="-\\.(asm|S)$@-Werror" +build:gcc-cl --per_file_copt="-\\.(asm|S)$@-Werror" +build:msvc-cl --per_file_copt="-\\.(asm|S)$@-WX" +# Ignore warnings for protobuf generated files and external projects. +build --per_file_copt="\\.pb\\.cc$@-w" +build --per_file_copt="-\\.(asm|S)$,external/.*@-w" +#build --per_file_copt="external/.*@-Wno-unused-result" +# Ignore minor warnings for host tools, which we generally can't control +build:clang-cl --host_copt="-Wno-inconsistent-missing-override" +build:clang-cl --host_copt="-Wno-microsoft-unqualified-friend" +# Ignore wchar_t -> char conversion warning on MSVC +build:msvc-cl --per_file_copt="external/boost/libs/regex/src/wc_regex_traits\\.cpp@-wd4244" +build --http_timeout_scaling=5.0 +build --verbose_failures + diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel_clang.yml similarity index 66% rename from .github/workflows/bazel.yml rename to .github/workflows/bazel_clang.yml index e502b0ac4..a52b85c4e 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel_clang.yml @@ -1,4 +1,4 @@ -name: Bazel +name: Bazel-Clang on: push: @@ -22,8 +22,8 @@ jobs: - name: Build working-directory: ${{github.workspace}} - run: bazel build --action_env=CXX=clang++-17 --action_env=CC=clang-17 ... + run: bazel build --config=clang-cl --action_env=CXX=clang++-17 --action_env=CC=clang-17 ... - name: Test working-directory: ${{github.workspace}} - run: bazel test --action_env=CXX=clang++-17 --action_env=CC=clang-17 --test_output=errors ... + run: bazel test --config=clang-cl --action_env=CXX=clang++-17 --action_env=CC=clang-17 --test_output=errors ... diff --git a/.github/workflows/bazel_gcc.yml b/.github/workflows/bazel_gcc.yml new file mode 100644 index 000000000..b14577e8a --- /dev/null +++ b/.github/workflows/bazel_gcc.yml @@ -0,0 +1,24 @@ +name: Bazel-GCC + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-22.04 + + steps: + - uses: actions/checkout@v2 + + + - name: Build + working-directory: ${{github.workspace}} + run: bazel build --config=gcc-cl ... + + - name: Test + working-directory: ${{github.workspace}} + run: bazel test --config=gcc-cl --test_output=errors ... diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml new file mode 100644 index 000000000..17fb9bc5a --- /dev/null +++ b/.github/workflows/code-coverage.yml @@ -0,0 +1,38 @@ +name: Upload CodeCov Report + +on: [ push, pull_request ] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - name: Checkout Code + uses: actions/checkout@v2 + with: + fetch-depth: 2 + + - name: Install newer Clang + run: | + wget https://apt.llvm.org/llvm.sh + chmod +x ./llvm.sh + sudo ./llvm.sh 17 + + - name: Configure + run: | + cmake -B ${{github.workspace}}/build \ + -DCMAKE_BUILD_TYPE=${{matrix.mode}} -DBUILD_WITH_LIBCXX=OFF -DYLT_ENABLE_SSL=ON \ + -DUSE_CCACHE=${{env.ccache}} -DCMAKE_C_COMPILER=clang-17 -DCMAKE_CXX_COMPILER=clang++-17 + + - name: Build with ${{ matrix.compiler }} + run: cmake --build ${{github.workspace}}/build --config Debug -- -j + + - name: Test + working-directory: ${{github.workspace}}/build + env: + CTEST_OUTPUT_ON_FAILURE: 1 + run: ctest -C ${{ matrix.configuration }} -j 1 -V + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v1 diff --git a/.github/workflows/linux_llvm_cov.yml b/.github/workflows/linux_llvm_cov.yml index 2c70a3257..d762e8f68 100644 --- a/.github/workflows/linux_llvm_cov.yml +++ b/.github/workflows/linux_llvm_cov.yml @@ -2,8 +2,9 @@ name: Ubuntu 22.04 (llvm cov) on: pull_request_target: - branches: [ doc ] - workflow_dispatch: + branches: + - main + - fix_coverage jobs: build: @@ -19,37 +20,49 @@ jobs: sudo apt-get install libssl-dev sudo apt-get install llvm + - name: Install newer Clang + run: | + wget https://apt.llvm.org/llvm.sh + chmod +x ./llvm.sh + sudo ./llvm.sh 17 + - name: Run Coverage run: | ls cp -r src/coro_rpc/tests/openssl_files . ls mkdir build && cd build - CC=clang CXX=clang++ cmake .. -DCOVERAGE_TEST=ON -DENABLE_SSL=ON - make -j test_rpc - export LLVM_PROFILE_FILE="test_rpc-%m.profraw" - ./tests/test_rpc - llvm-profdata merge -sparse test_rpc-*.profraw -o test_rpc.profdata - llvm-cov show ./tests/test_rpc -instr-profile=test_rpc.profdata -format=html -output-dir=../.coverage_llvm_cov -ignore-filename-regex="async_simple|thirdparty|tests|asio|util|logging|struct_pack" -show-instantiations=false + CC=clang-17 CXX=clang++-17 cmake .. -DCOVERAGE_TEST=ON -DYLT_ENABLE_SSL=ON + make -j + export LLVM_PROFILE_FILE="test_ylt-%m.profraw" + cd output + cd tests + ./coro_io_test + ./coro_rpc_test + ./easylog_test + ./struct_pack_test + ./struct_pack_test_with_optimize + llvm-profdata merge -sparse test_ylt-*.profraw -o test_ylt.profdata + llvm-cov show coro_io_test coro_rpc_test easylog_test struct_pack_test struct_pack_test_with_optimize -instr-profile=test_ylt.profdata -format=html -output-dir=../../.coverage_llvm_cov -ignore-filename-regex="thirdparty|asio|src" -show-instantiations=false echo "Done!" - name: Upload Coverage Results uses: actions/upload-artifact@v3 with: name: llvm-cov - path: ${{ github.workspace }}/.coverage_llvm_cov + path: ${{ github.workspace }}/build/.coverage_llvm_cov - name: Create Code Coverage Report - working-directory: ${{github.workspace}}/build + working-directory: ${{github.workspace}}/build/output/tests run: | echo "Code Coverage Report" > tmp.log echo "for detail, [goto summary](https://github.com/${{ github.repository_owner }}/${{ github.event.repository.name }}/actions/runs/${{github.run_id}}) download Artifacts `llvm-cov`" >> tmp.log echo "\`\`\`" >> tmp.log - llvm-cov report ./tests/test_rpc -instr-profile=test_rpc.profdata -ignore-filename-regex="thirdparty|tests" -show-region-summary=false >> tmp.log + llvm-cov report coro_io_test coro_rpc_test easylog_test struct_pack_test struct_pack_test_with_optimize -instr-profile=test_ylt.profdata -ignore-filename-regex="thirdparty|asio|src" -show-region-summary=false >> tmp.log echo "\`\`\`" >> tmp.log - name: Create Comment uses: peter-evans/create-or-update-comment@v2 with: issue-number: ${{ github.event.pull_request.number }} - body-file: '${{github.workspace}}/build/tmp.log' + body-file: '${{github.workspace}}/build/output/tests/tmp.log' diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 01df2ff96..790178fc0 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -36,7 +36,7 @@ jobs: key: ${{ github.job }}-${{ matrix.mode}}-ssl( ${{ matrix.ssl}} ) - name: Configure CMake - run: OPENSSL_ROOT_DIR=/usr/local/opt/openssl@3 CXX=clang++ CC=clang cmake -B ${{github.workspace}}/build -G Ninja -DCMAKE_BUILD_TYPE=${{matrix.mode}} -DENABLE_SSL=${{matrix.ssl}} -DUSE_CCACHE=ON + run: OPENSSL_ROOT_DIR=/usr/local/opt/openssl@3 CXX=clang++ CC=clang cmake -B ${{github.workspace}}/build -G Ninja -DCMAKE_BUILD_TYPE=${{matrix.mode}} -DYLT_ENABLE_SSL=${{matrix.ssl}} -DUSE_CCACHE=ON - name: Build run: cmake --build ${{github.workspace}}/build --config ${{matrix.mode}} diff --git a/.github/workflows/ubuntu_clang.yml b/.github/workflows/ubuntu_clang.yml index de39b9b58..fe4ae73c1 100644 --- a/.github/workflows/ubuntu_clang.yml +++ b/.github/workflows/ubuntu_clang.yml @@ -45,7 +45,7 @@ jobs: - name: Configure run: | cmake -B ${{github.workspace}}/build -G Ninja \ - -DCMAKE_BUILD_TYPE=${{matrix.mode}} -DBUILD_WITH_LIBCXX=${{matrix.libcxx}} -DENABLE_SSL=${{matrix.ssl}} \ + -DCMAKE_BUILD_TYPE=${{matrix.mode}} -DBUILD_WITH_LIBCXX=${{matrix.libcxx}} -DYLT_ENABLE_SSL=${{matrix.ssl}} \ -DUSE_CCACHE=${{env.ccache}} -DCMAKE_C_COMPILER=clang-17 -DCMAKE_CXX_COMPILER=clang++-17 - name: Build run: cmake --build ${{github.workspace}}/build --config ${{matrix.mode}} @@ -92,7 +92,7 @@ jobs: run: | CXX=clang++ CC=clang cmake -B ${{github.workspace}}/build -G Ninja \ - -DCMAKE_BUILD_TYPE=${{matrix.mode}} -DBUILD_WITH_LIBCXX=${{matrix.libcxx}} -DENABLE_SSL=${{matrix.ssl}} \ + -DCMAKE_BUILD_TYPE=${{matrix.mode}} -DBUILD_WITH_LIBCXX=${{matrix.libcxx}} -DYLT_ENABLE_SSL=${{matrix.ssl}} \ -DUSE_CCACHE=${{env.ccache}} -DCMAKE_C_COMPILER=clang-17 -DCMAKE_CXX_COMPILER=clang++-17\ -DBUILD_CORO_HTTP=OFF -DBUILD_CORO_IO=OFF -DBUILD_CORO_RPC=OFF -DBUILD_EASYLOG=OFF -DBUILD_STRUCT_JSON=OFF -DBUILD_STRUCT_XML=OFF -DBUILD_STRUCT_YAML=OFF -DBUILD_UTIL=OFF - name: Build @@ -138,7 +138,7 @@ jobs: cmake -B ${{github.workspace}}/build -G Ninja \ -DCMAKE_BUILD_TYPE=${{matrix.mode}} \ -DBUILD_WITH_LIBCXX=${{matrix.libcxx}} \ - -DENABLE_IO_URING=${{matrix.io_uring}} \ + -DYLT_ENABLE_IO_URING=${{matrix.io_uring}} \ -DUSE_CCACHE=${{env.ccache}} -DCMAKE_C_COMPILER=clang-17 -DCMAKE_CXX_COMPILER=clang++-17\ -DBUILD_STRUCT_JSON=OFF -DBUILD_STRUCT_XML=OFF -DBUILD_STRUCT_PACK=OFF -DBUILD_STRUCT_PB=OFF -DBUILD_STRUCT_YAML=OFF -DBUILD_UTIL=OFF diff --git a/.github/workflows/ubuntu_gcc.yml b/.github/workflows/ubuntu_gcc.yml index e4c53fa57..fa02eb7ea 100644 --- a/.github/workflows/ubuntu_gcc.yml +++ b/.github/workflows/ubuntu_gcc.yml @@ -38,7 +38,7 @@ jobs: run: | CXX=g++ CC=gcc cmake -B ${{github.workspace}}/build -G Ninja \ - -DCMAKE_BUILD_TYPE=${{matrix.mode}} -DBUILD_WITH_LIBCXX=${{matrix.libcxx}} -DENABLE_SSL=${{matrix.ssl}} \ + -DCMAKE_BUILD_TYPE=${{matrix.mode}} -DBUILD_WITH_LIBCXX=${{matrix.libcxx}} -DYLT_ENABLE_SSL=${{matrix.ssl}} \ -DUSE_CCACHE=${{env.ccache}} - name: Build @@ -78,7 +78,7 @@ jobs: run: | CXX=g++ CC=gcc cmake -B ${{github.workspace}}/build -G Ninja \ - -DCMAKE_BUILD_TYPE=${{matrix.mode}} -DBUILD_WITH_LIBCXX=${{matrix.libcxx}} -DENABLE_SSL=${{matrix.ssl}} \ + -DCMAKE_BUILD_TYPE=${{matrix.mode}} -DBUILD_WITH_LIBCXX=${{matrix.libcxx}} -DYLT_ENABLE_SSL=${{matrix.ssl}} \ -DUSE_CCACHE=${{env.ccache}} \ -DBUILD_CORO_HTTP=OFF -DBUILD_CORO_IO=OFF -DBUILD_CORO_RPC=OFF -DBUILD_EASYLOG=OFF -DBUILD_STRUCT_JSON=OFF -DBUILD_STRUCT_XML=OFF -DBUILD_STRUCT_YAML=OFF -DBUILD_UTIL=OFF @@ -116,7 +116,7 @@ jobs: CXX=g++ CC=gcc cmake -B ${{github.workspace}}/build -G Ninja \ -DCMAKE_BUILD_TYPE=${{matrix.mode}} \ - -DENABLE_IO_URING=${{matrix.io_uring}} \ + -DYLT_ENABLE_IO_URING=${{matrix.io_uring}} \ -DUSE_CCACHE=${{env.ccache}} \ -DBUILD_STRUCT_JSON=OFF -DBUILD_STRUCT_XML=OFF -DBUILD_STRUCT_PACK=OFF -DBUILD_STRUCT_PB=OFF -DBUILD_STRUCT_YAML=OFF -DBUILD_UTIL=OFF diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 2c7f0d85e..c90e6f117 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -38,7 +38,7 @@ jobs: with: key: ${{ github.job }}-${{ matrix.mode}}-ssl( ${{ matrix.ssl}} )-arch-${{ matrix.arch}} - name: Configure CMake - run: cmake -B ${{github.workspace}}\build -G Ninja -DCMAKE_BUILD_TYPE=${{ matrix.mode }} -DENABLE_SSL=${{matrix.ssl}} -DUSE_CCACHE=ON + run: cmake -B ${{github.workspace}}\build -G Ninja -DCMAKE_BUILD_TYPE=${{ matrix.mode }} -DYLT_ENABLE_SSL=${{matrix.ssl}} -DUSE_CCACHE=ON - name: Build run: cmake --build ${{github.workspace}}\build - name: Test @@ -72,7 +72,7 @@ jobs: with: key: ${{ github.job }}-${{ matrix.mode}}-arch-${{ matrix.arch}} - name: Configure CMake - run: cmake -B ${{github.workspace}}\build -G Ninja -DCMAKE_BUILD_TYPE=${{ matrix.mode }} -DENABLE_SSL=${{matrix.ssl}} -DUSE_CCACHE=ON -DENABLE_CPP_20=OFF + run: cmake -B ${{github.workspace}}\build -G Ninja -DCMAKE_BUILD_TYPE=${{ matrix.mode }} -DYLT_ENABLE_SSL=${{matrix.ssl}} -DUSE_CCACHE=ON -DENABLE_CPP_20=OFF - name: Build run: cmake --build ${{github.workspace}}\build - name: Test diff --git a/BUILD.bazel b/BUILD.bazel index d291c09d8..c89ffa4ce 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -1,13 +1,95 @@ load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library") +load("//bazel:defs.bzl", "YA_BIN_COPT", "YA_LT_COPT") + package(default_visibility = ["//visibility:public"]) cc_library( name = "ylt", - hdrs = glob([ - "include/**", - "src/include/**" + srcs = glob([ + "include/ylt/**/*.hpp", + "include/ylt/**/*.h", + "include/ylt/**/*.ipp", + "src/include/*.h", ]), - includes = ["include", "include/ylt/thirdparty","src/include"], + copts = YA_LT_COPT, + includes = [ + "include", + "include/ylt", + "include/ylt/thirdparty", + "src/include", + ], linkopts = ["-lpthread"], visibility = ["//visibility:public"], ) + +# List one example for ylt tests. +cc_test( + name = "easylog_test", + srcs = [ + "src/easylog/tests/main.cpp", + "src/easylog/tests/test_easylog.cpp", + ], + copts = YA_BIN_COPT, + includes = [ + "include", + "include/ylt/thirdparty", + "src/include", + ], + deps = [":ylt"], +) + +cc_binary( + name = "easylog_benchmark", + srcs = [ + "src/easylog/benchmark/main.cpp", + ], + copts = YA_BIN_COPT, + includes = [ + "include", + "include/ylt/thirdparty", + "src/include", + ], + deps = [":ylt"], +) + +cc_binary( + name = "coro_http_example", + srcs = ["src/coro_http/examples/example.cpp"], + copts = YA_BIN_COPT, + includes = [ + "include", + "include/ylt", + "include/ylt/thirdparty", + "src/include", + ], + linkopts = ["-lpthread"], + deps = [":ylt"], +) + +cc_binary( + name = "coro_http_channel", + srcs = ["src/coro_http/examples/channel.cpp"], + copts = YA_BIN_COPT, + includes = [ + "include", + "include/ylt", + "include/ylt/thirdparty", + "src/include", + ], + linkopts = ["-lpthread"], + deps = [":ylt"], +) + +cc_binary( + name = "coro_http_chat_room", + srcs = ["src/coro_http/examples/chat_room.cpp"], + copts = YA_BIN_COPT, + includes = [ + "include", + "include/ylt", + "include/ylt/thirdparty", + "src/include", + ], + linkopts = ["-lpthread"], + deps = [":ylt"], +) diff --git a/README.md b/README.md index b921975e4..e2dcd8097 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,15 @@ cd build cmake .. cmake --build . --config debug # add -j, if you have enough memory to parallel compile ctest . # run tests + +``` + +- Build in bazel: +```shell +bazel build ylt # Please make sure bazel in you bin path. +bazel build coro_http_example # Or replace in anyone you want to build and test. +# Actually you might take it in other project in prefix @com_alibaba_yalangtinglibs, like +bazel build @com_alibaba_yalangtinglibs://ylt ``` You can see the test/example/benchmark executable file in `./build/output/`. @@ -73,7 +82,6 @@ You can see the test/example/benchmark executable file in `./build/output/`. ```shell # You can use those option to skip build unit-test & benchmark & example: cmake .. -DBUILD_EXAMPLES=OFF -DBUILD_BENCHMARK=OFF -DBUILD_UNIT_TESTS=OFF -cmake --build . ``` 3. install @@ -400,18 +408,36 @@ See [async_simple](https://github.com/alibaba/async_simple) # Details -## CMAKE OPTION +## CMAKE OPTION + +## config option -These CMake options is used for yalantinglibs developing/installing itself. They are not effected for your project, because ylt is a head-only. +These option maybe useful for your project. You can enable it in your project if you import ylt by cmake fetchContent or find_package. -### INSTALL OPTION +|option|default value|description| +|----------|------------|------| +|YLT_ENABLE_SSL|OFF|enable optional ssl support for rpc/http| +|YLT_ENABLE_PMR|OFF|enable pmr optimize| +|YLT_ENABLE_IO_URING|OFF|enable io_uring in linux| +|YLT_ENABLE_FILE_IO_URING|OFF|enable file io_uring as backend in linux| +|YLT_ENABLE_STRUCT_PACK_UNPORTABLE_TYPE|OFF|enable unportable type(like wstring, int128_t) for struct_pack| +|YLT_ENABLE_STRUCT_PACK_OPTIMIZE|OFF|optimize struct_pack by radical template unwinding(will cost more compile time)| + +## thirdparty installation option + +In default, yalantinglibs will install thirdparty librarys in `ylt/thirdparty`. You need add it to include path when compile. + +If you don't want to install the thirdparty librarys, you can turn off cmake option `-DINSTALL_THIRDPARTY=OFF`. +If you want to install the thirdparty independently (direct install it in system include path so that you don't need add `ylt/thirdparty` to include path), you can use turn on cmake option `-DINSTALL_INDEPENDENT_THIRDPARTY=ON`. |option|default value| |----------|------------| |INSTALL_THIRDPARTY|ON| |INSTALL_INDEPENDENT_THIRDPARTY|OFF| -### ylt develop option +## develop option + +These CMake options is used for yalantinglibs developing/installing itself. They are not effected for your project, because ylt is a head-only. |option|default value| |----------|------------| @@ -423,25 +449,8 @@ These CMake options is used for yalantinglibs developing/installing itself. They |GENERATE_BENCHMARK_DATA|ON| |CORO_RPC_USE_OTHER_RPC|ON| -### ylt config option - -These option maybe useful for your project. If you want to enable it in your project, see the cmake code [here](https://github.com/alibaba/yalantinglibs/tree/main/cmake/config.cmake) - -|option|default value| -|----------|------------| -|ENABLE_SSL|OFF| -|ENABLE_PMR|OFF| -|ENABLE_IO_URING|OFF| -|ENABLE_FILE_IO_URING|OFF| -|ENABLE_STRUCT_PACK_UNPORTABLE_TYPE|OFF| -|ENABLE_STRUCT_PACK_OPTIMIZE|OFF| -## Thirdparty Dependency - -In default, yalantinglibs will install thirdparty librarys in `ylt/thirdparty`. You need add it to include path when compile. - -If you don't want to install the thirdparty librarys, you can turn off cmake option `-DINSTALL_THIRDPARTY=OFF`. -If you want to install the thirdparty independently (direct install it in system include path so that you don't need add `ylt/thirdparty` to include path), you can use turn on cmake option `-DINSTALL_INDEPENDENT_THIRDPARTY=ON`. +## Thirdparty Dependency List Here are the thirdparty libraries we used(Although async_simple is a part of ylt, it open source first, so we import it as a independence thirdparty library). diff --git a/WORKSPACE b/WORKSPACE new file mode 100644 index 000000000..54df50fca --- /dev/null +++ b/WORKSPACE @@ -0,0 +1 @@ +workspace(name = "com_alibaba_yalantinglibs") diff --git a/WORKSPACE.bazel b/bazel/BUILD similarity index 100% rename from WORKSPACE.bazel rename to bazel/BUILD diff --git a/bazel/defs.bzl b/bazel/defs.bzl new file mode 100644 index 000000000..d7492cd68 --- /dev/null +++ b/bazel/defs.bzl @@ -0,0 +1,15 @@ +YA_LT_COPT = [ + "-fno-tree-slp-vectorize", # -ftree-slp-vectorize with coroutine cause link error. disable it util gcc fix. +] + +YA_BIN_COPT = [ + "-fno-tree-slp-vectorize", # -ftree-slp-vectorize with coroutine cause link error. disable it util gcc fix. + "-Wno-unused-but-set-variable", + "-Wno-unused-value", + "-Wno-unused-variable", + "-Wno-sign-compare", + "-Wno-reorder", + "-Wno-unused-local-typedefs", + "-Wno-missing-braces", + "-Wno-uninitialized", +] diff --git a/cmake/build.cmake b/cmake/build.cmake index aed55333b..0a41a6c97 100644 --- a/cmake/build.cmake +++ b/cmake/build.cmake @@ -1,4 +1,4 @@ -message(STATUS "-------------COMPILE SETTING-------------") +message(STATUS "-------------YLT COMPILE SETTING------------") # CPP Standard foreach(i ${CMAKE_CXX_COMPILE_FEATURES}) @@ -79,3 +79,4 @@ if(CMAKE_BUILD_TYPE STREQUAL "Release") add_compile_options("$<$:/EHa>") endif() +message(STATUS "--------------------------------------------") \ No newline at end of file diff --git a/cmake/config.cmake b/cmake/config.cmake index 0897702be..01eb1f614 100644 --- a/cmake/config.cmake +++ b/cmake/config.cmake @@ -1,34 +1,53 @@ -message(STATUS "-------------PROJECT SETTING-------------") -option(ENABLE_SSL "Enable ssl support" OFF) -message(STATUS "ENABLE_SSL: ${ENABLE_SSL}") -if (ENABLE_SSL) +message(STATUS "-------------YLT CONFIG SETTING-------------") +option(YLT_ENABLE_SSL "Enable ssl support" OFF) +message(STATUS "ENABLE_SSL: ${YLT_ENABLE_SSL}") +if (YLT_ENABLE_SSL) find_package(OpenSSL REQUIRED) - add_compile_definitions(YLT_ENABLE_SSL) - link_libraries(OpenSSL::SSL OpenSSL::Crypto) + if(CMAKE_PROJECT_NAME STREQUAL "yaLanTingLibs") + add_compile_definitions("YLT_ENABLE_SSL") + link_libraries(OpenSSL::SSL OpenSSL::Crypto) + else () + target_compile_definitions(yalantinglibs INTERFACE "YLT_ENABLE_SSL") + target_link_libraries(yalantinglibs INTERFACE OpenSSL::SSL OpenSSL::Crypto) + endif () endif () -option(ENABLE_PMR "Enable pmr support" OFF) -message(STATUS "ENABLE_PMR: ${ENABLE_PMR}") -if (ENABLE_PMR) - add_compile_definitions(YLT_ENABLE_PMR IGUANA_ENABLE_PMR) +option(YLT_ENABLE_PMR "Enable pmr support" OFF) +message(STATUS "ENABLE_PMR: ${YLT_ENABLE_PMR}") +if (YLT_ENABLE_PMR) + if(CMAKE_PROJECT_NAME STREQUAL "yaLanTingLibs") + add_compile_definitions("YLT_ENABLE_PMR" IGUANA_ENABLE_PMR) + else () + target_compile_definitions(yalantinglibs INTERFACE "YLT_ENABLE_PMR" IGUANA_ENABLE_PMR) + endif () endif () -option(ENABLE_IO_URING "Enable io_uring" OFF) -message(STATUS "ENABLE_IO_URING: ${ENABLE_IO_URING}") -if (ENABLE_IO_URING) +option(YLT_ENABLE_IO_URING "Enable io_uring" OFF) +message(STATUS "ENABLE_IO_URING: ${YLT_ENABLE_IO_URING}") +if (YLT_ENABLE_IO_URING) find_package(uring REQUIRED) message(STATUS "Use IO_URING for all I/O in linux") - add_compile_definitions(ASIO_HAS_IO_URING ASIO_DISABLE_EPOLL ASIO_HAS_FILE YLT_ENABLE_FILE_IO_URING) - link_libraries(uring) + if(CMAKE_PROJECT_NAME STREQUAL "yaLanTingLibs") + add_compile_definitions(ASIO_HAS_IO_URING ASIO_DISABLE_EPOLL ASIO_HAS_FILE YLT_ENABLE_FILE_IO_URING) + link_libraries(uring) + else () + target_compile_definitions(yalantinglibs INTERFACE ASIO_HAS_IO_URING ASIO_DISABLE_EPOLL ASIO_HAS_FILE YLT_ENABLE_FILE_IO_URING) + target_link_libraries(yalantinglibs INTERFACE uring) + endif () endif() -option(ENABLE_FILE_IO_URING "Enable file io_uring" OFF) -if (NOT ENABLE_IO_URING) - if(ENABLE_FILE_IO_URING) +option(YLT_ENABLE_FILE_IO_URING "Enable file io_uring" OFF) +if (NOT YLT_ENABLE_IO_URING) + if(YLT_ENABLE_FILE_IO_URING) find_package(uring REQUIRED) message(STATUS "Enable io_uring for file I/O in linux") - add_compile_definitions(ASIO_HAS_IO_URING ASIO_HAS_FILE YLT_ENABLE_FILE_IO_URING) - link_libraries(uring) + if(CMAKE_PROJECT_NAME STREQUAL "yaLanTingLibs") + add_compile_definitions(ASIO_HAS_IO_URING ASIO_HAS_FILE "YLT_ENABLE_FILE_IO_URING") + link_libraries(uring) + else () + target_compile_definitions(yalantinglibs INTERFACE ASIO_HAS_IO_URING ASIO_HAS_FILE "YLT_ENABLE_FILE_IO_URING") + target_link_libraries(yalantinglibs INTERFACE uring) + endif () endif() endif() @@ -38,8 +57,13 @@ if(ENABLE_STRUCT_PACK_UNPORTABLE_TYPE) add_compile_definitions(STRUCT_PACK_ENABLE_UNPORTABLE_TYPE) endif() -option(ENABLE_STRUCT_PACK_OPTIMIZE "enable struct_pack optimize(but cost more compile time)" OFF) -message(STATUS "ENABLE_STRUCT_PACK_OPTIMIZE: ${ENABLE_STRUCT_PACK_OPTIMIZE}") +option(YLT_ENABLE_STRUCT_PACK_OPTIMIZE "enable struct_pack optimize(but cost more compile time)" OFF) +message(STATUS "ENABLE_STRUCT_PACK_OPTIMIZE: ${YLT_ENABLE_STRUCT_PACK_OPTIMIZE}") if(ENABLE_STRUCT_PACK_OPTIMIZE) - add_compile_definitions(ENABLE_STRUCT_PACK_OPTIMIZE) -endif() \ No newline at end of file + if(CMAKE_PROJECT_NAME STREQUAL "yaLanTingLibs") + add_compile_definitions(ENABLE_STRUCT_PACK_OPTIMIZE) + else () + target_compile_definitions(yalantinglibs INTERFACE ENABLE_STRUCT_PACK_OPTIMIZE) + endif () +endif() +message(STATUS "--------------------------------------------") \ No newline at end of file diff --git a/cmake/develop.cmake b/cmake/develop.cmake index 2daf3b317..061f1d7ee 100644 --- a/cmake/develop.cmake +++ b/cmake/develop.cmake @@ -1,4 +1,4 @@ -message(STATUS "-------------DEVELOP SETTING-------------") +message(STATUS "-------------YLT DEVELOP SETTING------------") # extra option(BUILD_EXAMPLES "Build examples" ON) message(STATUS "BUILD_EXAMPLES: ${BUILD_EXAMPLES}") @@ -65,3 +65,4 @@ if(ENABLE_WARNING) -Wfatal-errors) endif() endif() +message(STATUS "--------------------------------------------") \ No newline at end of file diff --git a/cmake/install.cmake b/cmake/install.cmake index 3967e2ee4..783f24eb2 100644 --- a/cmake/install.cmake +++ b/cmake/install.cmake @@ -1,4 +1,4 @@ -message(STATUS "-------------INSTALL SETTING-------------") +message(STATUS "-------------YLT INSTALL SETTING------------") option(INSTALL_THIRDPARTY "Install thirdparty" ON) message(STATUS "INSTALL_THIRDPARTY: " ${INSTALL_THIRDPARTY}) option(INSTALL_INDEPENDENT_THIRDPARTY "Install independent thirdparty" ON) @@ -25,8 +25,21 @@ install(TARGETS yalantinglibs ARCHIVE DESTINATION lib RUNTIME DESTINATION bin ) + +file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/yalantinglibsConfig.cmake" + "include(\$\{CMAKE_CURRENT_LIST_DIR\}/yalantinglibsConfigImpl.cmake)\n" + "include(\$\{CMAKE_CURRENT_LIST_DIR\}/config.cmake)\n" +) + +install(FILES "${CMAKE_CURRENT_BINARY_DIR}/yalantinglibsConfig.cmake" + DESTINATION ${ConfigPackageLocation}) + +install(FILES "${yaLanTingLibs_SOURCE_DIR}/cmake/config.cmake" + DESTINATION ${ConfigPackageLocation} + ) + install(EXPORT yalantinglibsTargets - FILE yalantinglibsConfig.cmake + FILE yalantinglibsConfigImpl.cmake NAMESPACE yalantinglibs:: DESTINATION ${ConfigPackageLocation} ) @@ -43,4 +56,5 @@ if (INSTALL_THIRDPARTY) $ ) endif() -endif() \ No newline at end of file +endif() +message(STATUS "--------------------------------------------") \ No newline at end of file diff --git a/cmake/subdir.cmake b/cmake/subdir.cmake index 527486465..90f1a08cc 100644 --- a/cmake/subdir.cmake +++ b/cmake/subdir.cmake @@ -1,3 +1,5 @@ +message(STATUS "-------------YLT PROJECT SETTING------------") + file(GLOB children src/*) foreach(child ${children}) @@ -40,4 +42,5 @@ foreach(child ${children}) endforeach() if (BUILD_STRUCT_PB) add_subdirectory(src/struct_pb) -endif() \ No newline at end of file +endif() +message(STATUS "--------------------------------------------") \ No newline at end of file diff --git a/coverage_gen.sh b/coverage_gen.sh index cf99f07ad..d6e976f95 100644 --- a/coverage_gen.sh +++ b/coverage_gen.sh @@ -25,18 +25,18 @@ mkdir build && cd build || exit 1 export CC=clang export CXX=clang++ cmake .. -DCOVERAGE_TEST=ON -DENABLE_SSL=ON -make -j test_rpc +make -j -# warning: test_rpc.profraw: malformed instrumentation profile data +# warning: test_ylt.profraw: malformed instrumentation profile data # error: no profile can be merged # add %m to fix the bug on ARM # https://groups.google.com/g/llvm-dev/c/oaA58fbNMGg # https://github.com/llvm/llvm-project/issues/50966 -export LLVM_PROFILE_FILE="coro_rpc_test-%m.profraw" +export LLVM_PROFILE_FILE="ylt_test-%m.profraw" ls -cd tests -./test_rpc -llvm-profdata merge -sparse coro_rpc_test-*.profraw -o coro_rpc_test.profdata -llvm-cov show ./coro_rpc_test -instr-profile=test_rpc.profdata -format=html -output-dir=../../.coverage_llvm_cov -ignore-filename-regex="async_simple|thirdparty|tests|asio|util|logging|struct_pack" -show-instantiations=false +cd output +./tests/coro_io_test ./tests/coro_rpc_test ./tests/easylog_test ./tests/struct_pack_test ./tests/struct_pack_test_with_optimize +llvm-profdata merge -sparse ylt_test-*.profraw -o ylt_test.profdata +llvm-cov show ./tests/coro_io_test ./tests/coro_rpc_test ./tests/easylog_test ./tests/struct_pack_test ./tests/struct_pack_test_with_optimize -instr-profile=test_ylt.profdata -format=html -output-dir=../../.coverage_llvm_cov -ignore-filename-regex="thirdparty|asio" -show-instantiations=false echo 'Done!!!' fi diff --git a/include/ylt/coro_io/channel.hpp b/include/ylt/coro_io/channel.hpp index 2587837a9..c9ee6d594 100644 --- a/include/ylt/coro_io/channel.hpp +++ b/include/ylt/coro_io/channel.hpp @@ -18,6 +18,7 @@ #include #include +#include #include #include "client_pool.hpp" @@ -26,7 +27,8 @@ namespace coro_io { enum class load_blance_algorithm { RR = 0, // round-robin - random = 1 + WRR, // weight round-robin + random, }; template @@ -51,6 +53,89 @@ class channel { co_return channel.client_pools_[i % channel.client_pools_.size()]; } }; + + /* + Supposing that there is a server set ''S'' = {S0, S1, …, Sn-1}; + W(Si) indicates the weight of Si; + ''i'' indicates the server selected last time, and ''i'' is initialized with + -1; + ''cw'' is the current weight in scheduling, and cw is initialized with zero; + max(S) is the maximum weight of all the servers in S; + gcd(S) is the greatest common divisor of all server weights in S; + + while (true) { + i = (i + 1) mod n; + if (i == 0) { + cw = cw - gcd(S); + if (cw <= 0) { + cw = max(S); + if (cw == 0) + return NULL; + } + } + if (W(Si) >= cw) + return Si; + } + */ + struct WRRLoadBlancer { + WRRLoadBlancer(const std::vector& weights) : weights_(weights) { + max_gcd_ = get_max_weight_gcd(); + max_weight_ = get_max_weight(); + } + + async_simple::coro::Lazy> operator()( + const channel& channel) { + int selected = select_host_with_weight_round_robin(); + if (selected == -1) { + selected = 0; + } + + wrr_current_ = selected; + co_return channel.client_pools_[selected % channel.client_pools_.size()]; + } + + private: + int select_host_with_weight_round_robin() { + while (true) { + wrr_current_ = (wrr_current_ + 1) % weights_.size(); + if (wrr_current_ == 0) { + weight_current_ = weight_current_ - max_gcd_; + if (weight_current_ <= 0) { + weight_current_ = max_weight_; + if (weight_current_ == 0) { + return -1; // can't find max weight server + } + } + } + + if (weights_[wrr_current_] >= weight_current_) { + return wrr_current_; + } + } + } + + int get_max_weight_gcd() { + int res = weights_[0]; + int cur_max = 0, cur_min = 0; + for (size_t i = 0; i < weights_.size(); i++) { + cur_max = (std::max)(res, weights_[i]); + cur_min = (std::min)(res, weights_[i]); + res = std::gcd(cur_max, cur_min); + } + return res; + } + + int get_max_weight() { + return *std::max_element(weights_.begin(), weights_.end()); + } + + std::vector weights_; + int max_gcd_ = 0; + int max_weight_ = 0; + int wrr_current_ = -1; + int weight_current_ = 0; + }; + struct RandomLoadBlancer { async_simple::coro::Lazy> operator()( const channel& channel) { @@ -97,20 +182,27 @@ class channel { return send_request(std::move(op), config_.pool_config.client_config); } - std::size_t size() const noexcept { return client_pools_.size(); } - static channel create(const std::vector& hosts, const channel_config& config = {}, + const std::vector& weights = {}, client_pools_t& client_pools = g_clients_pool()) { channel ch; - ch.init(hosts, config, client_pools); + ch.init(hosts, config, weights, client_pools); return ch; } + /** + * @brief return the channel's hosts size. + * + * @return std::size_t + */ + std::size_t size() const noexcept { return client_pools_.size(); } + private: void init(const std::vector& hosts, - const channel_config& config, client_pools_t& client_pools) { + const channel_config& config, const std::vector& weights, + client_pools_t& client_pools) { config_ = config; client_pools_.reserve(hosts.size()); for (auto& host : hosts) { @@ -120,6 +212,15 @@ class channel { case load_blance_algorithm::RR: lb_worker = RRLoadBlancer{}; break; + case load_blance_algorithm::WRR: { + if (hosts.empty() || weights.empty()) { + throw std::invalid_argument("host/weight list is empty!"); + } + if (hosts.size() != weights.size()) { + throw std::invalid_argument("hosts count is not equal with weights!"); + } + lb_worker = WRRLoadBlancer(weights); + } break; case load_blance_algorithm::random: default: lb_worker = RandomLoadBlancer{}; @@ -127,7 +228,7 @@ class channel { return; } channel_config config_; - std::variant lb_worker; + std::variant lb_worker; std::vector> client_pools_; }; diff --git a/include/ylt/coro_io/client_pool.hpp b/include/ylt/coro_io/client_pool.hpp index d797b060e..062755890 100644 --- a/include/ylt/coro_io/client_pool.hpp +++ b/include/ylt/coro_io/client_pool.hpp @@ -429,10 +429,22 @@ class client_pool : public std::enable_shared_from_this< return send_request(std::move(op), pool_config_.client_config); } + /** + * @brief approx connection of client pools + * + * @return std::size_t + */ std::size_t free_client_count() const noexcept { return free_clients_.size() + short_connect_clients_.size(); } + /** + * @brief approx connection of client pools + * + * @return std::size_t + */ + std::size_t size() const noexcept { return free_client_count(); } + std::string_view get_host_name() const noexcept { return host_name_; } private: diff --git a/include/ylt/coro_io/coro_io.hpp b/include/ylt/coro_io/coro_io.hpp index a7e4302f3..c2bb0479c 100644 --- a/include/ylt/coro_io/coro_io.hpp +++ b/include/ylt/coro_io/coro_io.hpp @@ -26,6 +26,7 @@ #include #include +#include #include #include #include @@ -36,6 +37,7 @@ #include #include "io_context_pool.hpp" +#include "ylt/util/type_traits.h" namespace coro_io { @@ -347,6 +349,29 @@ post(Func func, co_return co_await awaitor.await_resume(helper); } +template +async_simple::coro::Lazy async_send( + asio::experimental::channel &channel, T val) { + callback_awaitor awaitor; + co_return co_await awaitor.await_resume( + [&, val = std::move(val)](auto handler) { + channel.async_send({}, std::move(val), [handler](const auto &ec) { + handler.set_value_then_resume(ec); + }); + }); +} + +template +async_simple::coro::Lazy> async_receive( + asio::experimental::channel &channel) { + callback_awaitor> awaitor; + co_return co_await awaitor.await_resume([&](auto handler) { + channel.async_receive([handler](auto ec, auto val) { + handler.set_value_then_resume(std::make_pair(ec, std::move(val))); + }); + }); +} + template std::pair read_some(Socket &sock, AsioBuffer &&buffer) { diff --git a/include/ylt/coro_io/io_context_pool.hpp b/include/ylt/coro_io/io_context_pool.hpp index e10475503..81bce7d4a 100644 --- a/include/ylt/coro_io/io_context_pool.hpp +++ b/include/ylt/coro_io/io_context_pool.hpp @@ -27,7 +27,10 @@ #include #include #include -#include +#ifdef __linux__ +#include +#include +#endif namespace coro_io { @@ -108,12 +111,12 @@ get_current_executor() { class io_context_pool { public: using executor_type = asio::io_context::executor_type; - explicit io_context_pool(std::size_t pool_size) : next_io_context_(0) { + explicit io_context_pool(std::size_t pool_size, bool cpu_affinity = false) + : next_io_context_(0), cpu_affinity_(cpu_affinity) { if (pool_size == 0) { pool_size = 1; // set default value as 1 } - easylog::logger<>::instance(); for (std::size_t i = 0; i < pool_size; ++i) { io_context_ptr io_context(new asio::io_context(1)); work_ptr work(new asio::io_context::work(*io_context)); @@ -141,6 +144,16 @@ class io_context_pool { svr->run(); }, io_contexts_[i])); + +#ifdef __linux__ + if (cpu_affinity_) { + cpu_set_t cpuset; + CPU_ZERO(&cpuset); + CPU_SET(i, &cpuset); + pthread_setaffinity_np(threads.back()->native_handle(), + sizeof(cpu_set_t), &cpuset); + } +#endif } for (std::size_t i = 0; i < threads.size(); ++i) { @@ -199,6 +212,7 @@ class io_context_pool { std::promise promise_; std::atomic has_run_or_stop_ = false; std::once_flag flag_; + bool cpu_affinity_ = false; }; class multithread_context_pool { @@ -211,7 +225,7 @@ class multithread_context_pool { ~multithread_context_pool() { stop(); } void run() { - for (std::size_t i = 0; i < thd_num_; i++) { + for (int i = 0; i < thd_num_; i++) { thds_.emplace_back([this] { ioc_.run(); }); @@ -248,7 +262,7 @@ template inline T &g_io_context_pool( unsigned pool_size = std::thread::hardware_concurrency()) { static auto _g_io_context_pool = std::make_shared(pool_size); - static bool run_helper = [](auto pool) { + [[maybe_unused]] static bool run_helper = [](auto pool) { std::thread thrd{[pool] { pool->run(); }}; @@ -262,7 +276,7 @@ template inline T &g_block_io_context_pool( unsigned pool_size = std::thread::hardware_concurrency()) { static auto _g_io_context_pool = std::make_shared(pool_size); - static bool run_helper = [](auto pool) { + [[maybe_unused]] static bool run_helper = [](auto pool) { std::thread thrd{[pool] { pool->run(); }}; diff --git a/include/ylt/coro_io/rate_limiter.hpp b/include/ylt/coro_io/rate_limiter.hpp index 62340b775..bb2372770 100644 --- a/include/ylt/coro_io/rate_limiter.hpp +++ b/include/ylt/coro_io/rate_limiter.hpp @@ -42,6 +42,8 @@ class rate_limiter { do_set_rate(permitsPerSecond, current_time_mills()); } + virtual ~rate_limiter() {} + protected: virtual void do_set_rate( double permitsPerSecond, @@ -67,6 +69,9 @@ class rate_limiter { }; class abstract_smooth_rate_limiter : public rate_limiter { + public: + virtual ~abstract_smooth_rate_limiter() {} + protected: virtual void do_set_rate(double permits_per_second, double stable_internal_micros) = 0; diff --git a/include/ylt/struct_pack/type_calculate.hpp b/include/ylt/struct_pack/type_calculate.hpp index 3991bd857..b53a0d22c 100644 --- a/include/ylt/struct_pack/type_calculate.hpp +++ b/include/ylt/struct_pack/type_calculate.hpp @@ -794,8 +794,14 @@ constexpr bool check_if_has_container() { return false; } else if constexpr (unique_ptr) { - return check_if_has_container< - remove_cvref_t, Arg, ParentArgs...>(); + if constexpr (is_base_class) { + // We can't make sure if derived class has container or not + return true; + } + else { + return check_if_has_container< + remove_cvref_t, Arg, ParentArgs...>(); + } } else if constexpr (id == type_id::container_t || id == type_id::string_t || diff --git a/include/ylt/struct_pack/util.h b/include/ylt/struct_pack/util.h index d2d4d9342..ec9218c4c 100644 --- a/include/ylt/struct_pack/util.h +++ b/include/ylt/struct_pack/util.h @@ -125,10 +125,16 @@ constexpr void STRUCT_PACK_INLINE compile_time_unique( } } +#ifndef __clang__ +#define struct_pack_has_feature(X) false +#else +#define struct_pack_has_feature __has_feature +#endif + #if __cpp_lib_string_resize_and_overwrite >= 202110L template inline void resize(std::basic_string &str, std::size_t sz) { - str.resize_and_overwrite(sz, [](ch *, std::size_t sz) { + str.resize_and_overwrite(sz, [sz](ch *, std::size_t) { return sz; }); } @@ -177,34 +183,39 @@ template class string_thief inline void resize(std::basic_string &raw_str, std::size_t sz) { std::string &str = *reinterpret_cast(&raw_str); +#if defined(_GLIBCXX_USE_CXX11_ABI) + constexpr bool is_use_cxx11_abi = _GLIBCXX_USE_CXX11_ABI; +#else + constexpr bool is_use_cxx11_abi = true; +#endif + if constexpr (std::is_same_v == false && + is_use_cxx11_abi == false) { + raw_str.resize(sz); + } + else { #if defined(__SANITIZE_ADDRESS__) || \ struct_pack_has_feature(address_sanitizer) || \ (!defined(NDEBUG) && defined(_MSVC_STL_VERSION)) - raw_str.resize(sz); + raw_str.resize(sz); #elif defined(__GLIBCXX__) || defined(_LIBCPP_VERSION) || \ defined(_MSVC_STL_VERSION) - if constexpr (is_string_reserve_shrink) { - if (sz > raw_str.capacity()) { + if constexpr (is_string_reserve_shrink) { + if (sz > raw_str.capacity()) { + str.reserve(sz * sizeof(ch)); + } + } + else { str.reserve(sz * sizeof(ch)); } - } - else { - str.reserve(sz * sizeof(ch)); - } - string_set_length_hacker(str, sz); - for (auto i = sz; i < sz + sizeof(ch); ++i) str[i] = '\0'; + string_set_length_hacker(str, sz); + for (auto i = sz; i < sz + sizeof(ch); ++i) str[i] = '\0'; #else - raw_str.resize(sz); + raw_str.resize(sz); #endif + } } #endif diff --git a/include/ylt/thirdparty/async_simple/coro/Collect.h b/include/ylt/thirdparty/async_simple/coro/Collect.h index fb42de3dd..c0db35d38 100644 --- a/include/ylt/thirdparty/async_simple/coro/Collect.h +++ b/include/ylt/thirdparty/async_simple/coro/Collect.h @@ -361,7 +361,7 @@ inline auto collectAllWindowedImpl(size_t maxConcurrency, } size_t start = 0; while (start < input_size) { - size_t end = std::min(input_size, start + maxConcurrency); + size_t end = (std::min)(input_size, start + maxConcurrency); std::vector tmp_group( std::make_move_iterator(input.begin() + start), std::make_move_iterator(input.begin() + end)); diff --git a/include/ylt/thirdparty/cinatra/cinatra_log_wrapper.hpp b/include/ylt/thirdparty/cinatra/cinatra_log_wrapper.hpp index 2094c40ee..d9b783429 100644 --- a/include/ylt/thirdparty/cinatra/cinatra_log_wrapper.hpp +++ b/include/ylt/thirdparty/cinatra/cinatra_log_wrapper.hpp @@ -31,16 +31,16 @@ constexpr inline cinatra::null_logger_t NULL_LOGGER; #ifdef CINATRA_LOG_ERROR #else #define CINATRA_LOG_ERROR \ - cerr_logger_t {} + cinatra::cerr_logger_t {} #endif #ifdef CINATRA_LOG_WARNING #else #ifndef NDEBUG #define CINATRA_LOG_WARNING \ - cerr_logger_t {} + cinatra::cerr_logger_t {} #else -#define CINATRA_LOG_WARNING NULL_LOGGER +#define CINATRA_LOG_WARNING cinatra::NULL_LOGGER #endif #endif @@ -48,9 +48,9 @@ constexpr inline cinatra::null_logger_t NULL_LOGGER; #else #ifndef NDEBUG #define CINATRA_LOG_INFO \ - cout_logger_t {} + cinatra::cout_logger_t {} #else -#define CINATRA_LOG_INFO NULL_LOGGER +#define CINATRA_LOG_INFO cinatra::NULL_LOGGER #endif #endif @@ -58,9 +58,9 @@ constexpr inline cinatra::null_logger_t NULL_LOGGER; #else #ifndef NDEBUG #define CINATRA_LOG_DEBUG \ - cout_logger_t {} + cinatra::cout_logger_t {} #else -#define CINATRA_LOG_DEBUG NULL_LOGGER +#define CINATRA_LOG_DEBUG cinatra::NULL_LOGGER #endif #endif @@ -68,8 +68,8 @@ constexpr inline cinatra::null_logger_t NULL_LOGGER; #else #ifndef NDEBUG #define CINATRA_LOG_TRACE \ - cout_logger_t {} + cinatra::cout_logger_t {} #else -#define CINATRA_LOG_TRACE NULL_LOGGER +#define CINATRA_LOG_TRACE cinatra::NULL_LOGGER #endif #endif diff --git a/include/ylt/thirdparty/cinatra/coro_http_client.hpp b/include/ylt/thirdparty/cinatra/coro_http_client.hpp index 3d8b46b29..ad8a3bc3f 100644 --- a/include/ylt/thirdparty/cinatra/coro_http_client.hpp +++ b/include/ylt/thirdparty/cinatra/coro_http_client.hpp @@ -246,7 +246,7 @@ class coro_http_client : public std::enable_shared_from_this { return true; } - [[nodiscard]] bool init_ssl(int verify_mode = asio::ssl::verify_peer, + [[nodiscard]] bool init_ssl(int verify_mode = asio::ssl::verify_none, std::string full_path = "", const std::string &sni_hostname = "") { std::string base_path; @@ -811,6 +811,11 @@ class coro_http_client : public std::enable_shared_from_this { req_str_.clear(); total_len_ = 0; #endif + + // clear + head_buf_.consume(head_buf_.size()); + chunked_buf_.consume(chunked_buf_.size()); + resp_chunk_str_.clear(); } async_simple::coro::Lazy reconnect(std::string uri) { @@ -1040,7 +1045,7 @@ class coro_http_client : public std::enable_shared_from_this { else { host = std::string{u.host}; } - bool r = init_ssl(asio::ssl::verify_peer, "", host); + bool r = init_ssl(asio::ssl::verify_none, "", host); if (!r) { data.net_err = std::make_error_code(std::errc::invalid_argument); co_return data; @@ -1097,21 +1102,23 @@ class coro_http_client : public std::enable_shared_from_this { async_simple::coro::Lazy handle_shake() { #ifdef CINATRA_ENABLE_SSL - if (has_init_ssl_) { - if (socket_->ssl_stream_ == nullptr) { - co_return std::make_error_code(std::errc::not_a_stream); + if (!has_init_ssl_) { + bool r = init_ssl(asio::ssl::verify_none, "", host_); + if (!r) { + co_return std::make_error_code(std::errc::invalid_argument); } + } - auto ec = co_await coro_io::async_handshake( - socket_->ssl_stream_, asio::ssl::stream_base::client); - if (ec) { - CINATRA_LOG_ERROR << "handle failed " << ec.message(); - } - co_return ec; + if (socket_->ssl_stream_ == nullptr) { + co_return std::make_error_code(std::errc::not_a_stream); } - else { - co_return std::error_code{}; + + auto ec = co_await coro_io::async_handshake(socket_->ssl_stream_, + asio::ssl::stream_base::client); + if (ec) { + CINATRA_LOG_ERROR << "handle failed " << ec.message(); } + co_return ec; #else // please open CINATRA_ENABLE_SSL before request https! co_return std::make_error_code(std::errc::protocol_error); @@ -1774,8 +1781,8 @@ class coro_http_client : public std::enable_shared_from_this { head_buf_.consume(head_buf_.size()); size_t header_size = 2; std::shared_ptr sock = socket_; - auto on_ws_msg = std::move(on_ws_msg_); - auto on_ws_close = std::move(on_ws_close_); + auto on_ws_msg = on_ws_msg_; + auto on_ws_close = on_ws_close_; asio::streambuf &read_buf = sock->head_buf_; bool has_init_ssl = false; #ifdef CINATRA_ENABLE_SSL diff --git a/include/ylt/thirdparty/cinatra/coro_http_connection.hpp b/include/ylt/thirdparty/cinatra/coro_http_connection.hpp index 2b1160a02..f05903796 100644 --- a/include/ylt/thirdparty/cinatra/coro_http_connection.hpp +++ b/include/ylt/thirdparty/cinatra/coro_http_connection.hpp @@ -334,9 +334,11 @@ class coro_http_connection } response_.clear(); + request_.clear(); buffers_.clear(); body_.clear(); resp_str_.clear(); + multi_buf_ = true; if (need_shrink_every_time_) { body_.shrink_to_fit(); } @@ -344,11 +346,21 @@ class coro_http_connection } async_simple::coro::Lazy reply(bool need_to_bufffer = true) { - // avoid duplicate reply - if (need_to_bufffer) { - response_.to_buffers(buffers_); + std::error_code ec; + size_t size; + if (multi_buf_) { + if (need_to_bufffer) { + response_.to_buffers(buffers_); + } + std::tie(ec, size) = co_await async_write(buffers_); } - auto [ec, _] = co_await async_write(buffers_); + else { + if (need_to_bufffer) { + response_.build_resp_str(resp_str_); + } + std::tie(ec, size) = co_await async_write(asio::buffer(resp_str_)); + } + if (ec) { CINATRA_LOG_ERROR << "async_write error: " << ec.message(); close(); @@ -393,6 +405,8 @@ class coro_http_connection return ss.str(); } + void set_multi_buf(bool r) { multi_buf_ = r; } + async_simple::coro::Lazy write_data(std::string_view message) { std::vector buffers; buffers.push_back(asio::buffer(message)); @@ -760,13 +774,10 @@ class coro_http_connection private: bool check_keep_alive() { - bool keep_alive = true; - auto val = request_.get_header_value("connection"); - if (!val.empty() && iequal0(val, "close")) { - keep_alive = false; + if (parser_.has_close()) { + return false; } - - return keep_alive; + return true; } void build_ws_handshake_head() { @@ -822,5 +833,6 @@ class coro_http_connection bool use_ssl_ = false; #endif bool need_shrink_every_time_ = false; + bool multi_buf_ = true; }; } // namespace cinatra diff --git a/include/ylt/thirdparty/cinatra/coro_http_request.hpp b/include/ylt/thirdparty/cinatra/coro_http_request.hpp index b4b7bf4c4..f5788d31b 100644 --- a/include/ylt/thirdparty/cinatra/coro_http_request.hpp +++ b/include/ylt/thirdparty/cinatra/coro_http_request.hpp @@ -2,8 +2,10 @@ #include #include +#include #include #include +#include #include "async_simple/coro/Lazy.h" #include "define.h" @@ -186,17 +188,13 @@ class coro_http_request { coro_http_connection *get_conn() { return conn_; } bool is_upgrade() { - auto h = get_header_value("Connection"); - if (h.empty()) + if (!parser_.has_upgrade()) return false; auto u = get_header_value("Upgrade"); if (u.empty()) return false; - if (h != UPGRADE) - return false; - if (u != WEBSOCKET) return false; @@ -208,24 +206,21 @@ class coro_http_request { return true; } - void set_aspect_data(const std::string &&key, const std::any &data) { - aspect_data_[key] = data; + void set_aspect_data(std::string data) { + aspect_data_.push_back(std::move(data)); } - template - std::optional get_aspect_data(const std::string &&key) { - auto it = aspect_data_.find(key); - if (it == aspect_data_.end()) { - return std::optional{}; - } + void set_aspect_data(std::vector data) { + aspect_data_ = std::move(data); + } - try { - return std::any_cast(it->second); // throws - } catch (const std::bad_any_cast &e) { - return std::optional{}; - } + template + void set_aspect_data(Args... args) { + (aspect_data_.push_back(std::move(args)), ...); } + std::vector &get_aspect_data() { return aspect_data_; } + std::unordered_map get_cookies( std::string_view cookie_str) const { auto cookies = get_cookies_map(cookie_str); @@ -259,6 +254,12 @@ class coro_http_request { } bool has_session() { return !cached_session_id_.empty(); } + void clear() { + body_ = {}; + if (!aspect_data_.empty()) { + aspect_data_.clear(); + } + } std::unordered_map params_; std::smatch matches_; @@ -268,7 +269,7 @@ class coro_http_request { std::string_view body_; coro_http_connection *conn_; bool is_websocket_; - std::unordered_map aspect_data_; + std::vector aspect_data_; std::string cached_session_id_; }; } // namespace cinatra \ No newline at end of file diff --git a/include/ylt/thirdparty/cinatra/coro_http_response.hpp b/include/ylt/thirdparty/cinatra/coro_http_response.hpp index 4b8c3b88e..0d1d61acc 100644 --- a/include/ylt/thirdparty/cinatra/coro_http_response.hpp +++ b/include/ylt/thirdparty/cinatra/coro_http_response.hpp @@ -12,6 +12,9 @@ #include "async_simple/coro/SyncAwait.h" #include "cookie.hpp" #include "define.h" +#ifdef CINATRA_ENABLE_GZIP +#include "gzip.hpp" +#endif #include "response_cv.hpp" #include "time_util.hpp" #include "utils.hpp" @@ -46,9 +49,39 @@ class coro_http_response { content_ = std::move(content); has_set_content_ = true; } - void set_status_and_content(status_type status, std::string content = "") { + void set_status_and_content( + status_type status, std::string content = "", + content_encoding encoding = content_encoding::none) { + set_status_and_content_view(status, content, encoding, false); + } + + void set_status_and_content_view( + status_type status, std::string_view content = "", + content_encoding encoding = content_encoding::none, bool is_view = true) { status_ = status; - content_ = std::move(content); +#ifdef CINATRA_ENABLE_GZIP + if (encoding == content_encoding::gzip) { + std::string encode_str; + bool r = gzip_codec::compress(content, encode_str, true); + if (!r) { + set_status_and_content(status_type::internal_server_error, + "gzip compress error"); + } + else { + add_header("Content-Encoding", "gzip"); + set_content(std::move(encode_str)); + } + } + else +#endif + { + if (is_view) { + content_view_ = content; + } + else { + content_ = std::move(content); + } + } has_set_content_ = true; } void set_delay(bool r) { delay_ = r; } @@ -61,6 +94,7 @@ class coro_http_response { status_type status() { return status_; } std::string_view content() { return content_; } + size_t content_size() { return content_.size(); } void add_header(auto k, auto v) { resp_headers_.emplace_back(resp_header{std::move(k), std::move(v)}); @@ -79,12 +113,10 @@ class coro_http_response { buffers.push_back(asio::buffer(to_http_status_string(status_))); build_resp_head(buffers); if (!content_.empty()) { - if (fmt_type_ == format_type::chunked) { - to_chunked_buffers(buffers, content_, true); - } - else { - buffers.push_back(asio::buffer(content_)); - } + handle_content(buffers, content_); + } + else if (!content_view_.empty()) { + handle_content(buffers, content_view_); } } @@ -93,7 +125,7 @@ class coro_http_response { bool has_len = false; bool has_host = false; for (auto &[k, v] : resp_headers_) { - if (k == "Host") { + if (k == "Server") { has_host = true; } if (k == "Content-Length") { @@ -157,12 +189,15 @@ class coro_http_response { bool has_len = false; bool has_host = false; for (auto &[k, v] : resp_headers_) { - if (k == "Host") { + if (k == "Server") { has_host = true; } - if (k == "Content-Length") { + else if (k == "Content-Length") { has_len = true; } + else if (k == "Date") { + need_date_ = false; + } } if (!has_host) { @@ -186,11 +221,12 @@ class coro_http_response { } if (!content_.empty()) { - auto [ptr, ec] = std::to_chars(buf_, buf_ + 32, content_.size()); - buffers.emplace_back(asio::buffer(CONTENT_LENGTH_SV)); - buffers.emplace_back( - asio::buffer(std::string_view(buf_, std::distance(buf_, ptr)))); - buffers.emplace_back(asio::buffer(CRCF)); + if (!has_len) + handle_content_len(buffers, content_); + } + else if (!content_view_.empty()) { + if (!has_len) + handle_content_len(buffers, content_view_); } else { if (!has_len && boundary_.empty()) @@ -248,7 +284,33 @@ class coro_http_response { cookies_[cookie.get_name()] = cookie; } + void redirect(const std::string &url, bool is_forever = false) { + add_header("Location", url); + is_forever == false + ? set_status_and_content(status_type::moved_temporarily) + : set_status_and_content(status_type::moved_permanently); + } + private: + void handle_content(std::vector &buffers, + std::string_view content) { + if (fmt_type_ == format_type::chunked) { + to_chunked_buffers(buffers, content, true); + } + else { + buffers.push_back(asio::buffer(content)); + } + } + + void handle_content_len(std::vector &buffers, + std::string_view content) { + auto [ptr, ec] = std::to_chars(buf_, buf_ + 32, content.size()); + buffers.emplace_back(asio::buffer(CONTENT_LENGTH_SV)); + buffers.emplace_back( + asio::buffer(std::string_view(buf_, std::distance(buf_, ptr)))); + buffers.emplace_back(asio::buffer(CRCF)); + } + status_type status_; format_type fmt_type_; std::string content_; @@ -263,5 +325,6 @@ class coro_http_response { bool need_date_ = true; std::unordered_map cookies_; std::string_view content_type_; + std::string_view content_view_; }; } // namespace cinatra diff --git a/include/ylt/thirdparty/cinatra/coro_http_router.hpp b/include/ylt/thirdparty/cinatra/coro_http_router.hpp index c50b6ab9f..d60b3dcfc 100644 --- a/include/ylt/thirdparty/cinatra/coro_http_router.hpp +++ b/include/ylt/thirdparty/cinatra/coro_http_router.hpp @@ -28,23 +28,35 @@ constexpr inline bool is_lazy_v = is_template_instant_of>::value; -struct base_aspect { - virtual bool before(coro_http_request& req, coro_http_response& resp) { - return true; - } +template +struct has_before : std::false_type {}; - virtual bool after(coro_http_request& req, coro_http_response& resp) { - return true; - } -}; +template +struct has_before().before( + std::declval(), + std::declval()))>> + : std::true_type {}; + +template +struct has_after : std::false_type {}; + +template +struct has_after().after( + std::declval(), + std::declval()))>> + : std::true_type {}; + +template +constexpr bool has_before_v = has_before::value; + +template +constexpr bool has_after_v = has_after::value; class coro_http_router { public: // eg: "GET hello/" as a key - template - void set_http_handler( - std::string key, Func handler, - std::vector> aspects = {}) { + template + void set_http_handler(std::string key, Func handler, Aspects&&... asps) { constexpr auto method_name = cinatra::method_name(method); std::string whole_str; whole_str.append(method_name).append(" ").append(key); @@ -53,13 +65,32 @@ class coro_http_router { // std::string_view, avoid memcpy when route using return_type = typename util::function_traits::return_type; if constexpr (is_lazy_v) { + std::function(coro_http_request & req, + coro_http_response & resp)> + http_handler; + if constexpr (sizeof...(Aspects) > 0) { + http_handler = [this, handler = std::move(handler), + ... asps = std::forward(asps)]( + coro_http_request& req, + coro_http_response& resp) mutable + -> async_simple::coro::Lazy { + bool ok = true; + (do_before(asps, req, resp, ok), ...); + if (ok) { + co_await handler(req, resp); + + (do_after(asps, req, resp, ok), ...); + } + }; + } + else { + http_handler = std::move(handler); + } + if (whole_str.find(":") != std::string::npos) { - std::vector coro_method_names = {}; - std::string coro_method_str; - coro_method_str.append(method_name); - coro_method_names.push_back(coro_method_str); - coro_router_tree_->coro_insert(key, std::move(handler), - coro_method_names); + std::string method_str(method_name); + coro_router_tree_->coro_insert(key, std::move(http_handler), + method_str); } else { if (whole_str.find("{") != std::string::npos || @@ -71,7 +102,7 @@ class coro_http_router { } coro_regex_handles_.emplace_back(std::regex(pattern), - std::move(handler)); + std::move(http_handler)); } else { auto [it, ok] = coro_keys_.emplace(std::move(whole_str)); @@ -79,21 +110,33 @@ class coro_http_router { CINATRA_LOG_WARNING << key << " has already registered."; return; } - coro_handles_.emplace(*it, std::move(handler)); - if (!aspects.empty()) { - has_aspects_ = true; - aspects_.emplace(*it, std::move(aspects)); - } + coro_handles_.emplace(*it, std::move(http_handler)); } } } else { + std::function + http_handler; + if constexpr (sizeof...(Aspects) > 0) { + http_handler = [this, handler = std::move(handler), + ... asps = std::forward(asps)]( + coro_http_request& req, + coro_http_response& resp) mutable { + bool ok = true; + (do_before(asps, req, resp, ok), ...); + if (ok) { + handler(req, resp); + (do_after(asps, req, resp, ok), ...); + } + }; + } + else { + http_handler = std::move(handler); + } + if (whole_str.find(':') != std::string::npos) { - std::vector method_names = {}; - std::string method_str; - method_str.append(method_name); - method_names.push_back(method_str); - router_tree_->insert(whole_str, std::move(handler), method_names); + std::string method_str(method_name); + router_tree_->insert(whole_str, std::move(http_handler), method_str); } else if (whole_str.find("{") != std::string::npos || whole_str.find(")") != std::string::npos) { @@ -103,7 +146,8 @@ class coro_http_router { replace_all(pattern, "{}", "([^/]+)"); } - regex_handles_.emplace_back(std::regex(pattern), std::move(handler)); + regex_handles_.emplace_back(std::regex(pattern), + std::move(http_handler)); } else { auto [it, ok] = keys_.emplace(std::move(whole_str)); @@ -111,12 +155,33 @@ class coro_http_router { CINATRA_LOG_WARNING << key << " has already registered."; return; } - map_handles_.emplace(*it, std::move(handler)); - if (!aspects.empty()) { - has_aspects_ = true; - aspects_.emplace(*it, std::move(aspects)); - } + map_handles_.emplace(*it, std::move(http_handler)); + } + } + } + + template + void do_before(T& aspect, coro_http_request& req, coro_http_response& resp, + bool& ok) { + if constexpr (has_before_v) { + if (!ok) { + return; } + ok = aspect.before(req, resp); + } + else { + ok = true; + } + } + + template + void do_after(T& aspect, coro_http_request& req, coro_http_response& resp, + bool& ok) { + if constexpr (has_after_v) { + ok = aspect.after(req, resp); + } + else { + ok = true; } } @@ -139,19 +204,7 @@ class coro_http_router { void route(auto handler, auto& req, auto& resp, std::string_view key) { try { - if (has_aspects_) { - auto [it, ok] = handle_aspects(req, resp, key, true); - if (!ok) { - return; - } - (*handler)(req, resp); - if (it != aspects_.end()) { - handle_aspects(req, resp, it->second, false); - } - } - else { - (*handler)(req, resp); - } + (*handler)(req, resp); } catch (const std::exception& e) { CINATRA_LOG_WARNING << "exception in business function, reason: " << e.what(); @@ -165,19 +218,7 @@ class coro_http_router { async_simple::coro::Lazy route_coro(auto handler, auto& req, auto& resp, std::string_view key) { try { - if (has_aspects_) { - auto [it, ok] = handle_aspects(req, resp, key, true); - if (!ok) { - co_return; - } - co_await (*handler)(req, resp); - if (it != aspects_.end()) { - handle_aspects(req, resp, it->second, false); - } - } - else { - co_await (*handler)(req, resp); - } + co_await (*handler)(req, resp); } catch (const std::exception& e) { CINATRA_LOG_WARNING << "exception in business function, reason: " << e.what(); @@ -202,38 +243,6 @@ class coro_http_router { const auto& get_regex_handlers() { return regex_handles_; } - bool handle_aspects(auto& req, auto& resp, auto& aspects, bool before) { - bool r = true; - for (auto& aspect : aspects) { - if (before) { - r = aspect->before(req, resp); - } - else { - r = aspect->after(req, resp); - } - if (!r) { - break; - } - } - return r; - } - - auto handle_aspects(auto& req, auto& resp, std::string_view key, - bool before) { - decltype(aspects_.begin()) it; - if (it = aspects_.find(key); it != aspects_.end()) { - auto& aspects = it->second; - bool r = handle_aspects(req, resp, aspects, before); - if (!r) { - return std::make_pair(aspects_.end(), false); - } - } - - return std::make_pair(it, true); - } - - void handle_after() {} - private: std::set keys_; std::unordered_map< @@ -262,10 +271,5 @@ class coro_http_router { std::regex, std::function( coro_http_request& req, coro_http_response& resp)>>> coro_regex_handles_; - - std::unordered_map>> - aspects_; - bool has_aspects_ = false; }; } // namespace cinatra \ No newline at end of file diff --git a/include/ylt/thirdparty/cinatra/coro_http_server.hpp b/include/ylt/thirdparty/cinatra/coro_http_server.hpp index 0a4bf66f0..9f9ae0b97 100644 --- a/include/ylt/thirdparty/cinatra/coro_http_server.hpp +++ b/include/ylt/thirdparty/cinatra/coro_http_server.hpp @@ -8,11 +8,14 @@ #include "asio/streambuf.hpp" #include "async_simple/Promise.h" #include "async_simple/coro/Lazy.h" +#include "cinatra/coro_http_client.hpp" #include "cinatra/coro_http_response.hpp" #include "cinatra/coro_http_router.hpp" +#include "cinatra/define.h" #include "cinatra/mime_types.hpp" #include "cinatra_log_wrapper.hpp" #include "coro_http_connection.hpp" +#include "ylt/coro_io/channel.hpp" #include "ylt/coro_io/coro_file.hpp" #include "ylt/coro_io/coro_io.hpp" #include "ylt/coro_io/io_context_pool.hpp" @@ -27,8 +30,10 @@ class coro_http_server { coro_http_server(asio::io_context &ctx, unsigned short port) : out_ctx_(&ctx), port_(port), acceptor_(ctx), check_timer_(ctx) {} - coro_http_server(size_t thread_num, unsigned short port) - : pool_(std::make_unique(thread_num)), + coro_http_server(size_t thread_num, unsigned short port, + bool cpu_affinity = false) + : pool_(std::make_unique(thread_num, + cpu_affinity)), port_(port), acceptor_(pool_->get_executor()->get_asio_executor()), check_timer_(pool_->get_executor()->get_asio_executor()) {} @@ -124,42 +129,82 @@ class coro_http_server { // call it after server async_start or sync_start. uint16_t port() const { return port_; } - template - void set_http_handler( - std::string key, Func handler, - std::vector> aspects = {}) { + template + void set_http_handler(std::string key, Func handler, Aspects &&...asps) { static_assert(sizeof...(method) >= 1, "must set http_method"); if constexpr (sizeof...(method) == 1) { (router_.set_http_handler(std::move(key), std::move(handler), - std::move(aspects)), + std::forward(asps)...), ...); } else { - (router_.set_http_handler(key, handler, aspects), ...); + (router_.set_http_handler(key, handler, + std::forward(asps)...), + ...); } } - template - void set_http_handler( - std::string key, Func handler, Owner &&owner, - std::vector> aspects = {}) { + template + void set_http_handler(std::string key, Func handler, + util::class_type_t &owner, Aspects &&...asps) { static_assert(std::is_member_function_pointer_v, "must be member function"); using return_type = typename util::function_traits::return_type; if constexpr (is_lazy_v) { std::function(coro_http_request & req, coro_http_response & resp)> - f = std::bind(handler, owner, std::placeholders::_1, + f = std::bind(handler, &owner, std::placeholders::_1, std::placeholders::_2); set_http_handler(std::move(key), std::move(f), - std::move(aspects)); + std::forward(asps)...); } else { std::function - f = std::bind(handler, owner, std::placeholders::_1, + f = std::bind(handler, &owner, std::placeholders::_1, std::placeholders::_2); set_http_handler(std::move(key), std::move(f), - std::move(aspects)); + std::forward(asps)...); + } + } + + template + void set_http_proxy_handler(std::string url_path, + std::vector hosts, + coro_io::load_blance_algorithm type = + coro_io::load_blance_algorithm::random, + std::vector weights = {}, + Aspects &&...aspects) { + if (hosts.empty()) { + throw std::invalid_argument("not config hosts yet!"); + } + + auto channel = std::make_shared>( + coro_io::channel::create(hosts, {.lba = type}, + weights)); + auto handler = + [this, channel, type, url_path]( + coro_http_request &req, + coro_http_response &response) -> async_simple::coro::Lazy { + co_await channel->send_request( + [this, &req, &response]( + coro_http_client &client, + std::string_view host) -> async_simple::coro::Lazy { + uri_t uri; + uri.parse_from(host.data()); + co_await reply(client, uri.get_path(), req, response); + }); + }; + + if constexpr (sizeof...(method) == 0) { + set_http_handler(url_path, std::move(handler), + std::forward(aspects)...); + } + else { + set_http_handler(url_path, std::move(handler), + std::forward(aspects)...); } } @@ -196,9 +241,9 @@ class coro_http_server { void set_transfer_chunked_size(size_t size) { chunked_size_ = size; } - void set_static_res_dir( - std::string_view uri_suffix = "", std::string file_path = "www", - std::vector> aspects = {}) { + template + void set_static_res_dir(std::string_view uri_suffix = "", + std::string file_path = "www", Aspects &&...aspects) { bool has_double_dot = (file_path.find("..") != std::string::npos) || (uri_suffix.find("..") != std::string::npos); if (std::filesystem::path(file_path).has_root_path() || @@ -220,8 +265,12 @@ class coro_http_server { } files_.clear(); + std::error_code ec; for (const auto &file : - std::filesystem::recursive_directory_iterator(static_dir_)) { + std::filesystem::recursive_directory_iterator(static_dir_, ec)) { + if (ec) { + continue; + } if (!file.is_directory()) { files_.push_back(file.path().string()); } @@ -417,7 +466,7 @@ class coro_http_server { } } }, - aspects); + std::forward(aspects)...); } } @@ -676,6 +725,30 @@ class coro_http_server { co_return true; } + async_simple::coro::Lazy reply(coro_http_client &client, + std::string url_path, + coro_http_request &req, + coro_http_response &response) { + std::unordered_map req_headers; + for (auto &[k, v] : req_headers) { + req_headers.emplace(k, v); + } + + auto ctx = req_context{.content = req.get_body()}; + auto result = co_await client.async_request( + std::move(url_path), method_type(req.get_method()), std::move(ctx), + std::move(req_headers)); + + for (auto &[k, v] : result.resp_headers) { + response.add_header(std::string(k), std::string(v)); + } + + response.set_status_and_content_view( + static_cast(result.status), result.resp_body); + co_await response.get_conn()->reply(); + response.set_delay(true); + } + private: std::unique_ptr pool_; asio::io_context *out_ctx_ = nullptr; diff --git a/include/ylt/thirdparty/cinatra/coro_radix_tree.hpp b/include/ylt/thirdparty/cinatra/coro_radix_tree.hpp index 585446b87..14f76f22c 100644 --- a/include/ylt/thirdparty/cinatra/coro_radix_tree.hpp +++ b/include/ylt/thirdparty/cinatra/coro_radix_tree.hpp @@ -42,8 +42,8 @@ struct coro_handler_t { struct radix_tree_node { std::string path; - std::vector handlers; - std::vector coro_handlers; + handler_t handler; + coro_handler_t coro_handler; std::string indices; std::vector> children; int max_params; @@ -54,21 +54,18 @@ struct radix_tree_node { std::function get_handler(const std::string &method) { - for (auto &h : this->handlers) { - if (h.method == method) { - return h.handler; - } + if (handler.method == method) { + return handler.handler; } + return nullptr; } std::function(coro_http_request &req, coro_http_response &resp)> get_coro_handler(const std::string &method) { - for (auto &h : this->coro_handlers) { - if (h.method == method) { - return h.coro_handler; - } + if (coro_handler.method == method) { + return coro_handler.coro_handler; } return nullptr; } @@ -76,22 +73,17 @@ struct radix_tree_node { int add_handler( std::function handler, - const std::vector &methods) { - for (auto &m : methods) { - auto old_handler = this->get_handler(m); - this->handlers.push_back(handler_t{m, handler}); - } + const std::string &method) { + this->handler = handler_t{method, handler}; + return 0; } int add_coro_handler(std::function( coro_http_request &req, coro_http_response &resp)> coro_handler, - const std::vector &methods) { - for (auto &m : methods) { - auto old_coro_handler = this->get_coro_handler(m); - this->coro_handlers.push_back(coro_handler_t{m, coro_handler}); - } + const std::string &method) { + this->coro_handler = coro_handler_t{method, coro_handler}; return 0; } @@ -134,7 +126,7 @@ class radix_tree { const std::string &path, std::function handler, - const std::vector &methods) { + const std::string &method) { auto root = this->root; int i = 0, n = path.size(), param_count = 0, code = 0; while (i < n) { @@ -166,7 +158,7 @@ class radix_tree { ++param_count; } - code = root->add_handler(handler, methods); + code = root->add_handler(handler, method); break; } @@ -181,7 +173,7 @@ class radix_tree { ++param_count; if (i == n) { - code = root->add_handler(handler, methods); + code = root->add_handler(handler, method); break; } } @@ -193,7 +185,7 @@ class radix_tree { i += root->path.size() + 1; if (i == n) { - code = root->add_handler(handler, methods); + code = root->add_handler(handler, method); break; } } @@ -207,18 +199,18 @@ class radix_tree { if (j < m) { std::shared_ptr child( std::make_shared(root->path.substr(j))); - child->handlers = root->handlers; + child->handler = root->handler; child->indices = root->indices; child->children = root->children; root->path = root->path.substr(0, j); - root->handlers = {}; + root->handler = {}; root->indices = child->path[0]; root->children = {child}; } if (i == n) { - code = root->add_handler(handler, methods); + code = root->add_handler(handler, method); break; } } @@ -235,7 +227,7 @@ class radix_tree { std::function( coro_http_request &req, coro_http_response &resp)> coro_handler, - const std::vector &methods) { + std::string &method) { auto root = this->root; int i = 0, n = path.size(), param_count = 0, code = 0; while (i < n) { @@ -267,7 +259,7 @@ class radix_tree { ++param_count; } - code = root->add_coro_handler(coro_handler, methods); + code = root->add_coro_handler(coro_handler, method); break; } @@ -282,7 +274,7 @@ class radix_tree { ++param_count; if (i == n) { - code = root->add_coro_handler(coro_handler, methods); + code = root->add_coro_handler(coro_handler, method); break; } } @@ -294,7 +286,7 @@ class radix_tree { i += root->path.size() + 1; if (i == n) { - code = root->add_coro_handler(coro_handler, methods); + code = root->add_coro_handler(coro_handler, method); break; } } @@ -308,18 +300,18 @@ class radix_tree { if (j < m) { std::shared_ptr child( std::make_shared(root->path.substr(j))); - child->handlers = root->handlers; + child->handler = root->handler; child->indices = root->indices; child->children = root->children; root->path = root->path.substr(0, j); - root->handlers = {}; + root->handler = {}; root->indices = child->path[0]; root->children = {child}; } if (i == n) { - code = root->add_coro_handler(coro_handler, methods); + code = root->add_coro_handler(coro_handler, method); break; } } diff --git a/include/ylt/thirdparty/cinatra/define.h b/include/ylt/thirdparty/cinatra/define.h index 2a3cf3d84..d663c4067 100644 --- a/include/ylt/thirdparty/cinatra/define.h +++ b/include/ylt/thirdparty/cinatra/define.h @@ -8,16 +8,16 @@ using namespace std::string_view_literals; namespace cinatra { enum class http_method { - UNKNOW, - DEL, + NIL = 0, GET, HEAD, POST, PUT, + TRACE, PATCH, CONNECT, OPTIONS, - TRACE + DEL, }; constexpr inline auto GET = http_method::GET; constexpr inline auto POST = http_method::POST; @@ -52,10 +52,18 @@ inline constexpr std::string_view method_name(http_method mthd) { case cinatra::http_method::TRACE: return "TRACE"sv; default: - return "UNKONWN"sv; + return "NIL"sv; } } +inline constexpr std::array method_table = { + 3, 1, 9, 0, 0, 0, 4, 5, 0, 0, 8, 0, 0, 0, 2, 0, 0, 0, 6, 7}; + +inline constexpr http_method method_type(std::string_view mthd) { + int index = ((mthd[0] & ~0x20) ^ ((mthd[1] + 1) & ~0x20)) % 20; + return (http_method)method_table[index]; +} + enum class transfer_type { CHUNKED, ACCEPT_RANGES }; enum class content_type { diff --git a/include/ylt/thirdparty/cinatra/http_parser.hpp b/include/ylt/thirdparty/cinatra/http_parser.hpp index 6c2fa898b..15e1e2f55 100644 --- a/include/ylt/thirdparty/cinatra/http_parser.hpp +++ b/include/ylt/thirdparty/cinatra/http_parser.hpp @@ -8,6 +8,7 @@ #include #include "cinatra_log_wrapper.hpp" +#include "define.h" #include "picohttpparser.h" #include "url_encode_decode.hpp" @@ -64,9 +65,12 @@ class http_parser { size_t method_len; const char *url; size_t url_len; + + bool has_query{}; header_len_ = detail::phr_parse_request( data, size, &method, &method_len, &url, &url_len, &minor_version, - headers_.data(), &num_headers_, last_len); + headers_.data(), &num_headers_, last_len, has_connection_, has_close_, + has_upgrade_, has_query); if (header_len_ < 0) [[unlikely]] { CINATRA_LOG_WARNING << "parse http head failed"; @@ -76,21 +80,28 @@ class http_parser { << ", you can define macro " "CINATRA_MAX_HTTP_HEADER_FIELD_SIZE to expand it."; } + return header_len_; } method_ = {method, method_len}; url_ = {url, url_len}; - auto content_len = this->get_header_value("content-length"sv); - if (content_len.empty()) { + auto methd_type = method_type(method_); + if (methd_type == http_method::GET || methd_type == http_method::HEAD) { body_len_ = 0; } else { - body_len_ = atoi(content_len.data()); + auto content_len = this->get_header_value("content-length"sv); + if (content_len.empty()) { + body_len_ = 0; + } + else { + body_len_ = atoi(content_len.data()); + } } - size_t pos = url_.find('?'); - if (pos != std::string_view::npos) { + if (has_query) { + size_t pos = url_.find('?'); parse_query(url_.substr(pos + 1, url_len - pos - 1)); url_ = {url, pos}; } @@ -98,6 +109,12 @@ class http_parser { return header_len_; } + bool has_connection() { return has_connection_; } + + bool has_close() { return has_close_; } + + bool has_upgrade() { return has_upgrade_; } + std::string_view get_header_value(std::string_view key) const { for (size_t i = 0; i < num_headers_; i++) { if (iequal0(headers_[i].name, key)) @@ -247,6 +264,9 @@ class http_parser { size_t num_headers_ = 0; int header_len_ = 0; int body_len_ = 0; + bool has_connection_{}; + bool has_close_{}; + bool has_upgrade_{}; std::array headers_; std::string_view method_; std::string_view url_; diff --git a/include/ylt/thirdparty/cinatra/picohttpparser.h b/include/ylt/thirdparty/cinatra/picohttpparser.h index 031480cf5..310044b3d 100644 --- a/include/ylt/thirdparty/cinatra/picohttpparser.h +++ b/include/ylt/thirdparty/cinatra/picohttpparser.h @@ -808,7 +808,9 @@ static const char *parse_headers(const char *buf, const char *buf_end, static const char *parse_headers(const char *buf, const char *buf_end, http_header *headers, size_t *num_headers, - size_t max_headers, int *ret) { + size_t max_headers, int *ret, + bool &has_connection, bool &has_close, + bool &has_upgrade) { for (;; ++*num_headers) { const char *name; size_t name_len; @@ -877,6 +879,21 @@ static const char *parse_headers(const char *buf, const char *buf_end, NULL) { return NULL; } + if (name_len == 10) { + if (memcmp(name + 1, "onnection", name_len - 1) == 0) { + // has connection + has_connection = true; + char ch = *value; + if (ch == 'U') { + // has upgrade + has_upgrade = true; + } + else if (ch == 'c' || ch == 'C') { + // has_close + has_close = true; + } + } + } headers[*num_headers] = {std::string_view{name, name_len}, std::string_view{value, value_len}}; } @@ -885,12 +902,40 @@ static const char *parse_headers(const char *buf, const char *buf_end, #endif -static const char *parse_request(const char *buf, const char *buf_end, - const char **method, size_t *method_len, - const char **path, size_t *path_len, - int *minor_version, http_header *headers, - size_t *num_headers, size_t max_headers, - int *ret) { +#define ADVANCE_PATH(tok, toklen, has_query) \ + do { \ + const char *tok_start = buf; \ + static const char ALIGNED(16) ranges2[] = "\000\040\177\177"; \ + int found2; \ + buf = findchar_fast(buf, buf_end, ranges2, sizeof(ranges2) - 1, &found2); \ + if (!found2) { \ + CHECK_EOF(); \ + } \ + while (1) { \ + if (*buf == ' ') { \ + break; \ + } \ + else if (unlikely(!IS_PRINTABLE_ASCII(*buf))) { \ + if ((unsigned char)*buf < '\040' || *buf == '\177') { \ + *ret = -1; \ + return NULL; \ + } \ + } \ + else if (unlikely(*buf == '?')) { \ + has_query = true; \ + } \ + ++buf; \ + CHECK_EOF(); \ + } \ + tok = tok_start; \ + toklen = buf - tok_start; \ + } while (0) + +static const char *parse_request( + const char *buf, const char *buf_end, const char **method, + size_t *method_len, const char **path, size_t *path_len, int *minor_version, + http_header *headers, size_t *num_headers, size_t max_headers, int *ret, + bool &has_connection, bool &has_close, bool &has_upgrade, bool &has_query) { /* skip first empty line (some clients add CRLF after POST content) */ CHECK_EOF(); if (*buf == '\015') { @@ -904,7 +949,7 @@ static const char *parse_request(const char *buf, const char *buf_end, /* parse request line */ ADVANCE_TOKEN(*method, *method_len); ++buf; - ADVANCE_TOKEN(*path, *path_len); + ADVANCE_PATH(*path, *path_len, has_query); ++buf; if ((buf = parse_http_version(buf, buf_end, minor_version, ret)) == NULL) { return NULL; @@ -921,14 +966,17 @@ static const char *parse_request(const char *buf, const char *buf_end, return NULL; } - return parse_headers(buf, buf_end, headers, num_headers, max_headers, ret); + return parse_headers(buf, buf_end, headers, num_headers, max_headers, ret, + has_connection, has_close, has_upgrade); } inline int phr_parse_request(const char *buf_start, size_t len, const char **method, size_t *method_len, const char **path, size_t *path_len, int *minor_version, http_header *headers, - size_t *num_headers, size_t last_len) { + size_t *num_headers, size_t last_len, + bool &has_connection, bool &has_close, + bool &has_upgrade, bool &has_query) { const char *buf = buf_start, *buf_end = buf_start + len; size_t max_headers = *num_headers; int r; @@ -948,7 +996,8 @@ inline int phr_parse_request(const char *buf_start, size_t len, if ((buf = parse_request(buf + last_len, buf_end, method, method_len, path, path_len, minor_version, headers, num_headers, - max_headers, &r)) == NULL) { + max_headers, &r, has_connection, has_close, + has_upgrade, has_query)) == NULL) { return r; } @@ -987,7 +1036,10 @@ inline const char *parse_response(const char *buf, const char *buf_end, return NULL; } - return parse_headers(buf, buf_end, headers, num_headers, max_headers, ret); + bool has_connection, has_close, has_upgrade; + + return parse_headers(buf, buf_end, headers, num_headers, max_headers, ret, + has_connection, has_close, has_upgrade); } inline int phr_parse_response(const char *buf_start, size_t len, @@ -1033,8 +1085,9 @@ inline int phr_parse_headers(const char *buf_start, size_t len, return r; } - if ((buf = parse_headers(buf, buf_end, headers, num_headers, max_headers, - &r)) == NULL) { + bool has_connection, has_close, has_upgrade; + if ((buf = parse_headers(buf, buf_end, headers, num_headers, max_headers, &r, + has_connection, has_close, has_upgrade)) == NULL) { return r; } diff --git a/include/ylt/thirdparty/cinatra/string_resize.hpp b/include/ylt/thirdparty/cinatra/string_resize.hpp index bb000f361..7822cfa32 100644 --- a/include/ylt/thirdparty/cinatra/string_resize.hpp +++ b/include/ylt/thirdparty/cinatra/string_resize.hpp @@ -8,7 +8,7 @@ namespace cinatra::detail { #if __cpp_lib_string_resize_and_overwrite >= 202110L template inline void resize(std::basic_string &str, std::size_t sz) { - str.resize_and_overwrite(sz, [](ch *, std::size_t sz) { + str.resize_and_overwrite(sz, [sz](ch *, std::size_t) { return sz; }); } diff --git a/include/ylt/thirdparty/cinatra/uri.hpp b/include/ylt/thirdparty/cinatra/uri.hpp index 409432561..4c1aedaa7 100644 --- a/include/ylt/thirdparty/cinatra/uri.hpp +++ b/include/ylt/thirdparty/cinatra/uri.hpp @@ -294,7 +294,7 @@ struct context { std::string path; std::string query; std::string body; - http_method method = http_method::UNKNOW; + http_method method = http_method::NIL; context() = default; context(const uri_t &u, http_method mthd) diff --git a/include/ylt/thirdparty/iguana/detail/utf.hpp b/include/ylt/thirdparty/iguana/detail/utf.hpp index a6fa7012a..e0dae985f 100644 --- a/include/ylt/thirdparty/iguana/detail/utf.hpp +++ b/include/ylt/thirdparty/iguana/detail/utf.hpp @@ -3,6 +3,8 @@ #include #include +#include "iguana/define.h" + namespace iguana { // https://github.com/Tencent/rapidjson/blob/master/include/rapidjson/reader.h template @@ -48,4 +50,101 @@ inline void encode_utf8(OutputStream &os, unsigned codepoint) { os.push_back(static_cast(0x80 | (codepoint & 0x3F))); } } + +// https://github.com/Tencent/rapidjson/blob/master/include/rapidjson/encodings.h +static inline unsigned char GetRange(unsigned char c) { + static const unsigned char type[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0x10, 0x10, 0x10, 0x10, + 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, + 0x40, 0x40, 0x40, 0x40, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 8, 8, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 10, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, + 11, 6, 6, 6, 5, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, + }; + return type[c]; +} + +// https://github.com/Tencent/rapidjson/blob/master/include/rapidjson/encodings.h +template +inline bool decode_utf8(It &&it, unsigned &codepoint) { + auto c = *(it++); + bool result = true; + auto copy = [&]() IGUANA__INLINE_LAMBDA { + c = *(it++); + codepoint = (codepoint << 6) | (static_cast(c) & 0x3Fu); + }; + auto trans = [&](unsigned mask) IGUANA__INLINE_LAMBDA { + result &= ((GetRange(static_cast(c)) & mask) != 0); + }; + auto tail = [&]() IGUANA__INLINE_LAMBDA { + copy(); + trans(0x70); + }; + if (!(c & 0x80)) { + codepoint = static_cast(c); + return true; + } + unsigned char type = GetRange(static_cast(c)); + if (type >= 32) { + codepoint = 0; + } + else { + codepoint = (0xFFu >> type) & static_cast(c); + } + switch (type) { + case 2: + tail(); + return result; + case 3: + tail(); + tail(); + return result; + case 4: + copy(); + trans(0x50); + tail(); + return result; + case 5: + copy(); + trans(0x10); + tail(); + tail(); + return result; + case 6: + tail(); + tail(); + tail(); + return result; + case 10: + copy(); + trans(0x20); + tail(); + return result; + case 11: + copy(); + trans(0x60); + tail(); + tail(); + return result; + default: + return false; + } +} + } // namespace iguana diff --git a/include/ylt/thirdparty/iguana/json_reader.hpp b/include/ylt/thirdparty/iguana/json_reader.hpp index dfac6e094..1597d93bf 100644 --- a/include/ylt/thirdparty/iguana/json_reader.hpp +++ b/include/ylt/thirdparty/iguana/json_reader.hpp @@ -11,13 +11,13 @@ namespace detail { template , int> = 0> -IGUANA_INLINE void parse_item(U &value, It &&it, It &&end); +IGUANA_INLINE void from_json_impl(U &value, It &&it, It &&end); template , int> = 0> -IGUANA_INLINE void parse_item(U &value, It &&it, It &&end); +IGUANA_INLINE void from_json_impl(U &value, It &&it, It &&end); template , int> = 0> -IGUANA_INLINE void parse_item(U &value, It &&it, It &&end) { +IGUANA_INLINE void from_json_impl(U &value, It &&it, It &&end) { from_json(value, it, end); } @@ -61,7 +61,7 @@ IGUANA_INLINE void parse_escape(U &value, It &&it, It &&end) { } template , int> = 0> -IGUANA_INLINE void parse_item(U &value, It &&it, It &&end) { +IGUANA_INLINE void from_json_impl(U &value, It &&it, It &&end) { skip_ws(it, end); if constexpr (contiguous_iterator>) { const auto size = std::distance(it, end); @@ -89,7 +89,7 @@ IGUANA_INLINE void parse_item(U &value, It &&it, It &&end) { } template , int> = 0> -IGUANA_INLINE void parse_item(U &value, It &&it, It &&end) { +IGUANA_INLINE void from_json_impl(U &value, It &&it, It &&end) { skip_ws(it, end); auto start = it; while (it != end && is_numeric(*it)) { @@ -101,7 +101,7 @@ IGUANA_INLINE void parse_item(U &value, It &&it, It &&end) { template , int> = 0> -IGUANA_INLINE void parse_item(U &value, It &&it, It &&end) { +IGUANA_INLINE void from_json_impl(U &value, It &&it, It &&end) { if constexpr (!skip) { skip_ws(it, end); match<'"'>(it, end); @@ -140,7 +140,7 @@ IGUANA_INLINE void parse_item(U &value, It &&it, It &&end) { } template , int> = 0> -IGUANA_INLINE void parse_item(U &&value, It &&it, It &&end) { +IGUANA_INLINE void from_json_impl(U &&value, It &&it, It &&end) { skip_ws(it, end); if (it < end) @@ -166,7 +166,7 @@ IGUANA_INLINE void parse_item(U &&value, It &&it, It &&end) { template , int> = 0> -IGUANA_INLINE void parse_item(U &value, It &&it, It &&end) { +IGUANA_INLINE void from_json_impl(U &value, It &&it, It &&end) { if constexpr (!skip) { skip_ws(it, end); match<'"'>(it, end); @@ -208,7 +208,7 @@ IGUANA_INLINE void parse_item(U &value, It &&it, It &&end) { template , int> = 0> -IGUANA_INLINE void parse_item(U &value, It &&it, It &&end) { +IGUANA_INLINE void from_json_impl(U &value, It &&it, It &&end) { static_assert(contiguous_iterator>, "must be contiguous"); if constexpr (!skip) { skip_ws(it, end); @@ -229,16 +229,16 @@ IGUANA_INLINE void parse_item(U &value, It &&it, It &&end) { } template , int> = 0> -IGUANA_INLINE void parse_item(U &value, It &&it, It &&end) { +IGUANA_INLINE void from_json_impl(U &value, It &&it, It &&end) { static constexpr auto str_to_enum = get_enum_map>(); if constexpr (bool_v) { // not defined a specialization template using T = std::underlying_type_t>; - parse_item(reinterpret_cast(value), it, end); + from_json_impl(reinterpret_cast(value), it, end); } else { std::string_view enum_names; - parse_item(enum_names, it, end); + from_json_impl(enum_names, it, end); auto it = str_to_enum.find(enum_names); if (it != str_to_enum.end()) IGUANA_LIKELY { value = it->second; } @@ -250,7 +250,7 @@ IGUANA_INLINE void parse_item(U &value, It &&it, It &&end) { } template , int> = 0> -IGUANA_INLINE void parse_item(U &value, It &&it, It &&end) { +IGUANA_INLINE void from_json_impl(U &value, It &&it, It &&end) { using T = std::remove_reference_t; constexpr auto n = sizeof(T) / sizeof(decltype(std::declval()[0])); skip_ws(it, end); @@ -262,7 +262,7 @@ IGUANA_INLINE void parse_item(U &value, It &&it, It &&end) { auto value_it = std::begin(value); for (size_t i = 0; i < n; ++i) { if (*it != '"') - IGUANA_LIKELY { parse_item(*value_it++, it, end); } + IGUANA_LIKELY { from_json_impl(*value_it++, it, end); } } match<'"'>(it, end); return; @@ -280,7 +280,7 @@ IGUANA_INLINE void parse_item(U &value, It &&it, It &&end) { } auto value_it = std::begin(value); for (size_t i = 0; i < n; ++i) { - parse_item(*value_it++, it, end); + from_json_impl(*value_it++, it, end); skip_ws(it, end); if (it == end) { throw std::runtime_error("Unexpected end"); @@ -301,7 +301,7 @@ IGUANA_INLINE void parse_item(U &value, It &&it, It &&end) { template , int>> -IGUANA_INLINE void parse_item(U &value, It &&it, It &&end) { +IGUANA_INLINE void from_json_impl(U &value, It &&it, It &&end) { value.clear(); skip_ws(it, end); @@ -316,7 +316,7 @@ IGUANA_INLINE void parse_item(U &value, It &&it, It &&end) { if (i > 0) IGUANA_LIKELY { match<','>(it, end); } - parse_item(value.emplace_back(), it, end); + from_json_impl(value.emplace_back(), it, end); skip_ws(it, end); } throw std::runtime_error("Expected ]"); @@ -337,7 +337,7 @@ IGUANA_INLINE auto get_key(It &&it, It &&end) { // compile time versions of keys it = start; static thread_local std::string static_key{}; - detail::parse_item(static_key, it, end); + detail::from_json_impl(static_key, it, end); return std::string_view(static_key); } else @@ -352,14 +352,14 @@ IGUANA_INLINE auto get_key(It &&it, It &&end) { } else { static thread_local std::string static_key{}; - detail::parse_item(static_key, it, end); + detail::from_json_impl(static_key, it, end); return std::string_view(static_key); } } template , int> = 0> -IGUANA_INLINE void parse_item(U &value, It &&it, It &&end) { +IGUANA_INLINE void from_json_impl(U &value, It &&it, It &&end) { using T = std::remove_reference_t; using key_type = typename T::key_type; skip_ws(it, end); @@ -384,19 +384,19 @@ IGUANA_INLINE void parse_item(U &value, It &&it, It &&end) { match<':'>(it, end); if constexpr (string_v || string_view_v) { - parse_item(value[key_type(key)], it, end); + from_json_impl(value[key_type(key)], it, end); } else { static thread_local key_type key_value{}; - parse_item(key_value, key.begin(), key.end()); - parse_item(value[key_value], it, end); + from_json_impl(key_value, key.begin(), key.end()); + from_json_impl(value[key_value], it, end); } skip_ws(it, end); } } template , int> = 0> -IGUANA_INLINE void parse_item(U &value, It &&it, It &&end) { +IGUANA_INLINE void from_json_impl(U &value, It &&it, It &&end) { skip_ws(it, end); match<'['>(it, end); skip_ws(it, end); @@ -410,7 +410,7 @@ IGUANA_INLINE void parse_item(U &value, It &&it, It &&end) { match<','>(it, end); skip_ws(it, end); } - parse_item(v, it, end); + from_json_impl(v, it, end); skip_ws(it, end); }); @@ -418,7 +418,7 @@ IGUANA_INLINE void parse_item(U &value, It &&it, It &&end) { } template , int> = 0> -IGUANA_INLINE void parse_item(U &value, It &&it, It &&end) { +IGUANA_INLINE void from_json_impl(U &value, It &&it, It &&end) { skip_ws(it, end); if (it < end && *it == '"') IGUANA_LIKELY { ++it; } @@ -439,17 +439,17 @@ IGUANA_INLINE void parse_item(U &value, It &&it, It &&end) { using value_type = typename T::value_type; value_type t; if constexpr (string_v || string_view_v) { - parse_item(t, it, end); + from_json_impl(t, it, end); } else { - parse_item(t, it, end); + from_json_impl(t, it, end); } value = std::move(t); } } template , int>> -IGUANA_INLINE void parse_item(U &value, It &&it, It &&end) { +IGUANA_INLINE void from_json_impl(U &value, It &&it, It &&end) { skip_ws(it, end); if (it == end) IGUANA_UNLIKELY { throw std::runtime_error("Unexexpected eof"); } @@ -465,7 +465,7 @@ IGUANA_INLINE void parse_item(U &value, It &&it, It &&end) { else { value = std::make_shared(); } - parse_item(*value, it, end); + from_json_impl(*value, it, end); } } @@ -522,7 +522,10 @@ IGUANA_INLINE void from_json(T &value, It &&it, It &&end) { IGUANA_UNLIKELY { return; } skip_ws(it, end); match<':'>(it, end); - detail::parse_item(value.*member_ptr, it, end); + { + using namespace detail; + from_json_impl(value.*member_ptr, it, end); + } skip_ws(it, end); if (*it == '}') @@ -551,7 +554,8 @@ IGUANA_INLINE void from_json(T &value, It &&it, It &&end) { [&](auto &&member_ptr) IGUANA__INLINE_LAMBDA { using V = std::decay_t; if constexpr (std::is_member_pointer_v) { - detail::parse_item(value.*member_ptr, it, end); + using namespace detail; + from_json_impl(value.*member_ptr, it, end); } else { static_assert(!sizeof(V), "type not supported"); @@ -583,7 +587,8 @@ IGUANA_INLINE void from_json(T &value, It &&it, It &&end) { template , int> = 0> IGUANA_INLINE void from_json(T &value, It &&it, It &&end) { - detail::parse_item(value, it, end); + using namespace detail; + from_json_impl(value, it, end); } template @@ -635,10 +640,11 @@ IGUANA_INLINE void from_json(T &value, const Byte *data, size_t size, } template -void parse(jvalue &result, It &&it, It &&end); +void parse(jvalue &result, It &&it, It &&end, bool parse_as_double = false); template -inline void parse(jarray &result, It &&it, It &&end) { +inline void parse(jarray &result, It &&it, It &&end, + bool parse_as_double = false) { skip_ws(it, end); match<'['>(it, end); if (*it == ']') @@ -652,7 +658,7 @@ inline void parse(jarray &result, It &&it, It &&end) { } result.emplace_back(); - parse(result.back(), it, end); + parse(result.back(), it, end, parse_as_double); if (*it == ']') IGUANA_UNLIKELY { @@ -666,7 +672,8 @@ inline void parse(jarray &result, It &&it, It &&end) { } template -inline void parse(jobject &result, It &&it, It &&end) { +inline void parse(jobject &result, It &&it, It &&end, + bool parse_as_double = false) { skip_ws(it, end); match<'{'>(it, end); if (*it == '}') @@ -681,7 +688,8 @@ inline void parse(jobject &result, It &&it, It &&end) { if (it == end) IGUANA_UNLIKELY { throw std::runtime_error("Expected }"); } std::string key; - detail::parse_item(key, it, end); + using namespace detail; + from_json_impl(key, it, end); auto emplaced = result.try_emplace(key); if (!emplaced.second) @@ -689,7 +697,7 @@ inline void parse(jobject &result, It &&it, It &&end) { match<':'>(it, end); - parse(emplaced.first->second, it, end); + parse(emplaced.first->second, it, end, parse_as_double); if (*it == '}') IGUANA_UNLIKELY { @@ -702,7 +710,8 @@ inline void parse(jobject &result, It &&it, It &&end) { } template -inline void parse(jvalue &result, It &&it, It &&end) { +inline void parse(jvalue &result, It &&it, It &&end, bool parse_as_double) { + using namespace detail; skip_ws(it, end); switch (*it) { case 'n': @@ -713,7 +722,7 @@ inline void parse(jvalue &result, It &&it, It &&end) { case 'f': case 't': - detail::parse_item(result.template emplace(), it, end); + from_json_impl(result.template emplace(), it, end); break; case '0': case '1': @@ -727,8 +736,8 @@ inline void parse(jvalue &result, It &&it, It &&end) { case '9': case '-': { double d{}; - detail::parse_item(d, it, end); - if (static_cast(d) == d) + from_json_impl(d, it, end); + if (!parse_as_double && (static_cast(d) == d)) result.emplace(d); else result.emplace(d); @@ -737,20 +746,20 @@ inline void parse(jvalue &result, It &&it, It &&end) { case '"': if constexpr (Is_view) { result.template emplace(); - detail::parse_item(std::get(result), it, end); + from_json_impl(std::get(result), it, end); } else { result.template emplace(); - detail::parse_item(std::get(result), it, end); + from_json_impl(std::get(result), it, end); } break; case '[': result.template emplace(); - parse(std::get(result), it, end); + parse(std::get(result), it, end, parse_as_double); break; case '{': { result.template emplace(); - parse(std::get(result), it, end); + parse(std::get(result), it, end, parse_as_double); break; } default: @@ -760,11 +769,13 @@ inline void parse(jvalue &result, It &&it, It &&end) { skip_ws(it, end); } -// when Is_view is true, parse str as string_view +// set Is_view == true, parse str as std::string_view +// set parse_as_double == true, parse the number as double in any case template -inline void parse(jvalue &result, It &&it, It &&end, std::error_code &ec) { +inline void parse(jvalue &result, It &&it, It &&end, std::error_code &ec, + bool parse_as_double = false) { try { - parse(result, it, end); + parse(result, it, end, parse_as_double); ec = {}; } catch (const std::runtime_error &e) { result.template emplace(); @@ -774,15 +785,16 @@ inline void parse(jvalue &result, It &&it, It &&end, std::error_code &ec) { template , int> = 0> -inline void parse(T &result, const View &view) { - parse(result, std::begin(view), std::end(view)); +inline void parse(T &result, const View &view, bool parse_as_double = false) { + parse(result, std::begin(view), std::end(view), parse_as_double); } template , int> = 0> -inline void parse(T &result, const View &view, std::error_code &ec) noexcept { +inline void parse(T &result, const View &view, std::error_code &ec, + bool parse_as_double = false) noexcept { try { - parse(result, view); + parse(result, view, parse_as_double); ec = {}; } catch (std::runtime_error &e) { ec = iguana::make_error_code(e.what()); diff --git a/include/ylt/thirdparty/iguana/json_writer.hpp b/include/ylt/thirdparty/iguana/json_writer.hpp index 3d77ff638..80e2558da 100644 --- a/include/ylt/thirdparty/iguana/json_writer.hpp +++ b/include/ylt/thirdparty/iguana/json_writer.hpp @@ -8,41 +8,42 @@ namespace iguana { -template , int> = 0> IGUANA_INLINE void to_json(T &&t, Stream &s); +namespace detail { +template +IGUANA_INLINE void to_json_impl(Stream &ss, std::optional &val); -template -IGUANA_INLINE void render_json_value(Stream &ss, std::optional &val); - -template , int> = 0> -IGUANA_INLINE void render_json_value(Stream &ss, const T &t); +IGUANA_INLINE void to_json_impl(Stream &ss, const T &t); -template , int> = 0> -IGUANA_INLINE void render_json_value(Stream &ss, const T &v); +IGUANA_INLINE void to_json_impl(Stream &ss, const T &v); -template , int> = 0> -IGUANA_INLINE void render_json_value(Stream &ss, const T &v); +IGUANA_INLINE void to_json_impl(Stream &ss, const T &v); -template , int> = 0> -IGUANA_INLINE void render_json_value(Stream &ss, const T &o); +IGUANA_INLINE void to_json_impl(Stream &ss, const T &o); -template , int> = 0> -IGUANA_INLINE void render_json_value(Stream &s, T &&t); +template , int> = 0> +IGUANA_INLINE void to_json_impl(Stream &s, T &&t); -template , int> = 0> -IGUANA_INLINE void render_json_value(Stream &s, T &&t); +template , int> = 0> +IGUANA_INLINE void to_json_impl(Stream &s, T &&t); template IGUANA_INLINE void join(Stream &ss, InputIt first, InputIt last, const T &delim, const F &f) { if (first == last) return; - f(*first++); while (first != last) { ss.push_back(delim); @@ -50,75 +51,85 @@ IGUANA_INLINE void join(Stream &ss, InputIt first, InputIt last, const T &delim, } } -template -IGUANA_INLINE void render_json_value(Stream &ss, std::nullptr_t) { +template +IGUANA_INLINE void to_json_impl(Stream &ss, std::nullptr_t) { ss.append("null"); } -template -IGUANA_INLINE void render_json_value(Stream &ss, bool b) { +template +IGUANA_INLINE void to_json_impl(Stream &ss, bool b) { ss.append(b ? "true" : "false"); }; -template -IGUANA_INLINE void render_json_value(Stream &ss, char value) { +template +IGUANA_INLINE void to_json_impl(Stream &ss, char value) { ss.append("\""); ss.push_back(value); ss.append("\""); } -template , int> = 0> -IGUANA_INLINE void render_json_value(Stream &ss, T value) { +template , int> = 0> +IGUANA_INLINE void to_json_impl(Stream &ss, T value) { char temp[65]; auto p = detail::to_chars(temp, value); ss.append(temp, p - temp); } -template , int> = 0> -IGUANA_INLINE void render_json_value(Stream &ss, T v) { +IGUANA_INLINE void to_json_impl(Stream &ss, T v) { ss.append(v.value().data(), v.value().size()); } -template , int> = 0> -IGUANA_INLINE void render_json_value(Stream &ss, T &&t) { +IGUANA_INLINE void to_json_impl(Stream &ss, T &&t) { ss.push_back('"'); - ss.append(t.data(), t.size()); + if constexpr (Is_writing_escape) { + write_string_with_escape(t.data(), t.size(), ss); + } + else { + ss.append(t.data(), t.size()); + } ss.push_back('"'); } -template , int> = 0> +template , int> = 0> IGUANA_INLINE void render_key(Stream &ss, T &t) { ss.push_back('"'); - render_json_value(ss, t); + to_json_impl(ss, t); ss.push_back('"'); } -template , int> = 0> IGUANA_INLINE void render_key(Stream &ss, T &&t) { - render_json_value(ss, std::forward(t)); + to_json_impl(ss, std::forward(t)); } -template , int> = 0> -IGUANA_INLINE void render_json_value(Stream &ss, T &&t) { +IGUANA_INLINE void to_json_impl(Stream &ss, T &&t) { to_json(std::forward(t), ss); } -template , int> = 0> -IGUANA_INLINE void render_json_value(Stream &ss, T val) { +template , int> = 0> +IGUANA_INLINE void to_json_impl(Stream &ss, T val) { static constexpr auto enum_to_str = get_enum_map>(); if constexpr (bool_v) { - render_json_value(ss, static_cast>(val)); + to_json_impl( + ss, static_cast>(val)); } else { auto it = enum_to_str.find(val); if (it != enum_to_str.end()) IGUANA_LIKELY { auto str = it->second; - render_json_value(ss, std::string_view(str.data(), str.size())); + to_json_impl( + ss, std::string_view(str.data(), str.size())); } else { throw std::runtime_error( @@ -128,28 +139,29 @@ IGUANA_INLINE void render_json_value(Stream &ss, T val) { } } -template -IGUANA_INLINE void render_json_value(Stream &ss, std::optional &val) { +template +IGUANA_INLINE void to_json_impl(Stream &ss, std::optional &val) { if (!val) { ss.append("null"); } else { - render_json_value(ss, *val); + to_json_impl(ss, *val); } } -template +template IGUANA_INLINE void render_array(Stream &ss, const T &v) { ss.push_back('['); join(ss, std::begin(v), std::end(v), ',', [&ss](const auto &jsv) IGUANA__INLINE_LAMBDA { - render_json_value(ss, jsv); + to_json_impl(ss, jsv); }); ss.push_back(']'); } -template , int>> -IGUANA_INLINE void render_json_value(Stream &ss, const T &t) { +template , int>> +IGUANA_INLINE void to_json_impl(Stream &ss, const T &t) { if constexpr (std::is_same_v()[0])>>) { constexpr size_t n = sizeof(T) / sizeof(decltype(std::declval()[0])); @@ -166,30 +178,30 @@ IGUANA_INLINE void render_json_value(Stream &ss, const T &t) { ss.push_back('"'); } else { - render_array(ss, t); + render_array(ss, t); } } -template , int>> -IGUANA_INLINE void render_json_value(Stream &ss, const T &o) { +IGUANA_INLINE void to_json_impl(Stream &ss, const T &o) { ss.push_back('{'); join(ss, o.cbegin(), o.cend(), ',', [&ss](const auto &jsv) IGUANA__INLINE_LAMBDA { - render_key(ss, jsv.first); + render_key(ss, jsv.first); ss.push_back(':'); - render_json_value(ss, jsv.second); + to_json_impl(ss, jsv.second); }); ss.push_back('}'); } -template , int>> -IGUANA_INLINE void render_json_value(Stream &ss, const T &v) { +IGUANA_INLINE void to_json_impl(Stream &ss, const T &v) { ss.push_back('['); join(ss, v.cbegin(), v.cend(), ',', [&ss](const auto &jsv) IGUANA__INLINE_LAMBDA { - render_json_value(ss, jsv); + to_json_impl(ss, jsv); }); ss.push_back(']'); } @@ -203,24 +215,26 @@ constexpr auto write_json_key = [](auto &s, auto i, s.push_back('"'); }; -template , int>> -IGUANA_INLINE void render_json_value(Stream &ss, const T &v) { +template , int>> +IGUANA_INLINE void to_json_impl(Stream &ss, const T &v) { if (v) { - render_json_value(ss, *v); + to_json_impl(ss, *v); } else { ss.append("null"); } } -template , int>> -IGUANA_INLINE void render_json_value(Stream &s, T &&t) { +template , int>> +IGUANA_INLINE void to_json_impl(Stream &s, T &&t) { using U = typename std::decay_t; s.push_back('['); constexpr size_t size = std::tuple_size_v; for_each(std::forward(t), [&s, size](auto &v, auto i) IGUANA__INLINE_LAMBDA { - render_json_value(s, v); + to_json_impl(s, v); if (i != size - 1) IGUANA_LIKELY { s.push_back(','); } @@ -228,17 +242,20 @@ IGUANA_INLINE void render_json_value(Stream &s, T &&t) { s.push_back(']'); } -template , int>> -IGUANA_INLINE void render_json_value(Stream &s, T &&t) { +template , int>> +IGUANA_INLINE void to_json_impl(Stream &s, T &&t) { std::visit( [&s](auto value) { - render_json_value(s, value); + to_json_impl(s, value); }, t); } - -template , int>> +} // namespace detail +template , int>> IGUANA_INLINE void to_json(T &&t, Stream &s) { + using namespace detail; s.push_back('{'); for_each(std::forward(t), [&t, &s](const auto &v, auto i) IGUANA__INLINE_LAMBDA { @@ -249,17 +266,18 @@ IGUANA_INLINE void to_json(T &&t, Stream &s) { write_json_key(s, i, t); s.push_back(':'); - render_json_value(s, t.*v); + to_json_impl(s, t.*v); if (Idx < Count - 1) IGUANA_LIKELY { s.push_back(','); } }); s.push_back('}'); } -template , int> = 0> IGUANA_INLINE void to_json(T &&t, Stream &s) { - render_json_value(s, t); + using namespace detail; + to_json_impl(s, t); } } // namespace iguana diff --git a/include/ylt/thirdparty/iguana/reflection.hpp b/include/ylt/thirdparty/iguana/reflection.hpp index a32f3e1a6..41e15423b 100644 --- a/include/ylt/thirdparty/iguana/reflection.hpp +++ b/include/ylt/thirdparty/iguana/reflection.hpp @@ -17,7 +17,6 @@ #include #include -#include "detail/itoa.hpp" #include "detail/string_stream.hpp" #include "detail/traits.hpp" #include "frozen/string.h" diff --git a/include/ylt/thirdparty/iguana/util.hpp b/include/ylt/thirdparty/iguana/util.hpp index b5ec2c3e0..f910db795 100644 --- a/include/ylt/thirdparty/iguana/util.hpp +++ b/include/ylt/thirdparty/iguana/util.hpp @@ -13,6 +13,7 @@ #include "define.h" #include "detail/charconv.h" +#include "detail/utf.hpp" #include "enum_reflection.hpp" #include "error_code.h" #include "reflection.hpp" @@ -71,7 +72,7 @@ constexpr inline bool map_container_v = is_map_container>::value; template -constexpr inline bool c_array_v = std::is_array_v> && +constexpr inline bool c_array_v = std::is_array_v>&& std::extent_v> > 0; template @@ -138,15 +139,12 @@ struct is_variant> : std::true_type {}; template constexpr inline bool variant_v = is_variant>::value; -template -constexpr inline bool non_refletable_v = - container_v || c_array_v || tuple_v || optional_v || - smart_ptr_v || std::is_fundamental_v> || - variant_v; - template constexpr inline bool refletable_v = is_reflection_v>; +template +constexpr inline bool non_refletable_v = !refletable_v; + template constexpr inline bool plain_v = string_container_v || num_v || char_v || bool_v || enum_v; @@ -175,7 +173,7 @@ template using underline_type_t = typename underline_type>::type; template -IGUANA_INLINE void match(It &&it, It &&end) { +IGUANA_INLINE void match(It&& it, It&& end) { const auto n = static_cast(std::distance(it, end)); if (n < sizeof...(C)) IGUANA_UNLIKELY { @@ -203,4 +201,84 @@ inline constexpr auto has_qoute = [](uint64_t chunk) IGUANA__INLINE_LAMBDA { 0b0010001000100010001000100010001000100010001000100010001000100010); }; +// https://github.com/Tencent/rapidjson/blob/master/include/rapidjson/writer.h +template +inline void write_string_with_escape(const Ch* it, SizeType length, + Stream& ss) { + static const char hexDigits[16] = {'0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'A', 'B', 'C', 'D', 'E', 'F'}; + static const char escape[256] = { +#define Z16 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + // 0 1 2 3 4 5 6 7 8 9 A B C D E + // F + 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'b', 't', + 'n', 'u', 'f', 'r', 'u', 'u', // 00 + 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', + 'u', 'u', 'u', 'u', 'u', 'u', // 10 + 0, 0, '"', 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, // 20 + Z16, Z16, // 30~4F + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, '\\', 0, 0, 0, // 50 + Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16 // 60~FF +#undef Z16 + }; + auto end = it; + std::advance(end, length); + while (it < end) { + if (static_cast(*it) >= 0x80) + IGUANA_UNLIKELY { + unsigned codepoint = 0; + if (!decode_utf8(it, codepoint)) + IGUANA_UNLIKELY { + throw std::runtime_error("illegal unicode character"); + } + ss.push_back('\\'); + ss.push_back('u'); + if (codepoint <= 0xD7FF || + (codepoint >= 0xE000 && codepoint <= 0xFFFF)) { + ss.push_back(hexDigits[(codepoint >> 12) & 15]); + ss.push_back(hexDigits[(codepoint >> 8) & 15]); + ss.push_back(hexDigits[(codepoint >> 4) & 15]); + ss.push_back(hexDigits[(codepoint)&15]); + } + else { + if (codepoint < 0x010000 || codepoint > 0x10FFFF) + IGUANA_UNLIKELY { throw std::runtime_error("illegal codepoint"); } + // Surrogate pair + unsigned s = codepoint - 0x010000; + unsigned lead = (s >> 10) + 0xD800; + unsigned trail = (s & 0x3FF) + 0xDC00; + ss.push_back(hexDigits[(lead >> 12) & 15]); + ss.push_back(hexDigits[(lead >> 8) & 15]); + ss.push_back(hexDigits[(lead >> 4) & 15]); + ss.push_back(hexDigits[(lead)&15]); + ss.push_back('\\'); + ss.push_back('u'); + ss.push_back(hexDigits[(trail >> 12) & 15]); + ss.push_back(hexDigits[(trail >> 8) & 15]); + ss.push_back(hexDigits[(trail >> 4) & 15]); + ss.push_back(hexDigits[(trail)&15]); + } + } + else if (escape[static_cast(*it)]) + IGUANA_UNLIKELY { + ss.push_back('\\'); + ss.push_back(escape[static_cast(*it)]); + + if (escape[static_cast(*it)] == 'u') { + // escape other control characters + ss.push_back('0'); + ss.push_back('0'); + ss.push_back(hexDigits[static_cast(*it) >> 4]); + ss.push_back(hexDigits[static_cast(*it) & 0xF]); + } + ++it; + } + else { + ss.push_back(*(it++)); + } + } +} + } // namespace iguana diff --git a/include/ylt/thirdparty/iguana/yaml_writer.hpp b/include/ylt/thirdparty/iguana/yaml_writer.hpp index b6228b99a..a0794f2c3 100644 --- a/include/ylt/thirdparty/iguana/yaml_writer.hpp +++ b/include/ylt/thirdparty/iguana/yaml_writer.hpp @@ -5,28 +5,34 @@ namespace iguana { -template , int> = 0> IGUANA_INLINE void to_yaml(T &&t, Stream &s, size_t min_spaces = 0); -template , int> = 0> IGUANA_INLINE void render_yaml_value(Stream &ss, T &&t, size_t min_spaces) { ss.push_back('\n'); - to_yaml(std::forward(t), ss, min_spaces); + to_yaml(std::forward(t), ss, min_spaces); } -// TODO: support more string style, support escape -template , int> = 0> +template , int> = 0> IGUANA_INLINE void render_yaml_value(Stream &ss, T &&t, size_t min_spaces) { - ss.append(t.data(), t.size()); + if constexpr (Is_writing_escape) { + ss.push_back('"'); + write_string_with_escape(t.data(), t.size(), ss); + ss.push_back('"'); + } + else { + ss.append(t.data(), t.size()); + } if constexpr (appendLf) ss.push_back('\n'); } -template , int> = 0> +template , int> = 0> IGUANA_INLINE void render_yaml_value(Stream &ss, T value, size_t min_spaces) { char temp[65]; auto p = detail::to_chars(temp, value); @@ -35,7 +41,7 @@ IGUANA_INLINE void render_yaml_value(Stream &ss, T value, size_t min_spaces) { ss.push_back('\n'); } -template +template IGUANA_INLINE void render_yaml_value(Stream &ss, char value, size_t min_spaces) { ss.push_back(value); @@ -43,7 +49,7 @@ IGUANA_INLINE void render_yaml_value(Stream &ss, char value, ss.push_back('\n'); } -template +template IGUANA_INLINE void render_yaml_value(Stream &ss, bool value, size_t min_spaces) { ss.append(value ? "true" : "false"); @@ -51,21 +57,21 @@ IGUANA_INLINE void render_yaml_value(Stream &ss, bool value, ss.push_back('\n'); } -template , int> = 0> +template , int> = 0> IGUANA_INLINE void render_yaml_value(Stream &ss, T value, size_t min_spaces) { static constexpr auto enum_to_str = get_enum_map>(); if constexpr (bool_v) { - render_yaml_value(ss, static_cast>(value), - min_spaces); + render_yaml_value( + ss, static_cast>(value), min_spaces); } else { auto it = enum_to_str.find(value); if (it != enum_to_str.end()) IGUANA_LIKELY { auto str = it->second; - render_yaml_value(ss, std::string_view(str.data(), str.size()), - min_spaces); + render_yaml_value( + ss, std::string_view(str.data(), str.size()), min_spaces); } else { throw std::runtime_error( @@ -75,16 +81,17 @@ IGUANA_INLINE void render_yaml_value(Stream &ss, T value, size_t min_spaces) { } } -template , int> = 0> +template , int> = 0> IGUANA_INLINE void render_yaml_value(Stream &ss, const T &val, size_t min_spaces); -template , int> = 0> IGUANA_INLINE void render_yaml_value(Stream &ss, const T &val, size_t min_spaces); -template , int> = 0> IGUANA_INLINE void render_yaml_value(Stream &ss, const T &t, size_t min_spaces) { @@ -92,35 +99,37 @@ IGUANA_INLINE void render_yaml_value(Stream &ss, const T &t, for (const auto &v : t) { ss.append(min_spaces, ' '); ss.append("- "); - render_yaml_value(ss, v, min_spaces + 1); + render_yaml_value(ss, v, min_spaces + 1); } } -template , int> = 0> +template , int> = 0> IGUANA_INLINE void render_yaml_value(Stream &ss, T &&t, size_t min_spaces) { ss.push_back('\n'); for_each(std::forward(t), [&ss, min_spaces](auto &v, auto i) IGUANA__INLINE_LAMBDA { ss.append(min_spaces, ' '); ss.append("- "); - render_yaml_value(ss, v, min_spaces + 1); + render_yaml_value(ss, v, min_spaces + 1); }); } -template , int> = 0> IGUANA_INLINE void render_yaml_value(Stream &ss, const T &t, size_t min_spaces) { ss.push_back('\n'); for (const auto &[k, v] : t) { ss.append(min_spaces, ' '); - render_yaml_value(ss, k, 0); // key must be plaint type + render_yaml_value(ss, k, 0); // key must be plaint type ss.append(": "); - render_yaml_value(ss, v, min_spaces + 1); + render_yaml_value(ss, v, min_spaces + 1); } } -template , int>> +template , int>> IGUANA_INLINE void render_yaml_value(Stream &ss, const T &val, size_t min_spaces) { if (!val) { @@ -128,18 +137,19 @@ IGUANA_INLINE void render_yaml_value(Stream &ss, const T &val, ss.push_back('\n'); } else { - render_yaml_value(ss, *val, min_spaces); + render_yaml_value(ss, *val, min_spaces); } } -template , int>> +template , int>> IGUANA_INLINE void render_yaml_value(Stream &ss, const T &val, size_t min_spaces) { if (!val) { ss.push_back('\n'); } else { - render_yaml_value(ss, *val, min_spaces); + render_yaml_value(ss, *val, min_spaces); } } @@ -149,7 +159,8 @@ constexpr auto write_yaml_key = [](auto &s, auto i, s.append(name.data(), name.size()); }; -template , int>> +template , int>> IGUANA_INLINE void to_yaml(T &&t, Stream &s, size_t min_spaces) { for_each(std::forward(t), [&t, &s, min_spaces](const auto &v, auto i) IGUANA__INLINE_LAMBDA { @@ -161,21 +172,21 @@ IGUANA_INLINE void to_yaml(T &&t, Stream &s, size_t min_spaces) { write_yaml_key(s, i, t); s.append(": "); if constexpr (!is_reflection>::value) { - render_yaml_value(s, t.*v, min_spaces + 1); + render_yaml_value(s, t.*v, min_spaces + 1); } else { s.push_back('\n'); - to_yaml(t.*v, s, min_spaces + 1); + to_yaml(t.*v, s, min_spaces + 1); } }); } -template , int> = 0> IGUANA_INLINE void to_yaml(T &&t, Stream &s) { if constexpr (tuple_v || map_container_v || sequence_container_v || optional_v || smart_ptr_v) - render_yaml_value(s, std::forward(t), 0); + render_yaml_value(s, std::forward(t), 0); else static_assert(!sizeof(T), "don't suppport this type"); } diff --git a/include/ylt/util/function_name.h b/include/ylt/util/function_name.h index eebff9345..c1f78350a 100644 --- a/include/ylt/util/function_name.h +++ b/include/ylt/util/function_name.h @@ -17,9 +17,32 @@ #include #include "magic_names.hpp" + +template +constexpr std::string_view string_view_array_has( + const std::array& array, std::string_view value) { + for (const auto& v : array) { + if (value.find(v) == 0) + return v; + } + return std::string_view{""}; +} + namespace coro_rpc { template constexpr std::string_view get_func_name() { - return std::string_view{refvalue::qualified_name_of_v}; + constexpr std::array func_style_array{ + std::string_view{"__cdecl "}, std::string_view{"__clrcall "}, + std::string_view{"__stdcall "}, std::string_view{"__fastcall "}, + std::string_view{"__thiscall "}, std::string_view{"__vectorcall "}}; + constexpr auto qualified_name = + std::string_view{refvalue::qualified_name_of_v}; + constexpr auto func_style = + string_view_array_has(func_style_array, qualified_name); + if constexpr (func_style.length() > 0) { + return std::string_view{qualified_name.data() + func_style.length(), + qualified_name.length() - func_style.length()}; + } + return qualified_name; }; } // namespace coro_rpc diff --git a/src/coro_http/examples/BUILD.bazel b/src/coro_http/examples/BUILD.bazel deleted file mode 100644 index e9db42d44..000000000 --- a/src/coro_http/examples/BUILD.bazel +++ /dev/null @@ -1,23 +0,0 @@ -cc_binary( - name = "coro_http_example", - srcs = - [ - "example.cpp", - ], - copts = [ - "-std=c++20", - ], - deps = ["//:ylt"], -) - -cc_binary( - name = "coro_http_channel", - srcs = - [ - "channel.cpp", - ], - copts = [ - "-std=c++20", - ], - deps = ["//:ylt"], -) diff --git a/src/coro_http/examples/example.cpp b/src/coro_http/examples/example.cpp index 9ba4fb353..35356dede 100644 --- a/src/coro_http/examples/example.cpp +++ b/src/coro_http/examples/example.cpp @@ -253,7 +253,7 @@ async_simple::coro::Lazy static_file_server() { assert(result.resp_body.size() == 64); } -struct log_t : public base_aspect { +struct log_t { bool before(coro_http_request &, coro_http_response &) { std::cout << "before log" << std::endl; return true; @@ -266,9 +266,9 @@ struct log_t : public base_aspect { } }; -struct get_data : public base_aspect { +struct get_data { bool before(coro_http_request &req, coro_http_response &res) { - req.set_aspect_data("hello", std::string("hello world")); + req.set_aspect_data("hello world"); return true; } }; @@ -278,13 +278,11 @@ async_simple::coro::Lazy use_aspects() { server.set_http_handler( "/get", [](coro_http_request &req, coro_http_response &resp) { - std::optional val = - req.get_aspect_data("hello"); - assert(*val == "hello world"); + auto val = req.get_aspect_data(); + assert(val[0] == "hello world"); resp.set_status_and_content(status_type::ok, "ok"); }, - std::vector>{std::make_shared(), - std::make_shared()}); + log_t{}, get_data{}); server.async_start(); std::this_thread::sleep_for(300ms); // wait for server start @@ -400,11 +398,172 @@ async_simple::coro::Lazy basic_usage() { // make sure you have install openssl and enable CINATRA_ENABLE_SSL #ifdef CINATRA_ENABLE_SSL coro_http_client client2{}; - result = co_await client2.async_get("https://baidu.com"); + result = co_await client2.async_get("https://www.baidu.com"); + assert(result.status == 200); + + coro_http_client client3{}; + co_await client3.connect("https://www.baidu.com"); + result = co_await client3.async_get("/"); assert(result.status == 200); #endif } +#ifdef CINATRA_ENABLE_GZIP +std::string_view get_header_value(auto &resp_headers, std::string_view key) { + for (const auto &[k, v] : resp_headers) { + if (k == key) + return v; + } + return {}; +} + +void test_gzip() { + coro_http_server server(1, 8090); + server.set_http_handler( + "/gzip", [](coro_http_request &req, coro_http_response &res) { + assert(req.get_header_value("Content-Encoding") == "gzip"); + res.set_status_and_content(status_type::ok, "hello world", + content_encoding::gzip); + }); + server.async_start(); + + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + coro_http_client client{}; + std::string uri = "http://127.0.0.1:8090/gzip"; + client.add_header("Content-Encoding", "gzip"); + auto result = async_simple::coro::syncAwait(client.async_get(uri)); + auto content = get_header_value(result.resp_headers, "Content-Encoding"); + assert(get_header_value(result.resp_headers, "Content-Encoding") == "gzip"); + std::string decompress_data; + bool ret = gzip_codec::uncompress(result.resp_body, decompress_data); + assert(ret == true); + assert(decompress_data == "hello world"); + server.stop(); +} +#endif + +void http_proxy() { + cinatra::coro_http_server web_one(1, 9001); + + web_one.set_http_handler( + "/", + [](coro_http_request &req, + coro_http_response &response) -> async_simple::coro::Lazy { + co_await coro_io::post([&]() { + response.set_status_and_content(status_type::ok, "web1"); + }); + }); + + web_one.async_start(); + + cinatra::coro_http_server web_two(1, 9002); + + web_two.set_http_handler( + "/", + [](coro_http_request &req, + coro_http_response &response) -> async_simple::coro::Lazy { + co_await coro_io::post([&]() { + response.set_status_and_content(status_type::ok, "web2"); + }); + }); + + web_two.async_start(); + + cinatra::coro_http_server web_three(1, 9003); + + web_three.set_http_handler( + "/", [](coro_http_request &req, coro_http_response &response) { + response.set_status_and_content(status_type::ok, "web3"); + }); + + web_three.async_start(); + + std::this_thread::sleep_for(200ms); + + coro_http_server proxy_wrr(2, 8090); + proxy_wrr.set_http_proxy_handler( + "/wrr", {"127.0.0.1:9001", "127.0.0.1:9002", "127.0.0.1:9003"}, + coro_io::load_blance_algorithm::WRR, {10, 5, 5}); + + coro_http_server proxy_rr(2, 8091); + proxy_rr.set_http_proxy_handler( + "/rr", {"127.0.0.1:9001", "127.0.0.1:9002", "127.0.0.1:9003"}, + coro_io::load_blance_algorithm::RR); + + coro_http_server proxy_random(2, 8092); + proxy_random.set_http_proxy_handler( + "/random", {"127.0.0.1:9001", "127.0.0.1:9002", "127.0.0.1:9003"}); + + coro_http_server proxy_all(2, 8093); + proxy_all.set_http_proxy_handler( + "/all", {"127.0.0.1:9001", "127.0.0.1:9002", "127.0.0.1:9003"}); + + proxy_wrr.async_start(); + proxy_rr.async_start(); + proxy_random.async_start(); + proxy_all.async_start(); + + std::this_thread::sleep_for(200ms); + + coro_http_client client_rr; + resp_data resp_rr = client_rr.get("http://127.0.0.1:8091/rr"); + assert(resp_rr.resp_body == "web1"); + resp_rr = client_rr.get("http://127.0.0.1:8091/rr"); + assert(resp_rr.resp_body == "web2"); + resp_rr = client_rr.get("http://127.0.0.1:8091/rr"); + assert(resp_rr.resp_body == "web3"); + resp_rr = client_rr.get("http://127.0.0.1:8091/rr"); + assert(resp_rr.resp_body == "web1"); + resp_rr = client_rr.get("http://127.0.0.1:8091/rr"); + assert(resp_rr.resp_body == "web2"); + resp_rr = client_rr.post("http://127.0.0.1:8091/rr", "test content", + req_content_type::text); + assert(resp_rr.resp_body == "web3"); + + coro_http_client client_wrr; + resp_data resp = client_wrr.get("http://127.0.0.1:8090/wrr"); + assert(resp.resp_body == "web1"); + resp = client_wrr.get("http://127.0.0.1:8090/wrr"); + assert(resp.resp_body == "web1"); + resp = client_wrr.get("http://127.0.0.1:8090/wrr"); + assert(resp.resp_body == "web2"); + resp = client_wrr.get("http://127.0.0.1:8090/wrr"); + assert(resp.resp_body == "web3"); + + coro_http_client client_random; + resp_data resp_random = client_random.get("http://127.0.0.1:8092/random"); + std::cout << resp_random.resp_body << "\n"; + assert(!resp_random.resp_body.empty()); + + coro_http_client client_all; + resp_random = client_all.post("http://127.0.0.1:8093/all", "test content", + req_content_type::text); + std::cout << resp_random.resp_body << "\n"; + assert(!resp_random.resp_body.empty()); +} + +void coro_channel() { + auto ctx = coro_io::get_global_block_executor()->get_asio_executor(); + asio::experimental::channel ch(ctx, 10000); + auto ec = async_simple::coro::syncAwait(coro_io::async_send(ch, 41)); + assert(!ec); + ec = async_simple::coro::syncAwait(coro_io::async_send(ch, 42)); + assert(!ec); + + std::error_code err; + int val; + std::tie(err, val) = + async_simple::coro::syncAwait(coro_io::async_receive(ch)); + assert(!err); + assert(val == 41); + + std::tie(err, val) = + async_simple::coro::syncAwait(coro_io::async_receive(ch)); + assert(!err); + assert(val == 42); +} + int main() { async_simple::coro::syncAwait(basic_usage()); async_simple::coro::syncAwait(use_aspects()); @@ -412,5 +571,10 @@ int main() { async_simple::coro::syncAwait(use_websocket()); async_simple::coro::syncAwait(chunked_upload_download()); async_simple::coro::syncAwait(byte_ranges_download()); +#ifdef CINATRA_ENABLE_GZIP + test_gzip(); +#endif + http_proxy(); + coro_channel(); return 0; } \ No newline at end of file diff --git a/src/coro_io/tests/test_channel.cpp b/src/coro_io/tests/test_channel.cpp index 836e049e4..3f78e9434 100644 --- a/src/coro_io/tests/test_channel.cpp +++ b/src/coro_io/tests/test_channel.cpp @@ -41,6 +41,80 @@ TEST_CASE("test RR") { }()); } +TEST_CASE("test WRR") { + SUBCASE( + "exception tests: empty hosts, empty weights test or count not equal") { + CHECK_THROWS_AS( + coro_io::channel::create( + {}, {.lba = coro_io::load_blance_algorithm::WRR}, {2, 1}), + std::invalid_argument); + + CHECK_THROWS_AS(coro_io::channel::create( + {"127.0.0.1:8801", "127.0.0.1:8802"}, + {.lba = coro_io::load_blance_algorithm::WRR}), + std::invalid_argument); + + CHECK_THROWS_AS(coro_io::channel::create( + {"127.0.0.1:8801", "127.0.0.1:8802"}, + {.lba = coro_io::load_blance_algorithm::WRR}, {1}), + std::invalid_argument); + } + + coro_rpc::coro_rpc_server server1(1, 8801); + auto res = server1.async_start(); + REQUIRE_MESSAGE(res, "server start failed"); + coro_rpc::coro_rpc_server server2(1, 8802); + auto res2 = server2.async_start(); + REQUIRE_MESSAGE(res2, "server start failed"); + + async_simple::coro::syncAwait([]() -> async_simple::coro::Lazy { + auto hosts = + std::vector{"127.0.0.1:8801", "127.0.0.1:8802"}; + auto channel = coro_io::channel::create( + hosts, {.lba = coro_io::load_blance_algorithm::WRR}, {2, 1}); + for (int i = 0; i < 6; ++i) { + auto res = co_await channel.send_request( + [&i, &hosts]( + coro_rpc::coro_rpc_client &client, + std::string_view host) -> async_simple::coro::Lazy { + if (i == 0 || i == 1) { + CHECK(host == hosts[0]); + } + else if (i == 2 || i == 5) { + CHECK(host == hosts[1]); + } + else if (i == 3 || i == 4) { + CHECK(host == hosts[0]); + } + co_return; + }); + CHECK(res.has_value()); + } + }()); + + async_simple::coro::syncAwait([]() -> async_simple::coro::Lazy { + auto hosts = + std::vector{"127.0.0.1:8801", "127.0.0.1:8802"}; + auto channel = coro_io::channel::create( + hosts, {.lba = coro_io::load_blance_algorithm::WRR}, {0, 0}); + for (int i = 0; i < 6; ++i) { + auto res = co_await channel.send_request( + [&i, &hosts]( + coro_rpc::coro_rpc_client &client, + std::string_view host) -> async_simple::coro::Lazy { + if (i % 2 == 0) { + CHECK(host == hosts[0]); + } + else { + CHECK(host == hosts[1]); + } + co_return; + }); + CHECK(res.has_value()); + } + }()); +} + TEST_CASE("test Random") { async_simple::coro::syncAwait([]() -> async_simple::coro::Lazy { coro_rpc::coro_rpc_server server(1, 8801); @@ -92,10 +166,10 @@ TEST_CASE("test single host") { TEST_CASE("test send_request config") { async_simple::coro::syncAwait([]() -> async_simple::coro::Lazy { - coro_rpc::coro_rpc_server server(1, 8802); + coro_rpc::coro_rpc_server server(1, 9813); auto res = server.async_start(); REQUIRE_MESSAGE(res, "server start failed"); - auto hosts = std::vector{"127.0.0.1:8802"}; + auto hosts = std::vector{"127.0.0.1:9813"}; auto channel = coro_io::channel::create(hosts); for (int i = 0; i < 100; ++i) { auto config = coro_rpc::coro_rpc_client::config{.client_id = 114514}; diff --git a/src/coro_rpc/examples/base_examples/rpc_service.cpp b/src/coro_rpc/examples/base_examples/rpc_service.cpp index e85d8ca95..5b45d7c26 100644 --- a/src/coro_rpc/examples/base_examples/rpc_service.cpp +++ b/src/coro_rpc/examples/base_examples/rpc_service.cpp @@ -30,6 +30,8 @@ std::string hello_world() { return "hello_world"; } +bool return_bool_hello_world() { return true; } + int A_add_B(int a, int b) { ELOGV(INFO, "call A+B"); return a + b; diff --git a/src/coro_rpc/examples/base_examples/rpc_service.h b/src/coro_rpc/examples/base_examples/rpc_service.h index 145dfe1bd..1198009b7 100644 --- a/src/coro_rpc/examples/base_examples/rpc_service.h +++ b/src/coro_rpc/examples/base_examples/rpc_service.h @@ -21,6 +21,7 @@ #include std::string hello_world(); +bool return_bool_hello_world(); int A_add_B(int a, int b); void hello_with_delay(coro_rpc::context conn, std::string hello); std::string echo(std::string_view sv); diff --git a/src/coro_rpc/examples/base_examples/server.cpp b/src/coro_rpc/examples/base_examples/server.cpp index 5efb49d4a..e6cedb2dc 100644 --- a/src/coro_rpc/examples/base_examples/server.cpp +++ b/src/coro_rpc/examples/base_examples/server.cpp @@ -26,6 +26,8 @@ int main() { coro_rpc_server server2{/*thread=*/1, /*port=*/8802}; + server.register_handler(); + // regist normal function for rpc server.register_handler +#include +#include #include #include "ylt/struct_json/json_reader.h" #include "ylt/struct_json/json_writer.h" +void test_user_defined_struct(); + struct person { std::string name; int age; @@ -58,6 +62,16 @@ void use_smart_pointer() { assert(*p1.age == 42); } +void test_escape_serialize() { + person p{"老\t人", 20}; + std::string ss; + struct_json::to_json(p, ss); + std::cout << ss << std::endl; + person p1; + struct_json::from_json(p1, ss); + assert(p1.name == p.name); +} + int main() { person p{"tom", 20}; std::string str; @@ -79,4 +93,6 @@ int main() { test_inner_object(); use_smart_pointer(); + test_escape_serialize(); + test_user_defined_struct(); } \ No newline at end of file diff --git a/src/struct_json/examples/user_defined_struct.cpp b/src/struct_json/examples/user_defined_struct.cpp new file mode 100644 index 000000000..3b362bcf5 --- /dev/null +++ b/src/struct_json/examples/user_defined_struct.cpp @@ -0,0 +1,59 @@ +#include +#include + +#include + +#include "iguana/json_writer.hpp" + +namespace my_space { +struct my_struct { + int x, y, z; + bool operator==(const my_struct& o) const { + return x == o.x && y == o.y && z == o.z; + } +}; + +template +inline void to_json_impl(Stream& s, const my_struct& t) { + struct_json::to_json(*(int(*)[3]) & t, s); +} + +template +IGUANA_INLINE void from_json_impl(my_struct& value, It&& it, It&& end) { + struct_json::from_json(*(int(*)[3]) & value, it, end); +} + +} // namespace my_space + +struct nest { + std::string name; + my_space::my_struct value; + bool operator==(const nest& o) const { + return name == o.name && value == o.value; + } +}; + +REFLECTION(nest, name, value); + +void example1() { + my_space::my_struct v{1, 2, 3}, v2; + std::string s; + struct_json::to_json(v, s); + std::cout << s << std::endl; + struct_json::from_json(v2, s); + assert(v == v2); +}; + +void example2() { + nest v{"Hi", {1, 2, 3}}, v2; + std::string s; + struct_json::to_json(v, s); + std::cout << s << std::endl; + struct_json::from_json(v2, s); + assert(v == v2); +}; + +void test_user_defined_struct() { + example1(); + example2(); +} \ No newline at end of file diff --git a/src/struct_pack/examples/BUILD.bazel b/src/struct_pack/examples/BUILD.bazel index 3b21d3914..7d2c6885b 100644 --- a/src/struct_pack/examples/BUILD.bazel +++ b/src/struct_pack/examples/BUILD.bazel @@ -1,10 +1,18 @@ load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library") +load("//bazel:defs.bzl", "YA_BIN_COPT", "YA_LT_COPT") cc_binary( name = "serialize_example", - srcs = ["basic_usage.cpp","main.cpp","non_aggregated_type.cpp","serialize_config.cpp","user_defined_serialization.cpp","derived_class.cpp"], - copts = ["-std=c++20"], + srcs = [ + "basic_usage.cpp", + "derived_class.cpp", + "main.cpp", + "non_aggregated_type.cpp", + "serialize_config.cpp", + "user_defined_serialization.cpp", + ], + copts = YA_BIN_COPT, deps = [ - "//:ylt" + "//:ylt", ], ) diff --git a/src/struct_pack/tests/BUILD.bazel b/src/struct_pack/tests/BUILD.bazel index 211717cd2..c11b97334 100644 --- a/src/struct_pack/tests/BUILD.bazel +++ b/src/struct_pack/tests/BUILD.bazel @@ -1,15 +1,16 @@ load("@rules_cc//cc:defs.bzl", "cc_test") +load("//bazel:defs.bzl", "YA_BIN_COPT") cc_test( name = "test_serialize", - srcs = glob(["*.cpp"]) + ["test_struct.hpp"]+["test_derived.hpp"], - copts = ["-std=c++20"], - defines = ["STRUCT_PACK_ENABLE_UNPORTABLE_TYPE"], - deps = [ - "//:ylt" - ], + srcs = glob(["*.cpp"]) + ["test_struct.hpp"] + ["test_derived.hpp"], + copts = YA_BIN_COPT, data = [ "//src/struct_pack/tests/binary_data:test_cross_platform.dat", - "//src/struct_pack/tests/binary_data:test_cross_platform_without_debug_info.dat" + "//src/struct_pack/tests/binary_data:test_cross_platform_without_debug_info.dat", + ], + defines = ["STRUCT_PACK_ENABLE_UNPORTABLE_TYPE"], + deps = [ + "//:ylt", ], ) diff --git a/src/struct_pack/tests/test_derived.cpp b/src/struct_pack/tests/test_derived.cpp index ce826e43d..742044313 100644 --- a/src/struct_pack/tests/test_derived.cpp +++ b/src/struct_pack/tests/test_derived.cpp @@ -141,7 +141,7 @@ TEST_CASE("test unique_ptr") { } } -TEST_CASE("test unique_ptr with virtual base") { +TEST_CASE("test vector> with virtual base") { using namespace test3; static_assert(struct_pack::detail::is_base_class); std::vector> vec; @@ -158,4 +158,13 @@ TEST_CASE("test unique_ptr with virtual base") { for (std::size_t i = 0; i < vec.size(); ++i) { CHECK(vec[i]->get_name() == vec2[i]->get_name()); } +} + +TEST_CASE("test unique_ptr with virtual base") { + using namespace test3; + std::unique_ptr ptr = std::make_unique(); + auto buffer2 = struct_pack::serialize(ptr); + auto res2 = struct_pack::deserialize>(buffer2); + CHECK(res2); + CHECK(res2.value()->get_name() == std::make_unique()->get_name()); } \ No newline at end of file diff --git a/website/docs/zh/coro_http/coro_http_introduction.md b/website/docs/zh/coro_http/coro_http_introduction.md index 180a4ca34..e9eb0aafe 100644 --- a/website/docs/zh/coro_http/coro_http_introduction.md +++ b/website/docs/zh/coro_http/coro_http_introduction.md @@ -691,8 +691,7 @@ int main() { using namespace coro_http; //日志切面 - struct log_t : public base_aspect - { + struct log_t { bool before(coro_http_request& req, coro_http_response& res) { std::cout << "before log" << std::endl; return true; @@ -705,7 +704,7 @@ int main() { }; //校验的切面 - struct check : public base_aspect { + struct check { bool before(coro_http_request& req, coro_http_response& res) { std::cout << "before check" << std::endl; if (req.get_header_value("name").empty()) { @@ -722,9 +721,9 @@ int main() { }; //将信息从中间件传输到处理程序 - struct get_data : public base_aspect { + struct get_data { bool before(coro_http_request& req, coro_http_response& res) { - req.set_aspect_data("hello", std::string("hello world")); + req.set_aspect_data("hello world"); return true; } } @@ -733,12 +732,12 @@ int main() { coro_http_server server(std::thread::hardware_concurrency(), 8080); server.set_http_handler("/aspect", [](coro_http_request& req, coro_http_response& res) { res.set_status_and_content(status_type::ok, "hello world"); - }, std::vector{std::make_shared(), std::make_shared()}); + }, check{}, log_t{}); server.set_http_handler("/aspect/data", [](coro_http_request& req, coro_http_response& res) { - std::string hello = req.get_aspect_data("hello"); + std::string hello = req.get_aspect_data()[0]; res.set_status_and_content(status_type::ok, std::move(hello)); - }, std::vector{std::make_shared()}); + }, get_data{}); server.sync_start(); return 0; @@ -783,3 +782,70 @@ int main() { return 0; } ``` + + ### 反向代理 + 目前支持random, round robin 和 weight round robin三种负载均衡三种算法，设置代理服务器时指定算法类型即可。 + 假设需要代理的服务器有三个，分别是"127.0.0.1:9001", "127.0.0.1:9002", "127.0.0.1:9003"，coro_http_server设置路径、代理服务器列表和算法类型即可实现反向代理。 + + ```c++ + coro_http_server proxy_random(2, 8092); + proxy_random.set_http_proxy_handler( + "/random", {"127.0.0.1:9001", "127.0.0.1:9002", "127.0.0.1:9003"}); + + coro_http_server proxy_rr(2, 8091); + proxy_rr.set_http_proxy_handler( + "/rr", {"127.0.0.1:9001", "127.0.0.1:9002", "127.0.0.1:9003"}, + coro_io::load_blance_algorithm::RR); + + coro_http_server proxy_wrr(2, 8090); + proxy_wrr.set_http_proxy_handler( + "/wrr", {"127.0.0.1:9001", "127.0.0.1:9002", "127.0.0.1:9003"}, + coro_io::load_blance_algorithm::WRR, {10, 5, 5}); + + coro_http_server proxy_all(2, 8093); + proxy_all.set_http_proxy_handler( + "/all", {"127.0.0.1:9001", "127.0.0.1:9002", "127.0.0.1:9003"}); + + proxy_wrr.async_start(); + proxy_rr.async_start(); + proxy_random.async_start(); + proxy_all.async_start(); + + std::this_thread::sleep_for(200ms); + + coro_http_client client_rr; + resp_data resp_rr = client_rr.get("http://127.0.0.1:8091/rr"); + assert(resp_rr.resp_body == "web1"); + resp_rr = client_rr.get("http://127.0.0.1:8091/rr"); + assert(resp_rr.resp_body == "web2"); + resp_rr = client_rr.get("http://127.0.0.1:8091/rr"); + assert(resp_rr.resp_body == "web3"); + resp_rr = client_rr.get("http://127.0.0.1:8091/rr"); + assert(resp_rr.resp_body == "web1"); + resp_rr = client_rr.get("http://127.0.0.1:8091/rr"); + assert(resp_rr.resp_body == "web2"); + resp_rr = client_rr.post("http://127.0.0.1:8091/rr", "test content", + req_content_type::text); + assert(resp_rr.resp_body == "web3"); + + coro_http_client client_wrr; + resp_data resp = client_wrr.get("http://127.0.0.1:8090/wrr"); + assert(resp.resp_body == "web1"); + resp = client_wrr.get("http://127.0.0.1:8090/wrr"); + assert(resp.resp_body == "web1"); + resp = client_wrr.get("http://127.0.0.1:8090/wrr"); + assert(resp.resp_body == "web2"); + resp = client_wrr.get("http://127.0.0.1:8090/wrr"); + assert(resp.resp_body == "web3"); + + coro_http_client client_random; + resp_data resp_random = client_random.get("http://127.0.0.1:8092/random"); + std::cout << resp_random.resp_body << "\n"; + assert(!resp_random.resp_body.empty()); + + coro_http_client client_all; + resp_random = client_all.post("http://127.0.0.1:8093/all", "test content", + req_content_type::text); + std::cout << resp_random.resp_body << "\n"; + assert(!resp_random.resp_body.empty()); + ``` diff --git a/website/docs/zh/guide/what_is_yalantinglibs.md b/website/docs/zh/guide/what_is_yalantinglibs.md index b2d50c128..bec17e71e 100644 --- a/website/docs/zh/guide/what_is_yalantinglibs.md +++ b/website/docs/zh/guide/what_is_yalantinglibs.md @@ -114,7 +114,7 @@ coro是一个高度易用, head-only，基于协程的C++20高性能rpc框架库 [简介](https://alibaba.github.io/yalantinglibs/zh/coro_rpc/coro_rpc_introduction.html) -[API](https://alibaba.github.io/yalantinglibs/cn/html/group__coro__rpc.html) +[API](https://alibaba.github.io/yalantinglibs/doxygen_cn/html/group__coro__rpc.html) [Purecpp 演讲稿](https://alibaba.github.io/yalantinglibs/resource/coro_rpc_introduction_purecpp_talk.pdf). @@ -364,49 +364,47 @@ async_simple是一个C++20协程库，提供各种轻量且易用的组件，帮 # 其他 -## CMAKE 选项 +## 配置选项 -以下这些Cmake选项只适用于yalantinglibs自身的开发和安装。他们不会对你的项目造成影响，因为yalantinglibs是head-only的。 +yalantinglibs工程自身支持如下配置项，如果你使用cmake find_package或者fetchContent来导入yalantinglibs，你的工程也可以使用下面这些配置项。 -### 安装选项 +|工程选项|默认值|描述| +|----------|------------|------| +|YLT_ENABLE_SSL|OFF|为rpc/http启用可选的ssl支持| +|YLT_ENABLE_PMR|OFF|启用pmr优化| +|YLT_ENABLE_IO_URING|OFF|在linux上使用io_uring作为后端（代替epoll）| +|YLT_ENABLE_FILE_IO_URING|OFF|启用io_uring优化| +|YLT_ENABLE_STRUCT_PACK_UNPORTABLE_TYPE|OFF|struct_pack启用对不跨平台的特殊类型的支持（如wstring, in128_t）| +|YLT_ENABLE_STRUCT_PACK_OPTIMIZE|OFF|struct_pack启用激进的模板展开优化（会花费更多编译时间）| + +## 第三方安装选项 + +默认情况下，ylt会把第三方依赖安装到`ylt/thirdparty`目录下，你需要将它添加到头文件包含路径中。 + +如果你不想让ylt安装第三方依赖，你可以使用选项：`-DINSTALL_THIRDPARTY=OFF`。 + +如果你想让ylt将第三方依赖直接独立安装到系统默认的包含路径中，你可以开启选项：`-DINSTALL_INDEPENDENT_THIRDPARTY=ON`。 |选项|默认值| |----------|------------| |INSTALL_THIRDPARTY|ON| |INSTALL_INDEPENDENT_THIRDPARTY|OFF| -### ylt 开发选项 +## 开发选项 + +以下这些Cmake选项只适用于yalantinglibs自身的开发。它们不会对你的项目造成影响，因为yalantinglibs是head-only的。 |选项|默认值| |----------|------------| |BUILD_EXAMPLES|ON| |BUILD_BENCHMARK|ON| |BUILD_UNIT_TESTS|ON| -|BUILD_*(BUILD_CORO_RPC, BUILD_STRUCT_PACK 等)|ON| +|BUILD_*(BUILD_CORO_RPC, BUILD_STRUCT_PACK等)|ON| |COVERAGE_TEST|OFF| |GENERATE_BENCHMARK_DATA|ON| |CORO_RPC_USE_OTHER_RPC|ON| -### ylt 配置项 - -你可能也想在你的工程里启用这些配置项。你可以参考这些[cmake 代码](https://github.com/alibaba/yalantinglibs/tree/main/cmake/config.cmake)。 - -|选项|默认值| -|----------|------------| -|ENABLE_SSL|OFF| -|ENABLE_PMR|OFF| -|ENABLE_IO_URING|OFF| -|ENABLE_FILE_IO_URING|OFF| -|ENABLE_STRUCT_PACK_UNPORTABLE_TYPE|OFF| -|ENABLE_STRUCT_PACK_OPTIMIZE|OFF| - -## 第三方依赖 - -默认情况下，ylt会把第三方依赖安装到`ylt/thirdparty`目录下，你需要将它添加到头文件包含路径中。 - -如果你不想让ylt安装第三方依赖，你可以使用选项：`-DINSTALL_THIRDPARTY=OFF`。 - -如果你想让ylt将第三方依赖直接独立安装到系统默认的包含路径中，你可以开启选项：`-DINSTALL_INDEPENDENT_THIRDPARTY=ON`。 +## 第三方依赖清单 以下是我们使用的第三方依赖（async_simple虽然也是ylt的一部分，但其首先开源，故计为一个独立的第三方依赖）