From 69025273f4cbb4bfe76a362336f6497530e35f0d Mon Sep 17 00:00:00 2001 From: helintong Date: Fri, 14 Jun 2024 15:17:52 +0800 Subject: [PATCH] feat: add br compression --- cmake/FindBrotli.cmake | 21 +++++++ cmake/develop.cmake | 12 ++++ example/CMakeLists.txt | 6 ++ include/cinatra/brzip.hpp | 82 ++++++++++++++++++++++++++ include/cinatra/coro_http_client.hpp | 37 ++++++++++-- include/cinatra/coro_http_request.hpp | 2 + include/cinatra/coro_http_response.hpp | 36 +++++++++++ include/cinatra/define.h | 2 +- press_tool/CMakeLists.txt | 10 ++++ tests/CMakeLists.txt | 5 ++ tests/test_cinatra.cpp | 35 +++++++++++ 11 files changed, 243 insertions(+), 5 deletions(-) create mode 100644 cmake/FindBrotli.cmake create mode 100644 include/cinatra/brzip.hpp diff --git a/cmake/FindBrotli.cmake b/cmake/FindBrotli.cmake new file mode 100644 index 00000000..4b96b30c --- /dev/null +++ b/cmake/FindBrotli.cmake @@ -0,0 +1,21 @@ +include(FindPackageHandleStandardArgs) + +find_path(BROTLI_INCLUDE_DIR "brotli/decode.h") + +find_library(BROTLICOMMON_LIBRARY NAMES brotlicommon) +find_library(BROTLIDEC_LIBRARY NAMES brotlidec) +find_library(BROTLIENC_LIBRARY NAMES brotlienc) + +find_package_handle_standard_args(Brotli + FOUND_VAR + BROTLI_FOUND + REQUIRED_VARS + BROTLIDEC_LIBRARY + BROTLICOMMON_LIBRARY + BROTLI_INCLUDE_DIR + FAIL_MESSAGE + "Could NOT find Brotli" +) + +set(BROTLI_INCLUDE_DIRS ${BROTLI_INCLUDE_DIR}) +set(BROTLI_LIBRARIES ${BROTLIDEC_LIBRARY} ${BROTLIENC_LIBRARY} ${BROTLICOMMON_LIBRARY}) \ No newline at end of file diff --git a/cmake/develop.cmake b/cmake/develop.cmake index 864fb2b5..bf186e25 100644 --- a/cmake/develop.cmake +++ b/cmake/develop.cmake @@ -53,9 +53,12 @@ if(ENABLE_SANITIZER AND NOT MSVC) endif() endif() +set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake;${CMAKE_MODULE_PATH}") + SET(ENABLE_GZIP OFF) SET(ENABLE_SSL OFF) SET(ENABLE_CLIENT_SSL OFF) +SET(ENABLE_BROTLI OFF) if (ENABLE_SSL) add_definitions(-DCINATRA_ENABLE_SSL) @@ -100,4 +103,13 @@ if (ENABLE_GZIP) find_package(ZLIB REQUIRED) endif() +if (ENABLE_BROTLI) + find_package(Brotli REQUIRED) + if (Brotli_FOUND) + message(STATUS "Brotli found") + add_definitions(-DCINATRA_ENABLE_BROTLI) + endif (Brotli_FOUND) +endif(ENABLE_BROTLI) + + add_definitions(-DCORO_HTTP_PRINT_REQ_HEAD) \ No newline at end of file diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt index e8d17c79..e3be6220 100644 --- a/example/CMakeLists.txt +++ b/example/CMakeLists.txt @@ -29,6 +29,12 @@ if (ENABLE_GZIP) target_link_libraries(benchmark PRIVATE ${ZLIB_LIBRARIES}) endif() +if (ENABLE_BROTLI) + include_directories(${BROTLI_INCLUDE_DIRS}) + target_link_libraries(${project_name} ${BROTLI_LIBRARIES}) + #target_link_libraries(benchmark PRIVATE ${BROTLI_LIBRARIES}) +endif() + if (ENABLE_SIMD STREQUAL "AARCH64") if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "aarch64") add_library(neon INTERFACE IMPORTED) diff --git a/include/cinatra/brzip.hpp b/include/cinatra/brzip.hpp new file mode 100644 index 00000000..2f0ad94e --- /dev/null +++ b/include/cinatra/brzip.hpp @@ -0,0 +1,82 @@ +#pragma once +#include +#include +#include +#include +#include +#include + +namespace cinatra::br_codec { + +#define BROTLI_BUFFER_SIZE 1024 + +bool brotli_compress(std::string_view input, std::string &output) +{ + auto instance = BrotliEncoderCreateInstance(nullptr, nullptr, nullptr); + std::array buffer; + std::stringstream result; + + size_t available_in = input.size(), available_out = buffer.size(); + const uint8_t* next_in = reinterpret_cast(input.data()); + uint8_t* next_out = buffer.data(); + + do + { + int ret = BrotliEncoderCompressStream + ( + instance, BROTLI_OPERATION_FINISH, + &available_in, &next_in, &available_out, &next_out, nullptr + ); + if (!ret) + return false; + result.write(reinterpret_cast(buffer.data()), buffer.size() - available_out); + available_out = buffer.size(); + next_out = buffer.data(); + } + while (!(available_in == 0 && BrotliEncoderIsFinished(instance))); + + BrotliEncoderDestroyInstance(instance); + output = result.str(); + return true; +} + +bool brotli_decompress(std::string_view input, std::string &decompressed) +{ + if (input.size() == 0) + return false; + + size_t available_in = input.size(); + auto next_in = (const uint8_t *)(input.data()); + decompressed = std::string(available_in * 3, 0); + size_t available_out = decompressed.size(); + auto next_out = (uint8_t *)(decompressed.data()); + size_t total_out{0}; + bool done = false; + auto s = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr); + while (!done) { + auto result = BrotliDecoderDecompressStream( + s, &available_in, &next_in, &available_out, &next_out, &total_out); + if (result == BROTLI_DECODER_RESULT_SUCCESS) { + decompressed.resize(total_out); + BrotliDecoderDestroyInstance(s); + return true; + } + else if (result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) { + if (total_out != decompressed.size()) { + return false; + } + decompressed.resize(total_out * 2); + next_out = (uint8_t *)(decompressed.data() + total_out); + available_out = total_out; + } + else { + decompressed.resize(0); + BrotliDecoderDestroyInstance(s); + return true; + } + } + return true; +} +} + +// namespace cinatra::br_codec \ No newline at end of file diff --git a/include/cinatra/coro_http_client.hpp b/include/cinatra/coro_http_client.hpp index 858f3d64..ba5bcb60 100644 --- a/include/cinatra/coro_http_client.hpp +++ b/include/cinatra/coro_http_client.hpp @@ -24,6 +24,9 @@ #ifdef CINATRA_ENABLE_GZIP #include "gzip.hpp" #endif +#ifdef CINATRA_ENABLE_BROTLI +#include "brzip.hpp" +#endif #include "cinatra_log_wrapper.hpp" #include "http_parser.hpp" #include "multipart.hpp" @@ -99,6 +102,7 @@ struct resp_data { int status = 0; bool eof = false; std::string_view resp_body; + std::string_view br_data; std::span resp_headers; #ifdef BENCHMARK_TEST uint64_t total = 0; @@ -1436,7 +1440,6 @@ class coro_http_client : public std::enable_shared_from_this { if (is_redirect) redirect_uri_ = parser_.get_header_value("Location"); -#ifdef CINATRA_ENABLE_GZIP if (!parser_.get_header_value("Content-Encoding").empty()) { if (parser_.get_header_value("Content-Encoding").find("gzip") != std::string_view::npos) @@ -1444,11 +1447,12 @@ class coro_http_client : public std::enable_shared_from_this { else if (parser_.get_header_value("Content-Encoding").find("deflate") != std::string_view::npos) encoding_type_ = content_encoding::deflate; + else if (parser_.get_header_value("Content-Encoding").find("br") != std::string_view::npos) + encoding_type_ = content_encoding::br; } else { encoding_type_ = content_encoding::none; } -#endif size_t content_len = (size_t)parser_.body_len(); #ifdef BENCHMARK_TEST @@ -1572,9 +1576,34 @@ class coro_http_client : public std::enable_shared_from_this { else data.resp_body = reply; } - else +#endif + +#if (defined CINATRA_ENABLE_BROTLI) && (defined CINATRA_ENABLE_GZIP) + else if (encoding_type_ == content_encoding::br) +#endif +#if (defined CINATRA_ENABLE_BROTLI) && !defined(CINATRA_ENABLE_GZIP) + if (encoding_type_ == content_encoding::br) +#endif +#ifdef CINATRA_ENABLE_BROTLI + { + std::string unbr_str; + bool r = br_codec::brotli_decompress(reply, unbr_str); + if (r) { + data.resp_body = unbr_str; + data.br_data = unbr_str; + } + else + data.resp_body = reply; + } +#endif + +#if (defined(CINATRA_ENABLE_BROTLI) || defined(CINATRA_ENABLE_GZIP)) + else if (encoding_type_ == content_encoding::none) { #endif data.resp_body = reply; +#if (defined(CINATRA_ENABLE_BROTLI) || defined(CINATRA_ENABLE_GZIP)) + } +#endif head_buf_.consume(content_len); } @@ -2099,8 +2128,8 @@ class coro_http_client : public std::enable_shared_from_this { bool enable_ws_deflate_ = false; bool is_server_support_ws_deflate_ = false; std::string inflate_str_; - content_encoding encoding_type_ = content_encoding::none; #endif + content_encoding encoding_type_ = content_encoding::none; #ifdef BENCHMARK_TEST bool stop_bench_ = false; diff --git a/include/cinatra/coro_http_request.hpp b/include/cinatra/coro_http_request.hpp index 290baae5..21cfb23a 100644 --- a/include/cinatra/coro_http_request.hpp +++ b/include/cinatra/coro_http_request.hpp @@ -156,6 +156,8 @@ class coro_http_request { return content_encoding::gzip; else if (encoding_type.find("deflate") != std::string_view::npos) return content_encoding::deflate; + else if (encoding_type.find("br") != std::string_view::npos) + return content_encoding::br; else return content_encoding::none; } diff --git a/include/cinatra/coro_http_response.hpp b/include/cinatra/coro_http_response.hpp index 5d045357..7f5812cc 100644 --- a/include/cinatra/coro_http_response.hpp +++ b/include/cinatra/coro_http_response.hpp @@ -15,6 +15,9 @@ #ifdef CINATRA_ENABLE_GZIP #include "gzip.hpp" #endif +#ifdef CINATRA_ENABLE_BROTLI +#include "brzip.hpp" +#endif #include "picohttpparser.h" #include "response_cv.hpp" #include "time_util.hpp" @@ -111,6 +114,39 @@ class coro_http_response { } } } +#endif +#if (defined CINATRA_ENABLE_BROTLI) && (defined CINATRA_ENABLE_GZIP) + else if (encoding == content_encoding::br) +#endif +#if (defined CINATRA_ENABLE_BROTLI) && !defined(CINATRA_ENABLE_GZIP) + if (encoding == content_encoding::br) +#endif +#ifdef CINATRA_ENABLE_BROTLI + { + if (client_encoding_type.empty() || + client_encoding_type.find("br") != std::string_view::npos) { + std::string br_str; + bool r = br_codec::brotli_compress(content, br_str); + if (!r) { + set_status_and_content(status_type::internal_server_error, + "br compress error"); + } + else { + add_header("Content-Encoding", "br"); + set_content(std::move(br_str)); + } + } + else { + if (is_view) { + content_view_ = content; + } + else { + content_ = std::move(content); + } + } + } +#endif +#if (defined CINATRA_ENABLE_BROTLI) || (defined CINATRA_ENABLE_GZIP) else #endif { diff --git a/include/cinatra/define.h b/include/cinatra/define.h index 2ce1c4c5..20a489d6 100644 --- a/include/cinatra/define.h +++ b/include/cinatra/define.h @@ -19,7 +19,7 @@ enum class http_method { OPTIONS, DEL, }; -enum class content_encoding { gzip, deflate, none }; +enum class content_encoding { gzip, deflate, br, none }; constexpr inline auto GET = http_method::GET; constexpr inline auto POST = http_method::POST; constexpr inline auto DEL = http_method::DEL; diff --git a/press_tool/CMakeLists.txt b/press_tool/CMakeLists.txt index a734e5b1..4db1f8c4 100644 --- a/press_tool/CMakeLists.txt +++ b/press_tool/CMakeLists.txt @@ -26,6 +26,11 @@ if (ENABLE_GZIP) target_link_libraries(${project_name} ${ZLIB_LIBRARIES}) endif() +if (ENABLE_BROTLI) + include_directories(${BROTLI_INCLUDE_DIRS}) + target_link_libraries(${project_name} ${BROTLI_LIBRARIES}) +endif() + if (ENABLE_SIMD STREQUAL "AARCH64") if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "aarch64") add_library(neon INTERFACE IMPORTED) @@ -76,4 +81,9 @@ if (ENABLE_PRESS_TOOL_TESTS) target_link_libraries(${unittest_press_tool} ${ZLIB_LIBRARIES}) endif() + if (ENABLE_BROTLI) + include_directories(${BROTLI_INCLUDE_DIRS}) + target_link_libraries(${unittest_press_tool} ${BROTLI_LIBRARIES}) + endif() + endif() \ No newline at end of file diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 1d0bf56c..cdd6eb12 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -25,6 +25,11 @@ if (ENABLE_GZIP) target_link_libraries(${project_name} ${ZLIB_LIBRARIES}) endif() +if (ENABLE_BROTLI) + include_directories(${BROTLI_INCLUDE_DIRS}) + target_link_libraries(${project_name} ${BROTLI_LIBRARIES}) +endif() + # test_coro_file option(ENABLE_FILE_IO_URING "enable io_uring" OFF) if(ENABLE_FILE_IO_URING) diff --git a/tests/test_cinatra.cpp b/tests/test_cinatra.cpp index 96eb12a8..e4dde248 100644 --- a/tests/test_cinatra.cpp +++ b/tests/test_cinatra.cpp @@ -135,6 +135,41 @@ TEST_CASE("test encoding type") { } #endif +#ifdef CINATRA_ENABLE_BROTLI +TEST_CASE("test brotli type") { + coro_http_server server(1, 9001); + + server.set_http_handler( + "/get", [](coro_http_request &req, coro_http_response &resp) { + auto encoding_type = req.get_encoding_type(); + + if (encoding_type == + content_encoding::br) { + std::string decode_str; + bool r = br_codec::brotli_decompress(req.get_body(), decode_str); + CHECK(decode_str == "Hello World"); + } + resp.set_status_and_content(status_type::ok, "ok", content_encoding::br, + req.get_accept_encoding()); + }); + + server.async_start(); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + coro_http_client client{}; + std::unordered_map headers = { + {"Content-Encoding", "br"}, + }; + std::string ziped_str; + std::string_view data = "Hello World"; + bool r = br_codec::brotli_compress(data, ziped_str); + + auto result = async_simple::coro::syncAwait(client.async_post( + "http://127.0.0.1:9001/get", ziped_str, req_content_type::none, headers)); + CHECK(result.br_data == "ok"); +} +#endif + #ifdef CINATRA_ENABLE_SSL TEST_CASE("test ssl client") { {