Skip to content

Commit

Permalink
feat: add br compression
Browse files Browse the repository at this point in the history
  • Loading branch information
helintongh committed Jun 14, 2024
1 parent 2b0b5d8 commit adb2d2a
Show file tree
Hide file tree
Showing 11 changed files with 256 additions and 20 deletions.
21 changes: 21 additions & 0 deletions cmake/FindBrotli.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
include(FindPackageHandleStandardArgs)

find_path(BROTLI_INCLUDE_DIR "brotli/decode.h")

find_library(BROTLICOMMON_LIBRARY NAMES brotlicommon)
find_library(BROTLIDEC_LIBRARY NAMES brotlidec)
find_library(BROTLIENC_LIBRARY NAMES brotlienc)

find_package_handle_standard_args(Brotli
FOUND_VAR
BROTLI_FOUND
REQUIRED_VARS
BROTLIDEC_LIBRARY
BROTLICOMMON_LIBRARY
BROTLI_INCLUDE_DIR
FAIL_MESSAGE
"Could NOT find Brotli"
)

set(BROTLI_INCLUDE_DIRS ${BROTLI_INCLUDE_DIR})
set(BROTLI_LIBRARIES ${BROTLIDEC_LIBRARY} ${BROTLIENC_LIBRARY} ${BROTLICOMMON_LIBRARY})
12 changes: 12 additions & 0 deletions cmake/develop.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -53,9 +53,12 @@ if(ENABLE_SANITIZER AND NOT MSVC)
endif()
endif()

set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake;${CMAKE_MODULE_PATH}")

SET(ENABLE_GZIP OFF)
SET(ENABLE_SSL OFF)
SET(ENABLE_CLIENT_SSL OFF)
SET(ENABLE_BROTLI OFF)

if (ENABLE_SSL)
add_definitions(-DCINATRA_ENABLE_SSL)
Expand Down Expand Up @@ -100,4 +103,13 @@ if (ENABLE_GZIP)
find_package(ZLIB REQUIRED)
endif()

if (ENABLE_BROTLI)
find_package(Brotli REQUIRED)
if (Brotli_FOUND)
message(STATUS "Brotli found")
add_definitions(-DCINATRA_ENABLE_BROTLI)
endif (Brotli_FOUND)
endif(ENABLE_BROTLI)


add_definitions(-DCORO_HTTP_PRINT_REQ_HEAD)
6 changes: 6 additions & 0 deletions example/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,12 @@ if (ENABLE_GZIP)
target_link_libraries(benchmark PRIVATE ${ZLIB_LIBRARIES})
endif()

if (ENABLE_BROTLI)
include_directories(${BROTLI_INCLUDE_DIRS})
target_link_libraries(${project_name} ${BROTLI_LIBRARIES})
#target_link_libraries(benchmark PRIVATE ${BROTLI_LIBRARIES})
endif()

if (ENABLE_SIMD STREQUAL "AARCH64")
if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "aarch64")
add_library(neon INTERFACE IMPORTED)
Expand Down
82 changes: 82 additions & 0 deletions include/cinatra/brzip.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
#pragma once
#include <string>
#include <string_view>
#include <array>
#include <sstream>
#include <brotli/decode.h>
#include <brotli/encode.h>

namespace cinatra::br_codec {

#define BROTLI_BUFFER_SIZE 1024

inline bool brotli_compress(std::string_view input, std::string &output)
{
auto instance = BrotliEncoderCreateInstance(nullptr, nullptr, nullptr);
std::array<uint8_t, BROTLI_BUFFER_SIZE> buffer;
std::stringstream result;

size_t available_in = input.size(), available_out = buffer.size();
const uint8_t* next_in = reinterpret_cast<const uint8_t*>(input.data());
uint8_t* next_out = buffer.data();

do
{
int ret = BrotliEncoderCompressStream
(
instance, BROTLI_OPERATION_FINISH,
&available_in, &next_in, &available_out, &next_out, nullptr
);
if (!ret)
return false;
result.write(reinterpret_cast<const char*>(buffer.data()), buffer.size() - available_out);
available_out = buffer.size();
next_out = buffer.data();
}
while (!(available_in == 0 && BrotliEncoderIsFinished(instance)));

BrotliEncoderDestroyInstance(instance);
output = result.str();
return true;
}

inline bool brotli_decompress(std::string_view input, std::string &decompressed)
{
if (input.size() == 0)
return false;

size_t available_in = input.size();
auto next_in = (const uint8_t *)(input.data());
decompressed = std::string(available_in * 3, 0);
size_t available_out = decompressed.size();
auto next_out = (uint8_t *)(decompressed.data());
size_t total_out{0};
bool done = false;
auto s = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr);
while (!done) {
auto result = BrotliDecoderDecompressStream(
s, &available_in, &next_in, &available_out, &next_out, &total_out);
if (result == BROTLI_DECODER_RESULT_SUCCESS) {
decompressed.resize(total_out);
BrotliDecoderDestroyInstance(s);
return true;
}
else if (result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) {
if (total_out != decompressed.size()) {
return false;
}
decompressed.resize(total_out * 2);
next_out = (uint8_t *)(decompressed.data() + total_out);
available_out = total_out;
}
else {
decompressed.resize(0);
BrotliDecoderDestroyInstance(s);
return true;
}
}
return true;
}
}

// namespace cinatra::br_codec
43 changes: 37 additions & 6 deletions include/cinatra/coro_http_client.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@
#ifdef CINATRA_ENABLE_GZIP
#include "gzip.hpp"
#endif
#ifdef CINATRA_ENABLE_BROTLI
#include "brzip.hpp"
#endif
#include "cinatra_log_wrapper.hpp"
#include "http_parser.hpp"
#include "multipart.hpp"
Expand Down Expand Up @@ -99,6 +102,7 @@ struct resp_data {
int status = 0;
bool eof = false;
std::string_view resp_body;
std::string br_data;
std::span<http_header> resp_headers;
#ifdef BENCHMARK_TEST
uint64_t total = 0;
Expand Down Expand Up @@ -1436,19 +1440,19 @@ class coro_http_client : public std::enable_shared_from_this<coro_http_client> {
if (is_redirect)
redirect_uri_ = parser_.get_header_value("Location");

#ifdef CINATRA_ENABLE_GZIP
if (!parser_.get_header_value("Content-Encoding").empty()) {
if (parser_.get_header_value("Content-Encoding").find("gzip") !=
std::string_view::npos)
encoding_type_ = content_encoding::gzip;
else if (parser_.get_header_value("Content-Encoding").find("deflate") !=
std::string_view::npos)
encoding_type_ = content_encoding::deflate;
else if (parser_.get_header_value("Content-Encoding").find("br") != std::string_view::npos)
encoding_type_ = content_encoding::br;
}
else {
encoding_type_ = content_encoding::none;
}
#endif

size_t content_len = (size_t)parser_.body_len();
#ifdef BENCHMARK_TEST
Expand Down Expand Up @@ -1563,18 +1567,45 @@ class coro_http_client : public std::enable_shared_from_this<coro_http_client> {
data.resp_body = unziped_str;
else
data.resp_body = reply;

head_buf_.consume(content_len);
data.eof = (head_buf_.size() == 0);
co_return;
}
else if (encoding_type_ == content_encoding::deflate) {

if (encoding_type_ == content_encoding::deflate) {
std::string inflate_str;
bool r = gzip_codec::inflate(reply, inflate_str);
if (r)
data.resp_body = inflate_str;
else
data.resp_body = reply;

head_buf_.consume(content_len);
data.eof = (head_buf_.size() == 0);
co_return;
}
#endif

#ifdef CINATRA_ENABLE_BROTLI
if (encoding_type_ == content_encoding::br)
{
std::string unbr_str;
bool r = br_codec::brotli_decompress(reply, unbr_str);
if (r) {
data.resp_body = unbr_str;
data.br_data = unbr_str;
}
else
data.resp_body = reply;

head_buf_.consume(content_len);
data.eof = (head_buf_.size() == 0);
co_return;
}
else
#endif
data.resp_body = reply;

data.resp_body = reply;

head_buf_.consume(content_len);
}
Expand Down Expand Up @@ -2099,8 +2130,8 @@ class coro_http_client : public std::enable_shared_from_this<coro_http_client> {
bool enable_ws_deflate_ = false;
bool is_server_support_ws_deflate_ = false;
std::string inflate_str_;
content_encoding encoding_type_ = content_encoding::none;
#endif
content_encoding encoding_type_ = content_encoding::none;

#ifdef BENCHMARK_TEST
bool stop_bench_ = false;
Expand Down
2 changes: 2 additions & 0 deletions include/cinatra/coro_http_request.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,8 @@ class coro_http_request {
return content_encoding::gzip;
else if (encoding_type.find("deflate") != std::string_view::npos)
return content_encoding::deflate;
else if (encoding_type.find("br") != std::string_view::npos)
return content_encoding::br;
else
return content_encoding::none;
}
Expand Down
53 changes: 44 additions & 9 deletions include/cinatra/coro_http_response.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@
#ifdef CINATRA_ENABLE_GZIP
#include "gzip.hpp"
#endif
#ifdef CINATRA_ENABLE_BROTLI
#include "brzip.hpp"
#endif
#include "picohttpparser.h"
#include "response_cv.hpp"
#include "time_util.hpp"
Expand Down Expand Up @@ -69,7 +72,7 @@ class coro_http_response {
if (client_encoding_type.empty() ||
client_encoding_type.find("gzip") != std::string_view::npos) {
std::string encode_str;
bool r = gzip_codec::compress(content, encode_str, true);
bool r = gzip_codec::compress(content, encode_str);
if (!r) {
set_status_and_content(status_type::internal_server_error,
"gzip compress error");
Expand All @@ -87,8 +90,11 @@ class coro_http_response {
content_ = std::move(content);
}
}
has_set_content_ = true;
return;
}
else if (encoding == content_encoding::deflate) {

if (encoding == content_encoding::deflate) {
if (client_encoding_type.empty() ||
client_encoding_type.find("deflate") != std::string_view::npos) {
std::string deflate_str;
Expand All @@ -110,16 +116,45 @@ class coro_http_response {
content_ = std::move(content);
}
}
has_set_content_ = true;
return;
}
else
#endif
{
if (is_view) {
content_view_ = content;
}
else {
content_ = std::move(content);

#ifdef CINATRA_ENABLE_BROTLI
if (encoding == content_encoding::br)
{
if (client_encoding_type.empty() ||
client_encoding_type.find("br") != std::string_view::npos) {
std::string br_str;
bool r = br_codec::brotli_compress(content, br_str);
if (!r) {
set_status_and_content(status_type::internal_server_error,
"br compress error");
}
else {
add_header("Content-Encoding", "br");
set_content(std::move(br_str));
}
}
else {
if (is_view) {
content_view_ = content;
}
else {
content_ = std::move(content);
}
}
has_set_content_ = true;
return;
}
#endif

if (is_view) {
content_view_ = content;
}
else {
content_ = std::move(content);
}
has_set_content_ = true;
}
Expand Down
2 changes: 1 addition & 1 deletion include/cinatra/define.h
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ enum class http_method {
OPTIONS,
DEL,
};
enum class content_encoding { gzip, deflate, none };
enum class content_encoding { gzip, deflate, br, none };
constexpr inline auto GET = http_method::GET;
constexpr inline auto POST = http_method::POST;
constexpr inline auto DEL = http_method::DEL;
Expand Down
10 changes: 10 additions & 0 deletions press_tool/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,11 @@ if (ENABLE_GZIP)
target_link_libraries(${project_name} ${ZLIB_LIBRARIES})
endif()

if (ENABLE_BROTLI)
include_directories(${BROTLI_INCLUDE_DIRS})
target_link_libraries(${project_name} ${BROTLI_LIBRARIES})
endif()

if (ENABLE_SIMD STREQUAL "AARCH64")
if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "aarch64")
add_library(neon INTERFACE IMPORTED)
Expand Down Expand Up @@ -76,4 +81,9 @@ if (ENABLE_PRESS_TOOL_TESTS)
target_link_libraries(${unittest_press_tool} ${ZLIB_LIBRARIES})
endif()

if (ENABLE_BROTLI)
include_directories(${BROTLI_INCLUDE_DIRS})
target_link_libraries(${unittest_press_tool} ${BROTLI_LIBRARIES})
endif()

endif()
5 changes: 5 additions & 0 deletions tests/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,11 @@ if (ENABLE_GZIP)
target_link_libraries(${project_name} ${ZLIB_LIBRARIES})
endif()

if (ENABLE_BROTLI)
include_directories(${BROTLI_INCLUDE_DIRS})
target_link_libraries(${project_name} ${BROTLI_LIBRARIES})
endif()

# test_coro_file
option(ENABLE_FILE_IO_URING "enable io_uring" OFF)
if(ENABLE_FILE_IO_URING)
Expand Down
Loading

0 comments on commit adb2d2a

Please sign in to comment.