From 6a93302632d1dac6060063f0aee1497ed090a696 Mon Sep 17 00:00:00 2001 From: helintongh Date: Mon, 27 Nov 2023 10:46:00 +0800 Subject: [PATCH] feat: add file-content mapping (#449) --- README.md | 7 +- include/cinatra/connection.hpp | 32 +++++++ include/cinatra/http_server.hpp | 158 ++++++++++++++++++++++++-------- lang/english/README.md | 4 +- 4 files changed, 163 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 39de75a3..14d4a19d 100644 --- a/README.md +++ b/README.md @@ -247,13 +247,18 @@ cinatra目前支持了multipart和octet-stream格式的上传。 //chunked download //cinatra will send you the file, if the file is big file(more than 5M) the file will be downloaded by chunked. support continues download - 同时可以使用set_http_file_server(std::string path);函数来将cinatra转换为一个http文件下载服务器，path为该文件服务器的路径。示例如下: + http下载还提供了两个函数来优化其功能: + 1. 通过set_http_file_server(std::string path)函数来将cinatra转换为一个http文件下载服务器。当访问 http://ip:port/path 的时候会展示能够下载的文件 + 2. 通过set_file_mapping(std::size_t file_max_size)函数可以建立请求路径与文件缓存的映射，开启此选项server初始化时会读取设置的静态文件路径下小于file_max_size的所有文件，当客户端访问时服务器不会读文件而是直接返回文件缓存。此选项可以通过内存来优化服务器性能。 + + 示例如下: #include "cinatra.hpp" using namespace cinatra; int main() { http_server server(std::thread::hardware_concurrency()); + server.set_file_mapping(); server.set_http_file_server("http_file_server"); server.listen("0.0.0.0", "8080"); // 略 diff --git a/include/cinatra/connection.hpp b/include/cinatra/connection.hpp index 442268fa..fe0bf135 100644 --- a/include/cinatra/connection.hpp +++ b/include/cinatra/connection.hpp @@ -282,6 +282,38 @@ class connection : public base_connection, }); } + void write_chunked_data_v2(std::string_view buf, bool eof) { + std::vector buffers = + to_chunked_buffers(buf.data(), buf.length(), + chunk_size_str_, eof); + + if (buffers.empty()) { + handle_write(std::error_code{}); + return; + } + + auto self = this->shared_from_this(); + asio::async_write(socket(), buffers, + [this, self, buf = std::move(buf), eof]( + const std::error_code &ec, size_t) { + if (ec) { + return; + } + + if (eof) { + req_.set_state(data_proc_state::data_end); + } + else { + req_.set_state(data_proc_state::data_continue); + } + + call_back(); + if (keep_alive_) { + do_read(); + } + }); + } + void write_ranges_data(std::string &&buf, bool eof) { reset_timer(); diff --git a/include/cinatra/http_server.hpp b/include/cinatra/http_server.hpp index fdc4b92c..dcdcd5c6 100644 --- a/include/cinatra/http_server.hpp +++ b/include/cinatra/http_server.hpp @@ -177,6 +177,10 @@ class http_server_ : private noncopyable { download_display_file_dir_ = path; } + void set_file_mapping(std::size_t file_max_size = 3145728) { + build_file_to_content_mapping(file_max_size); + } + const std::string &static_dir() const { return static_dir_; } // xM @@ -359,6 +363,8 @@ class http_server_ : private noncopyable { } } + bool is_cache_store = false; + std::string cache_file_name = ""; auto state = req.get_state(); switch (state) { case cinatra::data_proc_state::data_begin: { @@ -367,50 +373,64 @@ class http_server_ : private noncopyable { std::string relative_file_name(u8relative_file_name.begin(), u8relative_file_name.end()); - std::string fullpath = static_dir_ + relative_file_name; - - auto mime = req.get_mime(relative_file_name); - auto in = std::make_shared(fullpath, - std::ios_base::binary); - if (!in->is_open()) { - if (not_found_) { - not_found_(req, res); + cache_file_name = relative_file_name.substr(1); + is_cache_store = (file_content_map_.find(cache_file_name) != + file_content_map_.end()); + if (is_cache_store) { + auto mime = req.get_mime(relative_file_name); + send_small_file(res, file_content_map_[cache_file_name], mime); + write_chunked_header(req, mime); + write_chunked_body(req, file_content_map_[cache_file_name]); + break; + } + else { + std::string fullpath = static_dir_ + relative_file_name; + + auto mime = req.get_mime(relative_file_name); + auto in = std::make_shared( + fullpath, std::ios_base::binary); + if (!in->is_open()) { + if (not_found_) { + not_found_(req, res); + return; + } + res.set_status_and_content( + status_type::not_found, + std::string(relative_file_name) + " not found"); return; } - res.set_status_and_content( - status_type::not_found, - std::string(relative_file_name) + " not found"); - return; - } - auto start_sv = req.get_header_value("cinatra_start_pos"); - if (!start_sv.empty()) { - std::string start_str(start_sv); - int64_t start = (int64_t)atoll(start_str.data()); - std::error_code code; - int64_t file_size = fs::file_size(fullpath, code); - if (start > 0 && !code && file_size >= start) { - in->seekg(start); + auto start_sv = req.get_header_value("cinatra_start_pos"); + if (!start_sv.empty()) { + std::string start_str(start_sv); + int64_t start = (int64_t)atoll(start_str.data()); + std::error_code code; + int64_t file_size = fs::file_size(fullpath, code); + if (start > 0 && !code && file_size >= start) { + in->seekg(start); + } } - } - req.get_conn()->set_tag(in); + req.get_conn()->set_tag(in); - if (is_small_file(in.get(), req)) { - send_small_file(res, in.get(), mime); - return; - } + if (is_small_file(in.get(), req)) { + send_small_file(res, in.get(), mime); + return; + } - if (transfer_type_ == transfer_type::CHUNKED) - write_chunked_header(req, in, mime); - else - write_ranges_header( - req, mime, fs::path(relative_file_name).filename().string(), - std::to_string(fs::file_size(fullpath))); + if (transfer_type_ == transfer_type::CHUNKED) + write_chunked_header(req, in, mime); + else + write_ranges_header( + req, mime, + fs::path(relative_file_name).filename().string(), + std::to_string(fs::file_size(fullpath))); + } } break; case cinatra::data_proc_state::data_continue: { - if (transfer_type_ == transfer_type::CHUNKED) + if (transfer_type_ == transfer_type::CHUNKED) { write_chunked_body(req); + } else write_ranges_data(req); } break; @@ -462,6 +482,26 @@ class http_server_ : private noncopyable { #endif } + void send_small_file(response &res, std::string_view in, + std::string_view mime) { + res.add_header("Access-Control-Allow-origin", "*"); + res.add_header("Content-type", + std::string(mime.data(), mime.size()) + "; charset=utf8"); + std::stringstream file_buffer; + file_buffer << std::string(in.data(), in.size()); + if (static_res_cache_max_age_ > 0) { + std::string max_age = + std::string("max-age=") + std::to_string(static_res_cache_max_age_); + res.add_header("Cache-Control", max_age.data()); + } +#ifdef CINATRA_ENABLE_GZIP + res.set_status_and_content(status_type::ok, file_buffer.str(), + req_content_type::none, content_encoding::gzip); +#else + res.set_status_and_content(status_type::ok, file_buffer.str()); +#endif + } + void write_chunked_header(request &req, std::shared_ptr in, std::string_view mime) { auto range_header = req.get_header_value("range"); @@ -495,6 +535,28 @@ class http_server_ : private noncopyable { std::string_view(res_content_header), req.is_range()); } + void write_chunked_header(request &req, std::string_view mime) { + auto range_header = req.get_header_value("range"); + req.set_range_flag(!range_header.empty()); + req.set_range_start_pos(range_header); + + std::string res_content_header = + std::string(mime.data(), mime.size()) + "; charset=utf8"; + res_content_header += + std::string("\r\n") + std::string("Access-Control-Allow-origin: *"); + res_content_header += + std::string("\r\n") + std::string("Accept-Ranges: bytes"); + if (static_res_cache_max_age_ > 0) { + std::string max_age = + std::string("max-age=") + std::to_string(static_res_cache_max_age_); + res_content_header += + std::string("\r\n") + std::string("Cache-Control: ") + max_age; + } + + req.get_conn()->write_chunked_header( + std::string_view(res_content_header), req.is_range()); + } + void write_chunked_body(request &req) { const size_t len = 3 * 1024 * 1024; auto str = get_send_data(req, len); @@ -503,6 +565,10 @@ class http_server_ : private noncopyable { req.get_conn()->write_chunked_data(std::move(str), eof); } + void write_chunked_body(request &req, std::string_view file_cache) { + req.get_conn()->write_chunked_data_v2(file_cache, true); + } + void write_ranges_header(request &req, std::string_view mime, std::string filename, std::string file_size) { std::string header_str = @@ -602,7 +668,7 @@ class http_server_ : private noncopyable { int get_static_dir_filenames(const std::string &dir, std::vector &filenames, - size_t dir_name_length) { + size_t dir_name_length, bool skip = false) { fs::path path(dir); if (!fs::exists(path)) return -1; @@ -616,7 +682,10 @@ class http_server_ : private noncopyable { auto u8_path_name = iter->path().u8string(); std::string relative_path(u8_path_name.begin(), u8_path_name.end()); size_t length = relative_path.length(); - relative_path = relative_path.substr(dir_name_length + 1, length); + if (!skip) + relative_path = relative_path.substr(dir_name_length + 1, length); + else + relative_path = relative_path.substr(dir_name_length, length); filenames.push_back(relative_path); } } @@ -648,6 +717,21 @@ class http_server_ : private noncopyable { } } + void build_file_to_content_mapping(std::size_t file_size = 3145728) { + std::vector files; + size_t static_dir_length = static_dir_.length(); + get_static_dir_filenames(static_dir_, files, static_dir_length, true); + for (const auto &file : files) { + std::string full_path = static_dir_ + file; + std::ifstream ifs(full_path); + std::string content((std::istreambuf_iterator(ifs)), + (std::istreambuf_iterator())); + if (content.size() < file_size) { + file_content_map_[file.substr(1)] = content; + } + } + } + service_pool_policy io_service_pool_; std::size_t max_req_buf_size_ = @@ -683,6 +767,8 @@ class http_server_ : private noncopyable { std::unordered_map>> conns_; std::mutex conns_mtx_; + std::unordered_map file_content_map_; + std::string download_dir_response_head_ = "" "http file download " diff --git a/lang/english/README.md b/lang/english/README.md index bf51a5bb..291bf181 100644 --- a/lang/english/README.md +++ b/lang/english/README.md @@ -249,7 +249,8 @@ http://127.0.0.1:8080/purecpp/static/show.jpg //cinatra will send you the file, if the file is big file(more than 5M) the file will be downloaded by chunked. support continues download ``` -At the same time, you can use the `set_http_file_server(std::string path);` function to build an http file download server. The parameter path is the path of all downloadable files. Examples are as follows: +1. Use the `set_http_file_server(std::string path)` function to convert cinatra into an http file download server. When accessing http://ip:port/path, files that can be downloaded will be displayed. +2. The `set_file_mapping(std::size_t file_max_size)` function can be used to establish the mapping between the request path and the file cache. After turning on this option, the server will read all files smaller than file_max_size in the set static file path. When the client accesses the server, the server will directly return the file cache. This option optimizes server performance through memory. ```cpp #include "cinatra.hpp" @@ -257,6 +258,7 @@ using namespace cinatra; int main() { http_server server(std::thread::hardware_concurrency()); + server.set_file_mapping(); server.set_http_file_server("http_file_server"); server.listen("0.0.0.0", "8080"); // ......