Skip to content

Commit

Permalink
feat: add file-content mapping (qicosmos#449)
Browse files Browse the repository at this point in the history
  • Loading branch information
@helintongh
helintongh authored Nov 27, 2023
1 parent 1a3dea8 commit 6a93302
Show file tree
Hide file tree
Showing 4 changed files with 163 additions and 38 deletions.
7 changes: 6 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -247,13 +247,18 @@ cinatra目前支持了multipart和octet-stream格式的上传。
//chunked download
//cinatra will send you the file, if the file is big file(more than 5M) the file will be downloaded by chunked. support continues download

同时可以使用set_http_file_server(std::string path);函数来将cinatra转换为一个http文件下载服务器，path为该文件服务器的路径。示例如下:
http下载还提供了两个函数来优化其功能:
1. 通过set_http_file_server(std::string path)函数来将cinatra转换为一个http文件下载服务器。当访问 http://ip:port/path 的时候会展示能够下载的文件
2. 通过set_file_mapping(std::size_t file_max_size)函数可以建立请求路径与文件缓存的映射，开启此选项server初始化时会读取设置的静态文件路径下小于file_max_size的所有文件，当客户端访问时服务器不会读文件而是直接返回文件缓存。此选项可以通过内存来优化服务器性能。

示例如下:

#include "cinatra.hpp"
using namespace cinatra;

int main() {
http_server server(std::thread::hardware_concurrency());
server.set_file_mapping();
server.set_http_file_server("http_file_server");
server.listen("0.0.0.0", "8080");
// 略
Expand Down
32 changes: 32 additions & 0 deletions include/cinatra/connection.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -282,6 +282,38 @@ class connection : public base_connection,
});
}

void write_chunked_data_v2(std::string_view buf, bool eof) {
std::vector<asio::const_buffer> buffers =
to_chunked_buffers<asio::const_buffer>(buf.data(), buf.length(),
chunk_size_str_, eof);

if (buffers.empty()) {
handle_write(std::error_code{});
return;
}

auto self = this->shared_from_this();
asio::async_write(socket(), buffers,
[this, self, buf = std::move(buf), eof](
const std::error_code &ec, size_t) {
if (ec) {
return;
}

if (eof) {
req_.set_state(data_proc_state::data_end);
}
else {
req_.set_state(data_proc_state::data_continue);
}

call_back();
if (keep_alive_) {
do_read();
}
});
}

void write_ranges_data(std::string &&buf, bool eof) {
reset_timer();

Expand Down
158 changes: 122 additions & 36 deletions include/cinatra/http_server.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,10 @@ class http_server_ : private noncopyable {
download_display_file_dir_ = path;
}

void set_file_mapping(std::size_t file_max_size = 3145728) {
build_file_to_content_mapping(file_max_size);
}

const std::string &static_dir() const { return static_dir_; }

// xM
Expand Down Expand Up @@ -359,6 +363,8 @@ class http_server_ : private noncopyable {
}
}

bool is_cache_store = false;
std::string cache_file_name = "";
auto state = req.get_state();
switch (state) {
case cinatra::data_proc_state::data_begin: {
Expand All @@ -367,50 +373,64 @@ class http_server_ : private noncopyable {
std::string relative_file_name(u8relative_file_name.begin(),
u8relative_file_name.end());

std::string fullpath = static_dir_ + relative_file_name;

auto mime = req.get_mime(relative_file_name);
auto in = std::make_shared<std::ifstream>(fullpath,
std::ios_base::binary);
if (!in->is_open()) {
if (not_found_) {
not_found_(req, res);
cache_file_name = relative_file_name.substr(1);
is_cache_store = (file_content_map_.find(cache_file_name) !=
file_content_map_.end());
if (is_cache_store) {
auto mime = req.get_mime(relative_file_name);
send_small_file(res, file_content_map_[cache_file_name], mime);
write_chunked_header(req, mime);
write_chunked_body(req, file_content_map_[cache_file_name]);
break;
}
else {
std::string fullpath = static_dir_ + relative_file_name;

auto mime = req.get_mime(relative_file_name);
auto in = std::make_shared<std::ifstream>(
fullpath, std::ios_base::binary);
if (!in->is_open()) {
if (not_found_) {
not_found_(req, res);
return;
}
res.set_status_and_content(
status_type::not_found,
std::string(relative_file_name) + " not found");
return;
}
res.set_status_and_content(
status_type::not_found,
std::string(relative_file_name) + " not found");
return;
}

auto start_sv = req.get_header_value("cinatra_start_pos");
if (!start_sv.empty()) {
std::string start_str(start_sv);
int64_t start = (int64_t)atoll(start_str.data());
std::error_code code;
int64_t file_size = fs::file_size(fullpath, code);
if (start > 0 && !code && file_size >= start) {
in->seekg(start);
auto start_sv = req.get_header_value("cinatra_start_pos");
if (!start_sv.empty()) {
std::string start_str(start_sv);
int64_t start = (int64_t)atoll(start_str.data());
std::error_code code;
int64_t file_size = fs::file_size(fullpath, code);
if (start > 0 && !code && file_size >= start) {
in->seekg(start);
}
}
}

req.get_conn<ScoketType>()->set_tag(in);
req.get_conn<ScoketType>()->set_tag(in);

if (is_small_file(in.get(), req)) {
send_small_file(res, in.get(), mime);
return;
}
if (is_small_file(in.get(), req)) {
send_small_file(res, in.get(), mime);
return;
}

if (transfer_type_ == transfer_type::CHUNKED)
write_chunked_header(req, in, mime);
else
write_ranges_header(
req, mime, fs::path(relative_file_name).filename().string(),
std::to_string(fs::file_size(fullpath)));
if (transfer_type_ == transfer_type::CHUNKED)
write_chunked_header(req, in, mime);
else
write_ranges_header(
req, mime,
fs::path(relative_file_name).filename().string(),
std::to_string(fs::file_size(fullpath)));
}
} break;
case cinatra::data_proc_state::data_continue: {
if (transfer_type_ == transfer_type::CHUNKED)
if (transfer_type_ == transfer_type::CHUNKED) {
write_chunked_body(req);
}
else
write_ranges_data(req);
} break;
Expand Down Expand Up @@ -462,6 +482,26 @@ class http_server_ : private noncopyable {
#endif
}

void send_small_file(response &res, std::string_view in,
std::string_view mime) {
res.add_header("Access-Control-Allow-origin", "*");
res.add_header("Content-type",
std::string(mime.data(), mime.size()) + "; charset=utf8");
std::stringstream file_buffer;
file_buffer << std::string(in.data(), in.size());
if (static_res_cache_max_age_ > 0) {
std::string max_age =
std::string("max-age=") + std::to_string(static_res_cache_max_age_);
res.add_header("Cache-Control", max_age.data());
}
#ifdef CINATRA_ENABLE_GZIP
res.set_status_and_content(status_type::ok, file_buffer.str(),
req_content_type::none, content_encoding::gzip);
#else
res.set_status_and_content(status_type::ok, file_buffer.str());
#endif
}

void write_chunked_header(request &req, std::shared_ptr<std::ifstream> in,
std::string_view mime) {
auto range_header = req.get_header_value("range");
Expand Down Expand Up @@ -495,6 +535,28 @@ class http_server_ : private noncopyable {
std::string_view(res_content_header), req.is_range());
}

void write_chunked_header(request &req, std::string_view mime) {
auto range_header = req.get_header_value("range");
req.set_range_flag(!range_header.empty());
req.set_range_start_pos(range_header);

std::string res_content_header =
std::string(mime.data(), mime.size()) + "; charset=utf8";
res_content_header +=
std::string("\r\n") + std::string("Access-Control-Allow-origin: *");
res_content_header +=
std::string("\r\n") + std::string("Accept-Ranges: bytes");
if (static_res_cache_max_age_ > 0) {
std::string max_age =
std::string("max-age=") + std::to_string(static_res_cache_max_age_);
res_content_header +=
std::string("\r\n") + std::string("Cache-Control: ") + max_age;
}

req.get_conn<ScoketType>()->write_chunked_header(
std::string_view(res_content_header), req.is_range());
}

void write_chunked_body(request &req) {
const size_t len = 3 * 1024 * 1024;
auto str = get_send_data(req, len);
Expand All @@ -503,6 +565,10 @@ class http_server_ : private noncopyable {
req.get_conn<ScoketType>()->write_chunked_data(std::move(str), eof);
}

void write_chunked_body(request &req, std::string_view file_cache) {
req.get_conn<ScoketType>()->write_chunked_data_v2(file_cache, true);
}

void write_ranges_header(request &req, std::string_view mime,
std::string filename, std::string file_size) {
std::string header_str =
Expand Down Expand Up @@ -602,7 +668,7 @@ class http_server_ : private noncopyable {

int get_static_dir_filenames(const std::string &dir,
std::vector<std::string> &filenames,
size_t dir_name_length) {
size_t dir_name_length, bool skip = false) {
fs::path path(dir);
if (!fs::exists(path))
return -1;
Expand All @@ -616,7 +682,10 @@ class http_server_ : private noncopyable {
auto u8_path_name = iter->path().u8string();
std::string relative_path(u8_path_name.begin(), u8_path_name.end());
size_t length = relative_path.length();
relative_path = relative_path.substr(dir_name_length + 1, length);
if (!skip)
relative_path = relative_path.substr(dir_name_length + 1, length);
else
relative_path = relative_path.substr(dir_name_length, length);
filenames.push_back(relative_path);
}
}
Expand Down Expand Up @@ -648,6 +717,21 @@ class http_server_ : private noncopyable {
}
}

void build_file_to_content_mapping(std::size_t file_size = 3145728) {
std::vector<std::string> files;
size_t static_dir_length = static_dir_.length();
get_static_dir_filenames(static_dir_, files, static_dir_length, true);
for (const auto &file : files) {
std::string full_path = static_dir_ + file;
std::ifstream ifs(full_path);
std::string content((std::istreambuf_iterator<char>(ifs)),
(std::istreambuf_iterator<char>()));
if (content.size() < file_size) {
file_content_map_[file.substr(1)] = content;
}
}
}

service_pool_policy io_service_pool_;

std::size_t max_req_buf_size_ =
Expand Down Expand Up @@ -683,6 +767,8 @@ class http_server_ : private noncopyable {
std::unordered_map<uint64_t, std::shared_ptr<connection<ScoketType>>> conns_;
std::mutex conns_mtx_;

std::unordered_map<std::string, std::string> file_content_map_;

std::string download_dir_response_head_ =
"<html>"
"<head><meta charset=\"utf-8\"><title>http file download "
Expand Down
4 changes: 3 additions & 1 deletion lang/english/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -249,14 +249,16 @@ http://127.0.0.1:8080/purecpp/static/show.jpg
//cinatra will send you the file, if the file is big file(more than 5M) the file will be downloaded by chunked. support continues download
```

At the same time, you can use the `set_http_file_server(std::string path);` function to build an http file download server. The parameter path is the path of all downloadable files. Examples are as follows:
1. Use the `set_http_file_server(std::string path)` function to convert cinatra into an http file download server. When accessing http://ip:port/path, files that can be downloaded will be displayed.
2. The `set_file_mapping(std::size_t file_max_size)` function can be used to establish the mapping between the request path and the file cache. After turning on this option, the server will read all files smaller than file_max_size in the set static file path. When the client accesses the server, the server will directly return the file cache. This option optimizes server performance through memory.

```cpp
#include "cinatra.hpp"
using namespace cinatra;

int main() {
http_server server(std::thread::hardware_concurrency());
server.set_file_mapping();
server.set_http_file_server("http_file_server");
server.listen("0.0.0.0", "8080");
// ......
Expand Down

0 comments on commit 6a93302

Please sign in to comment.