Skip to content

Commit

Permalink
Merge branch 'imap-nodiskio'
Browse files Browse the repository at this point in the history
  • Loading branch information
@jengelh
jengelh committed Jan 22, 2025
2 parents 3c827e5 + f51de60 commit a5d2275
Show file tree
Hide file tree
Showing 18 changed files with 859 additions and 1,344 deletions.
2 changes: 2 additions & 0 deletions doc/pop3.8gx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,8 @@ Lower clamp is 256.
Default: \fI1024\fP
.TP
\fBcontext_max_mem\fP
Network buffer per client.
.br
Default: \fI2M\fP
.TP
\fBcontext_num\fP
Expand Down
7 changes: 5 additions & 2 deletions exch/exmdb/names.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
// SPDX-License-Identifier: AGPL-3.0-or-later, OR GPL-2.0-or-later WITH linking exception
// SPDX-FileCopyrightText: 2020 grommunio GmbH
// SPDX-FileCopyrightText: 2025 grommunio GmbH
// This file is part of Gromox.
#include <gromox/defs.h>
#include <gromox/exmdb_common_util.hpp>
Expand Down Expand Up @@ -151,6 +151,9 @@ static constexpr const char *exmdb_rpc_names[] = {
E(movecopy_folder),
E(create_folder),
E(write_message_v2),
E(imapfile_read),
E(imapfile_write),
E(imapfile_delete),
};
#undef E

Expand All @@ -159,7 +162,7 @@ namespace exmdb {
const char *exmdb_rpc_idtoname(exmdb_callid i)
{
auto j = static_cast<uint8_t>(i);
static_assert(std::size(exmdb_rpc_names) == static_cast<uint8_t>(exmdb_callid::write_message_v2) + 1);
static_assert(std::size(exmdb_rpc_names) == static_cast<uint8_t>(exmdb_callid::imapfile_delete) + 1);
auto s = j < std::size(exmdb_rpc_names) ? exmdb_rpc_names[j] : nullptr;
return znul(s);
}
Expand Down
56 changes: 55 additions & 1 deletion exch/exmdb/store2.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
// SPDX-FileCopyrightText: 2022–2024 grommunio GmbH
// SPDX-FileCopyrightText: 2022–2025 grommunio GmbH
// This file is part of Gromox.
#define _GNU_SOURCE 1 /* AT_* */
#include <algorithm>
Expand All @@ -16,6 +16,7 @@
#include <utility>
#include <vector>
#include <fmt/core.h>
#include <libHX/io.h>
#include <libHX/string.h>
#include <sys/stat.h>
#include <gromox/database.h>
Expand Down Expand Up @@ -606,3 +607,56 @@ BOOL exmdb_server::recalc_store_size(const char *dir, uint32_t flags)
*/
return sql_transact.commit() == SQLITE_OK ? TRUE : false;
}

static bool imapfile_type_ok(const std::string &s)
{
return s == "eml" || s == "ext" || s == "tmp/imap.rfc822";
}

BOOL exmdb_server::imapfile_read(const char *dir, const std::string &type,
const std::string &mid, std::string *data)
{
if (!imapfile_type_ok(type) || mid.find('/') != mid.npos)
return false;
size_t slurp_size = 0;
std::unique_ptr<char[], stdlib_delete> slurp_data(HX_slurp_file((dir + "/"s + type + "/" + mid).c_str(), &slurp_size));
if (slurp_data == nullptr)
return false;
data->assign(slurp_data.get(), slurp_size);
return TRUE;
}

BOOL exmdb_server::imapfile_write(const char *dir, const std::string &type,
const std::string &mid, const std::string &data)
{
if (!imapfile_type_ok(type) || mid.find('/') != mid.npos)
return false;
gromox::tmpfile tf;
auto fd = tf.open_linkable(dir, O_WRONLY, FMODE_PRIVATE);
if (fd < 0)
return false;
auto wrret = HXio_fullwrite(fd, data.data(), data.size());
if (wrret < 0 || static_cast<size_t>(wrret) != data.size())
return false;
auto tgt = fmt::format("{}/{}/{}", dir, type, mid);
auto err = tf.link_to(tgt.c_str());
if (err != 0) {
mlog(LV_ERR, "E-1752: link_to %s: %s", tgt.c_str(), strerror(errno));
return false;
}
return TRUE;
}

BOOL exmdb_server::imapfile_delete(const char *dir, const std::string &type,
const std::string &mid)
{
if (!imapfile_type_ok(type) || mid.find('/') != mid.npos)
return false;
auto fn = dir + "/"s + type + "/" + mid;
if (remove(fn.c_str()) < 0 && errno != ENOENT) {
mlog(LV_WARN, "W-1370: remove %s: %s",
fn.c_str(), strerror(errno));
return false;
}
return TRUE;
}
159 changes: 48 additions & 111 deletions exch/midb/mail_engine.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -203,17 +203,6 @@ static std::string make_midb_path(const char *d)
return d + "/exmdb/midb.sqlite3"s;
}

static std::string make_eml_path(const char *d, std::string_view m)
{
/* P2591r5 only for C++26 */
return (d + "/eml/"s) += m;
}

static std::string make_ext_path(const char *d, std::string_view m)
{
return (d + "/ext/"s) += m;
}

static std::unique_ptr<char[]> me_ct_to_utf8(const char *charset,
const char *string) try
{
Expand Down Expand Up @@ -248,39 +237,26 @@ static std::unique_ptr<char[]> me_ct_to_utf8(const char *charset,
static uint64_t me_get_digest(sqlite3 *psqlite, const char *mid_string,
Json::Value &digest) try
{
size_t size;
auto ext_path = make_ext_path(cu_get_maildir(), mid_string);
size_t slurp_size = 0;
std::unique_ptr<char[], stdlib_delete> slurp_data(HX_slurp_file(ext_path.c_str(), &slurp_size));
if (slurp_data != nullptr) {
if (!json_from_str(slurp_data.get(), digest))
auto dir = cu_get_maildir();
std::string slurp_data;
if (exmdb_client::imapfile_read(dir, "ext", mid_string, &slurp_data)) {
if (!json_from_str(slurp_data.c_str(), digest))
return 0;
} else if (errno != ENOENT) {
mlog(LV_ERR, "E-2139: read %s: %s", ext_path.c_str(), strerror(errno));
return 0;
} else {
auto eml_path = make_eml_path(cu_get_maildir(), mid_string);
slurp_data.reset(HX_slurp_file(eml_path.c_str(), &slurp_size));
if (slurp_data == nullptr) {
mlog(LV_ERR, "E-1252: %s: %s", eml_path.c_str(), strerror(errno));
if (!exmdb_client::imapfile_read(dir, "eml", mid_string, &slurp_data))
return 0;
}
MAIL imail;
if (!imail.load_from_str(slurp_data.get(), slurp_size))
if (!imail.load_from_str(slurp_data.c_str(), slurp_data.size()))
return 0;
slurp_data.reset();
size_t size = 0;
if (imail.make_digest(&size, digest) <= 0)
return 0;
imail.clear();
digest["file"] = "";
auto djson = json_to_str(digest);
wrapfd fd = open(ext_path.c_str(), O_CREAT | O_TRUNC | O_WRONLY, FMODE_PRIVATE);
if (fd.get() >= 0) {
if (HXio_fullwrite(fd.get(), djson.c_str(), djson.size()) < 0 ||
fd.close_wr() != 0)
mlog(LV_ERR, "E-2082: write %s: %s", ext_path.c_str(), strerror(errno));
} else {
mlog(LV_ERR, "E-2138: open %s for write: %s", ext_path.c_str(), strerror(errno));
if (!exmdb_client::imapfile_write(dir, "ext", mid_string, djson)) {
mlog(LV_ERR, "E-1754: imapfile_write %s/ext/%s did not complete",
dir, mid_string);
return 0;
}
}
auto pstmt = gx_sql_prep(psqlite, "SELECT uid, recent, read,"
Expand Down Expand Up @@ -396,70 +372,56 @@ static std::unique_ptr<char[]> me_ct_decode_mime(const char *charset,
static void me_ct_enum_mime(MJSON_MIME *pmime, void *param) try
{
auto penum = static_cast<KEYWORD_ENUM *>(param);
size_t temp_len;
const char *charset;
const char *filename;

if (penum->b_result)
return;
if (pmime->get_mtype() != mime_type::single &&
pmime->get_mtype() != mime_type::single_obj)
return;

if (strncmp(pmime->get_ctype(), "text/", 5) != 0) {
filename = pmime->get_filename();
auto filename = pmime->get_filename();
if ('\0' != filename[0]) {
auto rs = me_ct_decode_mime(penum->charset, filename);
if (rs != nullptr &&
strcasestr(rs.get(), penum->keyword) != nullptr)
penum->b_result = TRUE;
}
}
auto length = pmime->get_content_length();
auto pbuff = std::make_unique<char[]>(2 * length + 1);
auto fd = penum->pjson->seek_fd(pmime->get_id(), MJSON_MIME_CONTENT);
if (fd == -1)
return;
auto read_len = HXio_fullread(fd, pbuff.get(), length);
if (read_len < 0 || static_cast<size_t>(read_len) != length)
std::string content;
if (!exmdb_client::imapfile_read(cu_get_maildir(), "eml",
penum->pjson->filename, &content))
return;
std::string_view ctview(content.data() + pmime->begin,
std::min(content.size(), pmime->get_content_length()));
if (strcasecmp(pmime->get_encoding(), "base64") == 0) {
if (decode64_ex(pbuff.get(), length, &pbuff[length],
length, &temp_len) != 0)
return;
pbuff[length + temp_len] = '\0';
content = base64_decode(ctview);
} else if (strcasecmp(pmime->get_encoding(), "quoted-printable") == 0) {
auto xl = qp_decode_ex(&pbuff[length], length, pbuff.get(), length);
auto xl = qp_decode_ex(&content[0], content.size(), content.c_str(), content.size());
if (xl < 0)
return;
temp_len = xl;
pbuff[length + temp_len] = '\0';
} else {
memcpy(&pbuff[length], pbuff.get(), length);
pbuff[2*length] = '\0';
content.resize(xl);
}

charset = pmime->get_charset();
auto charset = pmime->get_charset();
auto rs = me_ct_to_utf8(*charset != '\0' ?
charset : penum->charset, &pbuff[length]);
charset : penum->charset, content.c_str());
if (rs != nullptr && strcasestr(rs.get(), penum->keyword) != nullptr)
penum->b_result = TRUE;
} catch (const std::bad_alloc &) {
mlog(LV_ERR, "E-1970: ENOMEM");
}

static bool me_ct_search_head(const char *charset,
const char *file_path, const char *tag, const char *value)
static bool me_ct_search_head(const char *charset, const char *mid_string,
const char *tag, const char *value)
{
size_t slurp_size = 0;
std::unique_ptr<char[], stdlib_delete> ct(HX_slurp_file(file_path, &slurp_size));
if (ct == nullptr)
std::string content;
if (!exmdb_client::imapfile_read(cu_get_maildir(), "eml",
mid_string, &content))
return false;

vmime::parsingContext vpctx;
vpctx.setInternationalizedEmailSupport(true); /* RFC 6532 */
vmime::header hdr;
hdr.parse(vpctx, std::string(ct.get(), slurp_size));
hdr.parse(vpctx, content);

for (const auto &hf : hdr.getFieldList()) {
auto hk = hf->getName();
Expand Down Expand Up @@ -634,8 +596,7 @@ static bool me_ct_match_mail(sqlite3 *psqlite, const char *charset,
break;
}
case midb_cond::header:
b_result1 = me_ct_search_head(charset,
make_eml_path(cu_get_maildir(), mid_string).c_str(),
b_result1 = me_ct_search_head(charset, mid_string,
ptree_node->ct_headers[0],
ptree_node->ct_headers[1]);
break;
Expand Down Expand Up @@ -1371,15 +1332,9 @@ static void me_insert_message(xstmt &stm_insert, uint32_t *puidnext,

auto dir = cu_get_maildir();
std::string djson;
if (e.midstr.size() > 0) {
auto ext_path = make_ext_path(dir, e.midstr);
size_t slurp_size = 0;
std::unique_ptr<char[], stdlib_delete> slurp_data(HX_slurp_file(ext_path.c_str(), &slurp_size));
if (slurp_data == nullptr)
e.midstr.clear();
else
djson.assign(slurp_data.get(), slurp_size);
}
if (e.midstr.size() > 0 &&
!exmdb_client::imapfile_read(dir, "ext", e.midstr, &djson))
e.midstr.clear();
if (e.midstr.empty()) {
if (!cu_switch_allocator())
return;
Expand Down Expand Up @@ -1412,28 +1367,18 @@ static void me_insert_message(xstmt &stm_insert, uint32_t *puidnext,
digest["file"] = "";
djson = json_to_str(digest);
e.midstr = std::to_string(time(nullptr)) + "." + std::to_string(++g_sequence_id) + ".midb";
auto ext_path = make_ext_path(dir, e.midstr);
wrapfd fd = open(ext_path.c_str(), O_CREAT | O_TRUNC | O_WRONLY, FMODE_PRIVATE);
if (fd.get() < 0) {
mlog(LV_ERR, "E-1770: open %s for write: %s", ext_path.c_str(), strerror(errno));
return;
}
if (HXio_fullwrite(fd.get(), djson.c_str(), djson.size()) < 0 ||
fd.close_wr() != 0) {
mlog(LV_ERR, "E-1134: write %s: %s", ext_path.c_str(), strerror(errno));
if (!exmdb_client::imapfile_write(dir, "ext", e.midstr, djson)) {
mlog(LV_ERR, "E-1770: imapfile_write %s/ext/%s incomplete", dir, e.midstr.c_str());
return;
}
auto eml_path = make_eml_path(dir, e.midstr);
fd = open(eml_path.c_str(), O_CREAT | O_TRUNC | O_WRONLY, FMODE_PRIVATE);
if (fd.get() < 0) {
mlog(LV_ERR, "E-1771: open %s for write: %s", eml_path.c_str(), strerror(errno));
std::string emlcontent;
auto err = imail.to_str(emlcontent);
if (err != 0) {
mlog(LV_ERR, "E-1771: imail.to_string failed: %s", strerror(err));
return;
}
auto err = imail.to_fd(fd.get());
if (err == 0)
err = fd.close_wr();
if (err != 0) {
mlog(LV_ERR, "E-1772: to_file %s failed: %s", eml_path.c_str(), strerror(err));
if (!exmdb_client::imapfile_write(dir, "eml", e.midstr, emlcontent)) {
mlog(LV_ERR, "E-1772: imapfile_write %s/eml/%s failed", dir, e.midstr.c_str());
return;
}
}
Expand Down Expand Up @@ -2173,32 +2118,24 @@ static int me_minst(int argc, char **argv, int sockd) try

uint8_t b_unsent = strchr(argv[4], midb_flag::unsent) != nullptr;
uint8_t b_read = strchr(argv[4], midb_flag::seen) != nullptr;
auto eml_path = make_eml_path(argv[1], argv[3]);
size_t slurp_size = 0;
std::unique_ptr<char[], stdlib_delete> pbuff(HX_slurp_file(eml_path.c_str(), &slurp_size));
if (pbuff == nullptr) {
mlog(LV_ERR, "E-2071: read %s: %s", eml_path.c_str(), strerror(errno));
return errno == ENOMEM ? MIDB_E_NO_MEMORY : MIDB_E_DISK_ERROR;
std::string pbuff;
if (!exmdb_client::imapfile_read(argv[1], "eml", argv[3], &pbuff)) {
mlog(LV_ERR, "E-2071: imapfile_read %s/eml/%s failed", argv[1], argv[3]);
return MIDB_E_DISK_ERROR;
}

MAIL imail;
if (!imail.load_from_str(pbuff.get(), slurp_size))
if (!imail.load_from_str(pbuff.c_str(), pbuff.size()))
return MIDB_E_IMAIL_RETRIEVE;
Json::Value digest;
if (imail.make_digest(&mess_len, digest) <= 0)
return MIDB_E_IMAIL_DIGEST;
digest["file"] = "";
auto djson = json_to_str(digest);
auto ext_path = make_ext_path(argv[1], argv[3]);
wrapfd fd = open(ext_path.c_str(), O_CREAT | O_TRUNC | O_WRONLY, FMODE_PRIVATE);
if (fd.get() < 0) {
mlog(LV_ERR, "E-2073: Opening %s for writing failed: %s",
ext_path.c_str(), strerror(errno));
if (!exmdb_client::imapfile_write(argv[1], "ext", argv[3], djson)) {
mlog(LV_ERR, "E-2073: imapfile_write %s/ext/%s failed", argv[1], argv[3]);
return MIDB_E_DISK_ERROR;
}
if (HXio_fullwrite(fd.get(), djson.data(), djson.size()) < 0 ||
fd.close_wr() != 0)
mlog(LV_ERR, "E-2085: write %s: %s", ext_path.c_str(), strerror(errno));
auto pidb = me_get_idb(argv[1]);
if (pidb == nullptr)
return MIDB_E_HASHTABLE_FULL;
Expand All @@ -2222,7 +2159,7 @@ static int me_minst(int argc, char **argv, int sockd) try
auto pmsgctnt = oxcmail_import(charset, tmzone, &imail,
cu_alloc_bytes, cu_get_propids_create);
imail.clear();
pbuff.reset();
pbuff.clear();
if (pmsgctnt == nullptr)
return MIDB_E_OXCMAIL_IMPORT;
auto cl_msg = make_scope_exit([&]() { message_content_free(pmsgctnt); });
Expand Down
3 changes: 3 additions & 0 deletions include/gromox/exmdb_idef.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,3 +137,6 @@ EXMIDL(purge_datafiles, (const char *dir))
EXMIDL(autoreply_tsquery, (const char *dir, const char *peer, uint64_t window, IDLOUT uint64_t *tdiff))
EXMIDL(autoreply_tsupdate, (const char *dir, const char *peer))
EXMIDL(recalc_store_size, (const char *dir, uint32_t flags))
EXMIDL(imapfile_read, (const char *dir, const std::string &type, const std::string &mid, IDLOUT std::string *data))
EXMIDL(imapfile_write, (const char *dir, const std::string &type, const std::string &mid, const std::string &data))
EXMIDL(imapfile_delete, (const char *dir, const std::string &type, const std::string &mid))
Loading

0 comments on commit a5d2275

Please sign in to comment.