From 9fa8aa16a8039af645d97a3090c1558401dd7d1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=B8=D0=BB=D1=8F=D0=BD=20=D0=9F=D0=B0=D0=BB=D0=B0?= =?UTF-8?q?=D1=83=D0=B7=D0=BE=D0=B2?= Date: Fri, 25 Aug 2017 19:22:12 +0000 Subject: [PATCH 001/501] Good bye cyrusdb_sync It was used only by BDB. --- cunit/annotate.testc | 4 ++-- cunit/quota.testc | 2 +- imap/ctl_cyrusdb.c | 10 ---------- imap/ctl_mboxlist.c | 8 ++++---- imap/cyr_deny.c | 2 +- imap/mboxlist.c | 9 +++------ imap/mboxlist.h | 3 +-- imap/quota.c | 2 +- imap/quota.h | 3 +-- imap/quota_db.c | 7 ++----- imap/userdeny.h | 3 +-- imap/userdeny_db.c | 7 ++----- lib/cyrusdb.c | 11 ----------- lib/cyrusdb.h | 5 ----- lib/cyrusdb_flat.c | 1 - lib/cyrusdb_quotalegacy.c | 1 - lib/cyrusdb_skiplist.c | 1 - lib/cyrusdb_sql.c | 1 - lib/cyrusdb_twoskip.c | 1 - 19 files changed, 19 insertions(+), 62 deletions(-) diff --git a/cunit/annotate.testc b/cunit/annotate.testc index 09afcdcd32..f61648c283 100644 --- a/cunit/annotate.testc +++ b/cunit/annotate.testc @@ -1934,10 +1934,10 @@ static int set_up(void) isadmin = 0; auth_state = auth_newstate(userid); - quotadb_init(0); + quotadb_init(); quotadb_open(NULL); - mboxlist_init(0); + mboxlist_init(); mboxlist_open(NULL); memset(&mbentry, 0, sizeof(mbentry)); diff --git a/cunit/quota.testc b/cunit/quota.testc index 40626989cd..b680b8d570 100644 --- a/cunit/quota.testc +++ b/cunit/quota.testc @@ -1039,7 +1039,7 @@ static int set_up(void) cyrusdb_init(); config_quota_db = "skiplist"; - quotadb_init(0); + quotadb_init(); quotadb_open(NULL); return 0; diff --git a/imap/ctl_cyrusdb.c b/imap/ctl_cyrusdb.c index 966c350cc4..81e78370ee 100644 --- a/imap/ctl_cyrusdb.c +++ b/imap/ctl_cyrusdb.c @@ -332,17 +332,7 @@ int main(int argc, char *argv[]) break; case CHECKPOINT: - r2 = cyrusdb_sync(*dblist[i].configptr); - if (r2) { - syslog(LOG_ERR, "DBERROR: sync %s: %s", dirname, - cyrusdb_strerror(r2)); - fprintf(stderr, - "ctl_cyrusdb: unable to sync environment\n"); - } - /* ARCHIVE */ - r2 = 0; - if (!rotated) { /* rotate the backup directories -- ONE time only */ char *file; diff --git a/imap/ctl_mboxlist.c b/imap/ctl_mboxlist.c index b55bcc38f3..832ed7cd80 100644 --- a/imap/ctl_mboxlist.c +++ b/imap/ctl_mboxlist.c @@ -1002,7 +1002,7 @@ int main(int argc, char *argv[]) case CHECKPOINT: syslog(LOG_NOTICE, "checkpointing mboxlist"); - mboxlist_init(MBOXLIST_SYNC); + mboxlist_init(); mboxlist_open(NULL); mboxlist_close(); mboxlist_done(); @@ -1014,7 +1014,7 @@ int main(int argc, char *argv[]) GCC_FALLTHROUGH case DUMP: - mboxlist_init(0); + mboxlist_init(); mboxlist_open(mboxdb_fname); do_dump(op, partition, dopurge, dointermediary); @@ -1029,7 +1029,7 @@ int main(int argc, char *argv[]) break; case UNDUMP: - mboxlist_init(0); + mboxlist_init(); mboxlist_open(mboxdb_fname); do_undump(); @@ -1039,7 +1039,7 @@ int main(int argc, char *argv[]) break; case VERIFY: - mboxlist_init(0); + mboxlist_init(); mboxlist_open(mboxdb_fname); do_verify(); diff --git a/imap/cyr_deny.c b/imap/cyr_deny.c index 11a8483506..ce88d1754f 100644 --- a/imap/cyr_deny.c +++ b/imap/cyr_deny.c @@ -229,7 +229,7 @@ int main(int argc, char **argv) cyrus_init(alt_config, "cyr_deny", 0, 0); - denydb_init(0); + denydb_init(); r = denydb_open(/*create*/(mode == DENY)); if (r) { diff --git a/imap/mboxlist.c b/imap/mboxlist.c index 7623cc3e29..18012f8dbb 100644 --- a/imap/mboxlist.c +++ b/imap/mboxlist.c @@ -3834,7 +3834,7 @@ static void done_cb(void*rock __attribute__((unused))) static void init_internal() { if (!mboxlist_initialized) { - mboxlist_init(0); + mboxlist_init(); } if (!mboxlist_dbopen) { mboxlist_open(NULL); @@ -3842,11 +3842,8 @@ static void init_internal() } /* must be called after cyrus_init */ -EXPORTED void mboxlist_init(int myflags) +EXPORTED void mboxlist_init(void) { - if (myflags & MBOXLIST_SYNC) { - cyrusdb_sync(DB); - } cyrus_modules_add(done_cb, NULL); mboxlist_initialized = 1; } @@ -3865,7 +3862,7 @@ EXPORTED void mboxlist_open(const char *fname) fname = tofree; } - mboxlist_init(0); + mboxlist_init(); flags = CYRUSDB_CREATE; if (config_getswitch(IMAPOPT_IMPROVED_MBOXLIST_SORT)) { diff --git a/imap/mboxlist.h b/imap/mboxlist.h index d0a8bdc0ad..7f7b1b58c5 100644 --- a/imap/mboxlist.h +++ b/imap/mboxlist.h @@ -308,8 +308,7 @@ void mboxlist_open(const char *name); void mboxlist_close(void); /* initialize database structures */ -#define MBOXLIST_SYNC 0x02 -void mboxlist_init(int flags); +void mboxlist_init(void); /* done with database stuff */ void mboxlist_done(void); diff --git a/imap/quota.c b/imap/quota.c index ed6dbbd158..d1cdc89435 100644 --- a/imap/quota.c +++ b/imap/quota.c @@ -188,7 +188,7 @@ int main(int argc,char **argv) * Lock mailbox list to prevent mailbox creation/deletion * during work */ - mboxlist_init(0); + mboxlist_init(); mboxlist_open(NULL); quota_changelock(); diff --git a/imap/quota.h b/imap/quota.h index c8a07065e8..22992a9f0a 100644 --- a/imap/quota.h +++ b/imap/quota.h @@ -138,8 +138,7 @@ extern int quotadb_foreach(const char *prefix, size_t prefixlen, void quotadb_close(void); /* initialize database structures */ -#define QUOTADB_SYNC 0x02 -void quotadb_init(int flags); +void quotadb_init(void); /* done with database stuff */ void quotadb_done(void); diff --git a/imap/quota_db.c b/imap/quota_db.c index 7cfa170fe0..553fccbc72 100644 --- a/imap/quota_db.c +++ b/imap/quota_db.c @@ -656,7 +656,7 @@ static void done_cb(void*rock __attribute__((unused))) static void init_internal() { if (!quota_initialized) { - quotadb_init(0); + quotadb_init(); quota_initialized = 1; } if (!quota_dbopen) { @@ -665,11 +665,8 @@ static void init_internal() { } /* must be called after cyrus_init */ -EXPORTED void quotadb_init(int myflags) +EXPORTED void quotadb_init(void) { - if (myflags & QUOTADB_SYNC) { - cyrusdb_sync(QDB); - } cyrus_modules_add(done_cb, NULL); } diff --git a/imap/userdeny.h b/imap/userdeny.h index a1a207c305..2eea880358 100644 --- a/imap/userdeny.h +++ b/imap/userdeny.h @@ -63,8 +63,7 @@ int denydb_open(int create); void denydb_close(void); /* initialize database structures */ -#define DENYDB_SYNC 0x02 -void denydb_init(int flags); +void denydb_init(void); /* done with database stuff */ void denydb_done(void); diff --git a/imap/userdeny_db.c b/imap/userdeny_db.c index ca6f62754f..15c482d84b 100644 --- a/imap/userdeny_db.c +++ b/imap/userdeny_db.c @@ -348,7 +348,7 @@ static void done_cb(void*rock __attribute__((unused))) static void init_internal() { if (!denydb_initialized) { - denydb_init(0); + denydb_init(); cyrus_modules_add(done_cb, NULL); } if (!denydb) { @@ -357,11 +357,8 @@ static void init_internal() } /* must be called after cyrus_init */ -EXPORTED void denydb_init(int myflags) +EXPORTED void denydb_init(void) { - if (myflags & DENYDB_SYNC) { - cyrusdb_sync(DENYDB); - } denydb_initialized = 1; } diff --git a/lib/cyrusdb.c b/lib/cyrusdb.c index 59f6258a55..804a97050c 100644 --- a/lib/cyrusdb.c +++ b/lib/cyrusdb.c @@ -569,12 +569,6 @@ EXPORTED const char *cyrusdb_detect(const char *fname) return NULL; } -EXPORTED int cyrusdb_sync(const char *backend) -{ - struct cyrusdb_backend *db = cyrusdb_fromname(backend); - return db->sync(); -} - EXPORTED int cyrusdb_unlink(const char *backend, const char *fname, int flags) { struct cyrusdb_backend *db = cyrusdb_fromname(backend); @@ -621,11 +615,6 @@ HIDDEN int cyrusdb_generic_done(void) return 0; } -HIDDEN int cyrusdb_generic_sync(void) -{ - return 0; -} - HIDDEN int cyrusdb_generic_archive(const strarray_t *fnames, const char *dirname) { diff --git a/lib/cyrusdb.h b/lib/cyrusdb.h index e167abd450..07d0ef7ac4 100644 --- a/lib/cyrusdb.h +++ b/lib/cyrusdb.h @@ -106,9 +106,6 @@ struct cyrusdb_backend { * to reset state */ int (*done)(void); - /* checkpoints this database environment */ - int (*sync)(void); - /* archives this database environment, and specified databases * into the specified directory */ int (*archive)(const strarray_t *fnames, const char *dirname); @@ -301,7 +298,6 @@ extern int cyrusdb_compar(struct db *db, /* somewhat special case, because they don't take a DB */ -extern int cyrusdb_sync(const char *backend); extern cyrusdb_archiver *cyrusdb_getarchiver(const char *backend); extern int cyrusdb_canfetchnext(const char *backend); @@ -311,7 +307,6 @@ extern strarray_t *cyrusdb_backends(void); /* generic implementations */ int cyrusdb_generic_init(const char *dbdir, int myflags); int cyrusdb_generic_done(void); -int cyrusdb_generic_sync(void); int cyrusdb_generic_archive(const strarray_t *fnames, const char *dirname); int cyrusdb_generic_noarchive(const strarray_t *fnames, const char *dirname); int cyrusdb_generic_unlink(const char *fname, int flags); diff --git a/lib/cyrusdb_flat.c b/lib/cyrusdb_flat.c index c6320248e7..adffaea771 100644 --- a/lib/cyrusdb_flat.c +++ b/lib/cyrusdb_flat.c @@ -835,7 +835,6 @@ EXPORTED struct cyrusdb_backend cyrusdb_flat = &cyrusdb_generic_init, &cyrusdb_generic_done, - &cyrusdb_generic_sync, &cyrusdb_generic_archive, &cyrusdb_generic_unlink, diff --git a/lib/cyrusdb_quotalegacy.c b/lib/cyrusdb_quotalegacy.c index 6048821600..9fd3296720 100644 --- a/lib/cyrusdb_quotalegacy.c +++ b/lib/cyrusdb_quotalegacy.c @@ -879,7 +879,6 @@ HIDDEN struct cyrusdb_backend cyrusdb_quotalegacy = &cyrusdb_generic_init, &cyrusdb_generic_done, - &cyrusdb_generic_sync, &cyrusdb_generic_noarchive, &cyrusdb_generic_unlink, diff --git a/lib/cyrusdb_skiplist.c b/lib/cyrusdb_skiplist.c index 8768c81dbb..8d22d2cf68 100644 --- a/lib/cyrusdb_skiplist.c +++ b/lib/cyrusdb_skiplist.c @@ -2452,7 +2452,6 @@ EXPORTED struct cyrusdb_backend cyrusdb_skiplist = &myinit, &cyrusdb_generic_done, - &cyrusdb_generic_sync, &cyrusdb_generic_archive, &cyrusdb_generic_unlink, diff --git a/lib/cyrusdb_sql.c b/lib/cyrusdb_sql.c index 669811f0c3..0bf6e6a3f6 100644 --- a/lib/cyrusdb_sql.c +++ b/lib/cyrusdb_sql.c @@ -898,7 +898,6 @@ HIDDEN struct cyrusdb_backend cyrusdb_sql = &init, &done, - &cyrusdb_generic_sync, &cyrusdb_generic_noarchive, NULL, diff --git a/lib/cyrusdb_twoskip.c b/lib/cyrusdb_twoskip.c index 46c30dd946..2084fe4fe0 100644 --- a/lib/cyrusdb_twoskip.c +++ b/lib/cyrusdb_twoskip.c @@ -2404,7 +2404,6 @@ HIDDEN struct cyrusdb_backend cyrusdb_twoskip = &cyrusdb_generic_init, &cyrusdb_generic_done, - &cyrusdb_generic_sync, &cyrusdb_generic_archive, &cyrusdb_generic_unlink, From ce4692d7d57db7daf29b44082d203e15215f7c89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=B8=D0=BB=D1=8F=D0=BD=20=D0=9F=D0=B0=D0=BB=D0=B0?= =?UTF-8?q?=D1=83=D0=B7=D0=BE=D0=B2?= Date: Sat, 11 May 2019 21:36:30 +0000 Subject: [PATCH 002/501] =?UTF-8?q?imap/fetchnews.c:=20simplify:=20int=20r?= =?UTF-8?q?=3D0;=20if=20(r!=3D0)=20=E2=80=A6=20is=20a=20no-op?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- imap/fetchnews.c | 38 ++++++++++++++++---------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/imap/fetchnews.c b/imap/fetchnews.c index 8ed00ec478..bfb5df80f0 100644 --- a/imap/fetchnews.c +++ b/imap/fetchnews.c @@ -77,32 +77,26 @@ static int newsrc_dbopen = 0; static int newsrc_init(const char *fname, int myflags __attribute__((unused))) { char buf[1024]; - int r = 0; - - if (r != 0) - syslog(LOG_ERR, "DBERROR: init %s: %s", buf, - cyrusdb_strerror(r)); - else { - char *tofree = NULL; + int r; + char *tofree = NULL; - if (!fname) - fname = config_getstring(IMAPOPT_NEWSRC_DB_PATH); + if (!fname) + fname = config_getstring(IMAPOPT_NEWSRC_DB_PATH); - /* create db file name */ - if (!fname) { - tofree = strconcat(config_dir, FNAME_NEWSRCDB, (char *)NULL); - fname = tofree; - } + /* create db file name */ + if (!fname) { + tofree = strconcat(config_dir, FNAME_NEWSRCDB, (char *)NULL); + fname = tofree; + } - r = cyrusdb_open(DB, fname, CYRUSDB_CREATE, &newsrc_db); - if (r != 0) - syslog(LOG_ERR, "DBERROR: opening %s: %s", fname, - cyrusdb_strerror(r)); - else - newsrc_dbopen = 1; + r = cyrusdb_open(DB, fname, CYRUSDB_CREATE, &newsrc_db); + if (r != 0) + syslog(LOG_ERR, "DBERROR: opening %s: %s", fname, + cyrusdb_strerror(r)); + else + newsrc_dbopen = 1; - free(tofree); - } + free(tofree); return r; } From 5e25c5eba7f645ecddc9b4133fe293faf9739223 Mon Sep 17 00:00:00 2001 From: Michael Menge Date: Tue, 18 Feb 2020 11:19:06 +0100 Subject: [PATCH 003/501] Add option to allow special-use subfolders --- imap/imapd.c | 4 ++-- imap/mboxlist.c | 4 ++-- lib/imapoptions | 8 ++++++++ 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/imap/imapd.c b/imap/imapd.c index 0b97014041..614b84e6c0 100644 --- a/imap/imapd.c +++ b/imap/imapd.c @@ -6910,8 +6910,8 @@ static void cmd_create(char *tag, char *name, struct dlist *extargs, int localon } use = dlist_getchild(extargs, "USE"); if (use) { - /* only user mailboxes can have specialuse, and they must be user toplevel folders */ - if (!mbname_userid(mbname) || strarray_size(mbname_boxes(mbname)) != 1) { + /* only user mailboxes can have specialuse, and if allowspecialusesubfolders is not enabled they must be user toplevel folders */ + if (!mbname_userid(mbname) || (!config_getswitch(IMAPOPT_ALLOWSPECIALUSESUBFOLDER ) && strarray_size(mbname_boxes(mbname)) != 1)) { r = IMAP_MAILBOX_SPECIALUSE; goto err; } diff --git a/imap/mboxlist.c b/imap/mboxlist.c index 056dc70883..a86949307f 100644 --- a/imap/mboxlist.c +++ b/imap/mboxlist.c @@ -1974,8 +1974,8 @@ static int _rename_check_specialuse(const char *oldname, const char *newname) strarray_t *check = strarray_split(protect, NULL, STRARRAY_TRIM); strarray_t *uses = strarray_split(buf_cstring(&attrib), NULL, 0); if (strarray_intersect_case(uses, check)) { - /* then target must be a single-depth mailbox too */ - if (strarray_size(mbname_boxes(new)) != 1) + /* then if llowspecialusesubfolders is not enabled the target must be a single-depth mailbox too */ + if (!config_getswitch(IMAPOPT_ALLOWSPECIALUSESUBFOLDER) && strarray_size(mbname_boxes(new)) != 1) r = IMAP_MAILBOX_SPECIALUSE; /* and have a userid as well */ if (!mbname_userid(new)) diff --git a/lib/imapoptions b/lib/imapoptions index 27f82663d2..0170d71568 100644 --- a/lib/imapoptions +++ b/lib/imapoptions @@ -172,6 +172,14 @@ are listed with ``''. /* Defaults to enabled. If disabled, disallows the use of the SETACL command at all via IMAP. */ +{ "allowspecialusesubfolder", 0, SWITCH } +/* If enabled, allows special-use folders to be moved into subfolders +.PP + By default, cyrus does not allow a special-use folder to be moved or + renamed if the new folder is not a direct folder of the users INBOX. + This was implemented to prevent clients renaming Sent to Trash/Sent, + which would confused other clients. */ + { "allowusermoves", 0, SWITCH, "2.3.17" } /* Allow moving user accounts (with associated meta-data) via RENAME or XFER. From dfe0ecfe05a3718620530ce73f94438690b7834f Mon Sep 17 00:00:00 2001 From: Michael Menge Date: Mon, 9 Mar 2020 08:41:16 +0100 Subject: [PATCH 004/501] add version for allowspecialusesubfolder --- lib/imapoptions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/imapoptions b/lib/imapoptions index 0170d71568..344539b3ed 100644 --- a/lib/imapoptions +++ b/lib/imapoptions @@ -172,7 +172,7 @@ are listed with ``''. /* Defaults to enabled. If disabled, disallows the use of the SETACL command at all via IMAP. */ -{ "allowspecialusesubfolder", 0, SWITCH } +{ "allowspecialusesubfolder", 0, SWITCH, "3.0.14" } /* If enabled, allows special-use folders to be moved into subfolders .PP By default, cyrus does not allow a special-use folder to be moved or From 56edb8a16818f1bb704bb904873b60d69705548f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=B8=D0=BB=D1=8F=D0=BD=20=D0=9F=D0=B0=D0=BB=D0=B0?= =?UTF-8?q?=D1=83=D0=B7=D0=BE=D0=B2?= Date: Sat, 23 May 2020 16:53:32 +0000 Subject: [PATCH 005/501] RelNotes: 2854 (squatter -F) is fixed in 3.2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit https://lists.andrew.cmu.edu/pipermail/cyrus-devel/2019-June/004488.html “squatter -F increases index size” is likely fixed in 3.2. --- docsrc/imap/download/release-notes/3.2/x/3.2.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docsrc/imap/download/release-notes/3.2/x/3.2.0.rst b/docsrc/imap/download/release-notes/3.2/x/3.2.0.rst index 8ddde3a090..a30d92acb4 100644 --- a/docsrc/imap/download/release-notes/3.2/x/3.2.0.rst +++ b/docsrc/imap/download/release-notes/3.2/x/3.2.0.rst @@ -113,7 +113,7 @@ Security fixes Significant bugfixes ==================== -* Contains fix for :issue:`2839` +* Contains fix for :issue:`2839` and :issue:`2854`. .. _Xapian: https://xapian.org From 15245a53589edb0455e86935040ce2a564e36821 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=B8=D0=BB=D1=8F=D0=BD=20=D0=9F=D0=B0=D0=BB=D0=B0?= =?UTF-8?q?=D1=83=D0=B7=D0=BE=D0=B2?= Date: Fri, 19 Feb 2021 18:05:36 +0200 Subject: [PATCH 006/501] lib/cyrusdb_flat.c: simplify --- lib/cyrusdb_flat.c | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/lib/cyrusdb_flat.c b/lib/cyrusdb_flat.c index c5945c1657..700347db0b 100644 --- a/lib/cyrusdb_flat.c +++ b/lib/cyrusdb_flat.c @@ -236,7 +236,6 @@ static struct txn *new_txn(void) static int starttxn_or_refetch(struct dbengine *db, struct txn **mytid) { - int r = 0; struct stat sbuf; assert(db); @@ -246,8 +245,7 @@ static int starttxn_or_refetch(struct dbengine *db, struct txn **mytid) /* start txn; grab lock */ - r = lock_reopen(db->fd, db->fname, &sbuf, &lockfailaction); - if (r < 0) { + if (lock_reopen(db->fd, db->fname, &sbuf, &lockfailaction) < 0) { syslog(LOG_ERR, "IOERROR: %s %s: %m", lockfailaction, db->fname); return CYRUSDB_IOERROR; } @@ -424,22 +422,6 @@ static int myfetch(struct dbengine *db, return r; } -static int fetch(struct dbengine *mydb, - const char *key, size_t keylen, - const char **data, size_t *datalen, - struct txn **mytid) -{ - return myfetch(mydb, key, keylen, data, datalen, mytid); -} - -static int fetchlock(struct dbengine *db, - const char *key, size_t keylen, - const char **data, size_t *datalen, - struct txn **mytid) -{ - return myfetch(db, key, keylen, data, datalen, mytid); -} - static int getentry(struct dbengine *db, const char *p, struct buf *keybuf, const char **dataendp) { @@ -844,8 +826,8 @@ EXPORTED struct cyrusdb_backend cyrusdb_flat = &myopen, &myclose, - &fetch, - &fetchlock, + &myfetch, + &myfetch, NULL, &foreach, From 1e6cd859b06ae8369ab8bbc890da2cf973594da8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=B8=D0=BB=D1=8F=D0=BD=20=D0=9F=D0=B0=D0=BB=D0=B0?= =?UTF-8?q?=D1=83=D0=B7=D0=BE=D0=B2?= Date: Sat, 27 Feb 2021 14:01:55 +0200 Subject: [PATCH 007/501] Update 3.4.0-beta release notes --- docsrc/imap/download/release-notes/3.4/x/3.4.0-beta1.rst | 4 ++++ docsrc/imap/download/release-notes/3.4/x/3.4.0-beta2.rst | 4 ++++ docsrc/imap/download/release-notes/3.4/x/3.4.0-beta3.rst | 4 ++++ docsrc/imap/reference/architecture.rst | 2 +- 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docsrc/imap/download/release-notes/3.4/x/3.4.0-beta1.rst b/docsrc/imap/download/release-notes/3.4/x/3.4.0-beta1.rst index 545d3db0a8..5ec506d821 100644 --- a/docsrc/imap/download/release-notes/3.4/x/3.4.0-beta1.rst +++ b/docsrc/imap/download/release-notes/3.4/x/3.4.0-beta1.rst @@ -15,6 +15,9 @@ Major changes since the 3.2 series ================================== * DAV improvements + + * Allow clients to set schedule-default-calendar-URL + * Improved performance for users with large folders * LITERAL- maximum size is now honoured (:rfc:`7888`) * Support for the ESORT (but not CONTEXT) extension from :rfc:`5267` @@ -73,6 +76,7 @@ Major changes since the 3.2 series users/domains. * The HTTP Admin module's Currently Running Services feature now works on the major BSDs (thanks Felix J. Ogris) +* Prefer SPNEGO over BASIC WWW-Auth in Firefox/Thunderbird :issue:`2882`. Updates to default configuration diff --git a/docsrc/imap/download/release-notes/3.4/x/3.4.0-beta2.rst b/docsrc/imap/download/release-notes/3.4/x/3.4.0-beta2.rst index 8d18db5ee2..2fcdd0054d 100644 --- a/docsrc/imap/download/release-notes/3.4/x/3.4.0-beta2.rst +++ b/docsrc/imap/download/release-notes/3.4/x/3.4.0-beta2.rst @@ -15,6 +15,9 @@ Major changes since the 3.2 series ================================== * DAV improvements + + * Allow clients to set schedule-default-calendar-URL + * Improved performance for users with large folders * LITERAL- maximum size is now honoured (:rfc:`7888`) * Support for the ESORT (but not CONTEXT) extension from :rfc:`5267` @@ -73,6 +76,7 @@ Major changes since the 3.2 series users/domains. * The HTTP Admin module's Currently Running Services feature now works on the major BSDs (thanks Felix J. Ogris) +* Prefer SPNEGO over BASIC WWW-Auth in Firefox/Thunderbird :issue:`2882`. Updates to default configuration diff --git a/docsrc/imap/download/release-notes/3.4/x/3.4.0-beta3.rst b/docsrc/imap/download/release-notes/3.4/x/3.4.0-beta3.rst index 363af947fe..048aef7f88 100644 --- a/docsrc/imap/download/release-notes/3.4/x/3.4.0-beta3.rst +++ b/docsrc/imap/download/release-notes/3.4/x/3.4.0-beta3.rst @@ -15,6 +15,9 @@ Major changes since the 3.2 series ================================== * DAV improvements + + * Allow clients to set schedule-default-calendar-URL + * Improved performance for users with large folders * LITERAL- maximum size is now honoured (:rfc:`7888`) * Support for the ESORT (but not CONTEXT) extension from :rfc:`5267` @@ -78,6 +81,7 @@ Major changes since the 3.2 series (Squat backend only) * :cyrusman:`squatter(8)` now supports long options * Improvements to search query normalisation performance +* Prefer SPNEGO over BASIC WWW-Auth in Firefox/Thunderbird :issue:`2882`. Updates to default configuration diff --git a/docsrc/imap/reference/architecture.rst b/docsrc/imap/reference/architecture.rst index 13ed75eaf3..a47fff8ebc 100644 --- a/docsrc/imap/reference/architecture.rst +++ b/docsrc/imap/reference/architecture.rst @@ -147,7 +147,7 @@ know the imap credentials on the replica to allow it to send details to the replica. It is the port configuration on the replica to know where to listen for change updates. -There's two standard channel configurations: +There are two standard channel configurations: 1. Single master keeping all replicas up to date. From e2f5039f0c6ad81d52a974902fc30abdeeb50f67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=B8=D0=BB=D1=8F=D0=BD=20=D0=9F=D0=B0=D0=BB=D0=B0?= =?UTF-8?q?=D1=83=D0=B7=D0=BE=D0=B2?= Date: Sat, 27 Feb 2021 19:35:56 +0200 Subject: [PATCH 008/501] http_ca(l,rd)dav: list collections: html escape the displayname Otherwise >, <, and & in the displayname of calendars/address books are not converted to > < or & and the generated HTML might not work. When the DAV:displayname contains ' or ", generating valid HTML for is very hard. Therefore the rendered HTML in list_calendars() includes an id attribute to the with the displayname and passes around the id. The callee uses then window.confirm(document.getElementById(id).innerText) which can show ' and " to the mortals. --- imap/http_caldav.c | 16 ++++++++++------ imap/http_caldav.js | 8 ++++---- imap/http_carddav.c | 12 ++++++++---- imap/http_carddav.js | Bin 5709 -> 5740 bytes lib/charset.c | 2 +- 5 files changed, 23 insertions(+), 15 deletions(-) diff --git a/imap/http_caldav.c b/imap/http_caldav.c index b795761348..68624951c3 100644 --- a/imap/http_caldav.c +++ b/imap/http_caldav.c @@ -2491,10 +2491,12 @@ static int list_calendars(struct transaction_t *txn) /* Sort calendars by displayname */ qsort(lrock.cal, lrock.len, sizeof(struct cal_info), &cal_compare); + charset_t utf8 = charset_lookupname("utf-8"); /* Add available calendars with action items */ for (i = 0; i < lrock.len; i++) { struct cal_info *cal = &lrock.cal[i]; + char *displayname = charset_convert(cal->displayname, utf8, CHARSET_KEEPCASE | CHARSET_ESCAPEHTML); /* Send a body chunk once in a while */ if (buf_len(body) > PROT_BUFSIZE) { @@ -2504,17 +2506,18 @@ static int list_calendars(struct transaction_t *txn) /* Calendar name */ buf_printf_markup(body, level++, ""); - buf_printf_markup(body, level, "%s%s%s", + buf_printf_markup(body, level, "%s%s%s", i, (cal->flags & CAL_IS_DEFAULT) ? "" : "", - cal->displayname, + displayname, (cal->flags & CAL_IS_DEFAULT) ? "" : ""); + free(displayname); /* Supported components list */ buf_printf_markup(body, level++, ""); buf_printf_markup(body, level++, "", + " onclick=\"deleteCalendar('%s%s', '%i')\">", !(cal->flags & CAL_CAN_DELETE) ? " disabled" : "", - base_path, cal->shortname, cal->displayname); + base_path, cal->shortname, i); /* Public (shared) checkbox */ buf_printf_markup(body, level, @@ -2560,6 +2563,7 @@ static int list_calendars(struct transaction_t *txn) buf_printf_markup(body, --level, ""); } + charset_free(&utf8); free(lrock.cal); /* Finish list */ diff --git a/imap/http_caldav.js b/imap/http_caldav.js index b5fc1d1fb5..6ed2a74a2d 100644 --- a/imap/http_caldav.js +++ b/imap/http_caldav.js @@ -197,10 +197,10 @@ function transpCalendar(url, transp) { // Adjust supported components on a calendar collection -function compsetCalendar(url, name, comps) { +function compsetCalendar(url, id, comps) { if (!window.confirm('Are you sure you want to change' + ' component types on calendar \"' + - name + '\"?')) { + document.getElementById(id).innerText + '\"?')) { // Reset selected options for (var i = 0; i < comps.length; i++) { @@ -242,9 +242,9 @@ function compsetCalendar(url, name, comps) { // Delete a calendar collection -function deleteCalendar(url, name) { +function deleteCalendar(url, id) { if (window.confirm('Are you sure you want to delete calendar \"' + - name + '\"?')) { + document.getElementById(id).innerText + '\"?')) { // Send DELETE request var req = new XMLHttpRequest(); req.open('DELETE', url, false); diff --git a/imap/http_carddav.c b/imap/http_carddav.c index 0b7e6c7a59..05cbb86f45 100644 --- a/imap/http_carddav.c +++ b/imap/http_carddav.c @@ -1067,10 +1067,12 @@ static int list_addressbooks(struct transaction_t *txn) /* Sort addressbooks by displayname */ qsort(lrock.addr, lrock.len, sizeof(struct addr_info), &addr_compare); + charset_t utf8 = charset_lookupname("utf-8"); /* Add available addressbooks with action items */ for (i = 0; i < lrock.len; i++) { struct addr_info *addr = &lrock.addr[i]; + char *displayname = charset_convert(addr->displayname, utf8, CHARSET_KEEPCASE | CHARSET_ESCAPEHTML); /* Send a body chunk once in a while */ if (buf_len(body) > PROT_BUFSIZE) { @@ -1080,10 +1082,11 @@ static int list_addressbooks(struct transaction_t *txn) /* Addressbook name */ buf_printf_markup(body, level++, ""); - buf_printf_markup(body, level, "%s%s%s", + buf_printf_markup(body, level, "%s%s%s", i, (addr->flags & ADDR_IS_DEFAULT) ? "" : "", - addr->displayname, + displayname, (addr->flags & ADDR_IS_DEFAULT) ? "" : ""); + free(displayname); /* Download link */ buf_printf_markup(body, level, "Download", @@ -1092,9 +1095,9 @@ static int list_addressbooks(struct transaction_t *txn) /* Delete button */ buf_printf_markup(body, level, "", + " onclick=\"deleteAddressbook('%s%s', '%i')\">", !(addr->flags & ADDR_CAN_DELETE) ? " disabled" : "", - base_path, addr->shortname, addr->displayname); + base_path, addr->shortname, i); /* Public (shared) checkbox */ buf_printf_markup(body, level, @@ -1108,6 +1111,7 @@ static int list_addressbooks(struct transaction_t *txn) buf_printf_markup(body, --level, ""); } + charset_free(&utf8); free(lrock.addr); /* Finish list */ diff --git a/imap/http_carddav.js b/imap/http_carddav.js index 5270dbd509aeded08cc0f65c5cf8588aba1aab1c..0f8cd3354c7cb970d2ffbc482a2e6686dbd12264 100644 GIT binary patch delta 57 zcmX@B^G0Vwh6qz;%H$l8RMnLHu delta 26 ecmaE(b5>_Vh6qbuVs7f>ERj?&WAhi03Pu2rj0!RU diff --git a/lib/charset.c b/lib/charset.c index 6c974f6ce0..d4c5d11b0a 100644 --- a/lib/charset.c +++ b/lib/charset.c @@ -968,7 +968,7 @@ static void table2uni(struct convert_rock *rock, uint32_t c) struct charmap *map; if (c == U_REPLACEMENT) { - convert_putc(rock->next, c); + convert_putc(rock->next, U_REPLACEMENT); return; } From 6acbd600f12b32a33d08341d25dcfb504c446506 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=B8=D0=BB=D1=8F=D0=BD=20=D0=9F=D0=B0=D0=BB=D0=B0?= =?UTF-8?q?=D1=83=D0=B7=D0=BE=D0=B2?= Date: Wed, 7 Apr 2021 13:48:48 +0300 Subject: [PATCH 009/501] master:reap_child() survive SIGINT When the master process receives SIGINT, it shall recycle all children and then exit. If the signal is recieved while waitpid() in reap_child() is executed, waitpid() returns 0, the child exit status is not consumed and the centry is not updated. Later on, the janitor utilizes 100% the CPU by iterating over its slots, to see if the process, whose centry was not updated, is in the meantime terminated. The child is in fact terminated, but due the interruption of waitpid() the master process never updates its internal status for the child. --- master/master.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/master/master.c b/master/master.c index be526488b2..7bb0a9755b 100644 --- a/master/master.c +++ b/master/master.c @@ -1067,7 +1067,14 @@ static void reap_child(void) struct service *s; int failed; - while ((pid = waitpid((pid_t) -1, &status, WNOHANG)) > 0) { + while ((pid = waitpid((pid_t) -1, &status, WNOHANG)) > -1) { + if (pid == 0) { + if (errno == EINTR) { + errno = 0; + continue; + } + break; + } /* account for the child */ c = centry_find(pid); From dcf0c95ec640ec49e2dfc7cc0a53a60d618fd883 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=B8=D0=BB=D1=8F=D0=BD=20=D0=9F=D0=B0=D0=BB=D0=B0?= =?UTF-8?q?=D1=83=D0=B7=D0=BE=D0=B2?= Date: Tue, 26 Oct 2021 07:49:21 +0300 Subject: [PATCH 010/501] jmap_sieve.c,sieve/test.c:getheader() unnecessary assignment --- imap/jmap_sieve.c | 2 -- sieve/test.c | 2 -- sieve/test_mailbox.c | 2 -- 3 files changed, 6 deletions(-) diff --git a/imap/jmap_sieve.c b/imap/jmap_sieve.c index 3129d75fe8..0276b5e1ae 100644 --- a/imap/jmap_sieve.c +++ b/imap/jmap_sieve.c @@ -1421,8 +1421,6 @@ static int getheader(void *v, const char *phead, const char ***body) { message_data_t *m = (message_data_t *) v; - *body = NULL; - if (!m->cache_full) fill_cache(m); *body = spool_getheader(m->cache, phead); diff --git a/sieve/test.c b/sieve/test.c index 8cf5e3afea..9fd5572da9 100644 --- a/sieve/test.c +++ b/sieve/test.c @@ -142,8 +142,6 @@ static int getheader(void *v, const char *phead, const char ***body) { message_data_t *m = (message_data_t *) v; - *body = NULL; - if (!m->cache_full) { fill_cache(m); } diff --git a/sieve/test_mailbox.c b/sieve/test_mailbox.c index e75499dac6..e33156c151 100644 --- a/sieve/test_mailbox.c +++ b/sieve/test_mailbox.c @@ -137,8 +137,6 @@ static int getheader(void *v, const char *phead, const char ***body) { message_data_t *m = (message_data_t *) v; - *body = NULL; - if (!m->cache_full) { fill_cache(m); } From 57ef06eecede7047ae8969a4bd687a1581e79078 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=B8=D0=BB=D1=8F=D0=BD=20=D0=9F=D0=B0=D0=BB=D0=B0?= =?UTF-8?q?=D1=83=D0=B7=D0=BE=D0=B2?= Date: Sun, 10 Mar 2019 09:30:56 +0000 Subject: [PATCH 011/501] http_caldav_sched.c:imip_send_sendmail: print less redundant information in the text/plain part about attendees When the email address (property value) and name (CN= property parameter) of an attendee/organizer coincide, print only the email address. --- imap/http_caldav_sched.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/imap/http_caldav_sched.c b/imap/http_caldav_sched.c index 8418ae9840..269e9e514f 100644 --- a/imap/http_caldav_sched.c +++ b/imap/http_caldav_sched.c @@ -387,8 +387,12 @@ static int imip_send_sendmail(const char *userid, icalcomponent *ical, const cha buf_appendcstr(&msgbuf, "Content-Type: text/plain; charset=utf-8\r\n"); buf_appendcstr(&msgbuf, "Content-Disposition: inline\r\n"); - buf_printf(&plainbuf, "You have received %s from %s <%s>\r\n\r\n", msg_type, - originator->name ? originator->name : "", originator->addr); + if (originator->name && strcasecmp(originator->name, originator->addr)) + buf_printf(&plainbuf, "You have received %s from %s <%s>\r\n\r\n", + msg_type, originator->name, originator->addr); + else + buf_printf(&plainbuf, "You have received %s from %s\r\n\r\n", msg_type, + originator->addr); if (summary) { buf_setcstr(&tmpbuf, summary); buf_replace_all(&tmpbuf, "\n", "\r\n" TEXT_INDENT); @@ -408,8 +412,10 @@ static int imip_send_sendmail(const char *userid, icalcomponent *ical, const cha if (status) buf_printf(&plainbuf, "Status : %s\r\n", status); for (cp = "Attendees : ", recip=recipients; recip; recip=recip->next) { - buf_printf(&plainbuf, "%s* %s <%s>", - cp, recip->name ? recip->name : "", recip->addr); + if (recip->name && strcasecmp(recip->name, recip->addr)) + buf_printf(&plainbuf, "%s* %s <%s>", cp, recip->name, recip->addr); + else + buf_printf(&plainbuf, "%s* %s", cp, recip->addr); if (recip->role) buf_printf(&plainbuf, "\t(%s)", recip->role); buf_appendcstr(&plainbuf, "\r\n"); From cd18cbb53e1e1ffb338bea641cc2e43b262dae12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=B8=D0=BB=D1=8F=D0=BD=20=D0=9F=D0=B0=D0=BB=D0=B0?= =?UTF-8?q?=D1=83=D0=B7=D0=BE=D0=B2?= Date: Sat, 21 Aug 2021 08:49:27 +0300 Subject: [PATCH 012/501] imap/annotate.[ch]: Remove unused dupentryatt() and dupattvalues() --- imap/annotate.c | 23 ----------------------- imap/annotate.h | 4 ---- 2 files changed, 27 deletions(-) diff --git a/imap/annotate.c b/imap/annotate.c index 204e7bd801..1501489c0f 100644 --- a/imap/annotate.c +++ b/imap/annotate.c @@ -328,16 +328,6 @@ EXPORTED void appendattvalue(struct attvaluelist **l, buf_copy(&(*tail)->value, value); } -/* - * Duplicate the attvaluelist @src to @dst. - */ -void dupattvalues(struct attvaluelist **dst, - const struct attvaluelist *src) -{ - for ( ; src ; src = src->next) - appendattvalue(dst, src->attrib, &src->value); -} - /* * Free the attvaluelist 'l' */ @@ -457,19 +447,6 @@ EXPORTED void clearentryatt(struct entryattlist **l, const char *entry, } } -/* - * Duplicate the entryattlist @src to @dst. - */ -void dupentryatt(struct entryattlist **dst, - const struct entryattlist *src) -{ - for ( ; src ; src = src->next) { - struct attvaluelist *attvalues = NULL; - dupattvalues(&attvalues, src->attvalues); - appendentryatt(dst, src->entry, attvalues); - } -} - /* * Count the storage used by entryattlist 'l' */ diff --git a/imap/annotate.h b/imap/annotate.h index 8821ca3f3f..2fc0c585c7 100644 --- a/imap/annotate.h +++ b/imap/annotate.h @@ -112,8 +112,6 @@ void freestrlist(struct strlist *l); /* Attribute Management (also used by ID) */ void appendattvalue(struct attvaluelist **l, const char *attrib, const struct buf *value); -void dupattvalues(struct attvaluelist **dst, - const struct attvaluelist *src); void freeattvalues(struct attvaluelist *l); /* Entry Management */ @@ -123,8 +121,6 @@ void setentryatt(struct entryattlist **l, const char *entry, const char *attrib, const struct buf *value); void clearentryatt(struct entryattlist **l, const char *entry, const char *attrib); -void dupentryatt(struct entryattlist **l, - const struct entryattlist *); size_t sizeentryatts(const struct entryattlist *); char *dumpentryatt(const struct entryattlist *l); void freeentryatts(struct entryattlist *l); From 2d2925600f219ac921c9371c915e5d61b31f5053 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=B8=D0=BB=D1=8F=D0=BD=20=D0=9F=D0=B0=D0=BB=D0=B0?= =?UTF-8?q?=D1=83=D0=B7=D0=BE=D0=B2?= Date: Sat, 30 Oct 2021 23:19:21 +0300 Subject: [PATCH 013/501] mupdate-client.c: remove unused service_name variable --- imap/mupdate-client.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/imap/mupdate-client.c b/imap/mupdate-client.c index 0934a9db16..7aa8e35f89 100644 --- a/imap/mupdate-client.c +++ b/imap/mupdate-client.c @@ -72,8 +72,6 @@ #include "xstrlcpy.h" #include "xstrlcat.h" -const char service_name[] = "mupdate"; - static struct protocol_t mupdate_protocol = { "mupdate", "mupdate", TYPE_STD, { { { 1, "* OK" }, From 90aca51037aac7cf5f8f14f15b7a5d1c51b4a137 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=B8=D0=BB=D1=8F=D0=BD=20=D0=9F=D0=B0=D0=BB=D0=B0?= =?UTF-8?q?=D1=83=D0=B7=D0=BE=D0=B2?= Date: Tue, 16 Nov 2021 16:45:07 +0200 Subject: [PATCH 014/501] imapoptions:lock_debugtime: not implemented for flock locks --- lib/imapoptions | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/imapoptions b/lib/imapoptions index 65dae9c967..97f708fe72 100644 --- a/lib/imapoptions +++ b/lib/imapoptions @@ -1530,7 +1530,8 @@ Blank lines and lines beginning with ``#'' are ignored. { "lock_debugtime", NULL, STRING, "3.1.4" } /* A floating point number of seconds. If set, time how long we wait for any lock, and syslog the filename and time if it's longer than this - value. The default of NULL means not to time locks. */ + value. The default of NULL means not to time locks. This is + implemented for fcntl(2) locks and has no effect on flock(2) locks. */ # xxx how does this tie into virtual domains? { "loginrealms", "", STRING, "2.3.17" } From 19f5156eb2b009699a2a5bcd5360c8e542afdd77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=B8=D0=BB=D1=8F=D0=BD=20=D0=9F=D0=B0=D0=BB=D0=B0?= =?UTF-8?q?=D1=83=D0=B7=D0=BE=D0=B2?= Date: Sat, 30 Apr 2022 20:36:53 +0300 Subject: [PATCH 015/501] httpd.c: return value of httpdate_gen() always ignored --- imap/httpd.c | 4 +--- imap/httpd.h | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/imap/httpd.c b/imap/httpd.c index 8a15861b51..a569dd8858 100644 --- a/imap/httpd.c +++ b/imap/httpd.c @@ -2559,7 +2559,7 @@ void parse_query_params(struct transaction_t *txn, const char *query) /* Create HTTP-date ('buf' must be at least 30 characters) */ -EXPORTED char *httpdate_gen(char *buf, size_t len, time_t t) +EXPORTED void httpdate_gen(char *buf, size_t len, time_t t) { struct tm *tm = gmtime(&t); @@ -2567,8 +2567,6 @@ EXPORTED char *httpdate_gen(char *buf, size_t len, time_t t) wday[tm->tm_wday], tm->tm_mday, monthname[tm->tm_mon], tm->tm_year + 1900, tm->tm_hour, tm->tm_min, tm->tm_sec); - - return buf; } diff --git a/imap/httpd.h b/imap/httpd.h index 14df43ec81..a65dde289e 100644 --- a/imap/httpd.h +++ b/imap/httpd.h @@ -592,7 +592,7 @@ extern void free_accept(struct accept *a); extern void parse_query_params(struct transaction_t *txn, const char *query); extern time_t calc_compile_time(const char *time, const char *date); extern const char *http_statusline(unsigned ver, long code); -extern char *httpdate_gen(char *buf, size_t len, time_t t); +void httpdate_gen(char *buf, size_t len, time_t t) __attribute__((nonnull)); extern void simple_hdr(struct transaction_t *txn, const char *name, const char *value, ...) __attribute__((format(printf, 3, 4))); From effc57d9dfc02d70ce0504c5b98c08908e9f0ef2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=B8=D0=BB=D1=8F=D0=BD=20=D0=9F=D0=B0=D0=BB=D0=B0?= =?UTF-8?q?=D1=83=D0=B7=D0=BE=D0=B2?= Date: Wed, 18 May 2022 10:14:26 +0300 Subject: [PATCH 016/501] Remove useless assignments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit or, as the Clang Static Analyzer calls them, “Dead initializations”. --- imap/caldav_alarm.c | 2 +- imap/http_caldav.c | 2 +- imap/httpd.c | 3 +-- imap/ical_support.c | 4 ++-- imap/jmap_mail.c | 6 ++---- imap/message.c | 6 ++---- imap/search_squat.c | 3 +-- imap/sievedir.c | 3 ++- 8 files changed, 12 insertions(+), 17 deletions(-) diff --git a/imap/caldav_alarm.c b/imap/caldav_alarm.c index c6175a40a1..0044e23142 100644 --- a/imap/caldav_alarm.c +++ b/imap/caldav_alarm.c @@ -534,7 +534,7 @@ static int process_alarm_cb(icalcomponent *comp, struct icaltriggertype trigger = icalvalue_get_trigger(val); /* XXX validate trigger */ - icaltimetype alarmtime = icaltime_null_time(); + icaltimetype alarmtime; unsigned is_duration = (icalvalue_isa(val) == ICAL_DURATION_VALUE); if (is_duration) { icalparameter *param = diff --git a/imap/http_caldav.c b/imap/http_caldav.c index 85a039f8ea..57869c3212 100644 --- a/imap/http_caldav.c +++ b/imap/http_caldav.c @@ -1311,7 +1311,7 @@ static void update_refcount(const char *mid, short *op, static int open_attachments(const char *userid, struct mailbox **attachments, struct webdav_db **webdavdb) { - char *mailboxname = mailboxname = caldav_mboxname(userid, MANAGED_ATTACH); + char *mailboxname = caldav_mboxname(userid, MANAGED_ATTACH); int r, ret = 0; /* Open attachments collection for writing */ diff --git a/imap/httpd.c b/imap/httpd.c index 7fcfef79f9..a5a8de5130 100644 --- a/imap/httpd.c +++ b/imap/httpd.c @@ -3300,9 +3300,8 @@ EXPORTED void response_header(long code, struct transaction_t *txn) simple_hdr(txn, "Content-Type", "%s", resp_body->type); if (resp_body->dispo.fname) { /* Construct Content-Disposition header */ - const unsigned char *p = (const unsigned char *)resp_body->dispo.fname; char *encfname = NULL; - for (p = (unsigned char *)resp_body->dispo.fname; p && *p; p++) { + for (const unsigned char *p = (unsigned char *)resp_body->dispo.fname; p && *p; p++) { if (*p >= 0x80) { encfname = charset_encode_mimexvalue(resp_body->dispo.fname, NULL); break; diff --git a/imap/ical_support.c b/imap/ical_support.c index 4b763324d6..2c72a8a9e8 100644 --- a/imap/ical_support.c +++ b/imap/ical_support.c @@ -553,9 +553,9 @@ EXPORTED int icalcomponent_myforeach(icalcomponent *ical, ICAL_RDATE_PROPERTY)) { struct icaldatetimeperiodtype rdate = icalproperty_get_datetimeperiod(prop); - icaltimetype mystart = rdate.time; - icaltimetype myend = rdate.time; + icaltimetype mystart, myend; if (icalperiodtype_is_null_period(rdate.period)) { + mystart = rdate.time; myend = icaltime_add(mystart, event_length); } else { diff --git a/imap/jmap_mail.c b/imap/jmap_mail.c index 6e59405c27..f602b59d8d 100644 --- a/imap/jmap_mail.c +++ b/imap/jmap_mail.c @@ -2885,8 +2885,7 @@ static void emailsearch_init(struct emailsearch *search, search->sort = _email_buildsort(jsort, &search->sort_savedate); } else { - struct sortcrit *sort = search->sort; - sort = xzmalloc(3 * sizeof(struct sortcrit)); + struct sortcrit *sort = xzmalloc(3 * sizeof(struct sortcrit)); sort[0].key = SORT_ARRIVAL; sort[0].flags |= SORT_REVERSE; sort[1].key = SORT_GUID; @@ -7435,7 +7434,6 @@ static json_t *_email_get_bodypart(jmap_req_t *req, /* FastMail extension properties */ if (jmap_wantprop(bodyprops, "imageSize")) { - json_t *imagesize = json_null(); if (msg->mr && msg->imagesize_by_partid == NULL) { /* This is the first attempt to read the vendor annotation. * Load the annotation value, if any, for top-level messages. @@ -7446,7 +7444,7 @@ static json_t *_email_get_bodypart(jmap_req_t *req, if (!msg->imagesize_by_partid) msg->imagesize_by_partid = json_null(); } - imagesize = json_object_get(msg->imagesize_by_partid, part->part_id); + json_t *imagesize = json_object_get(msg->imagesize_by_partid, part->part_id); json_object_set(jbodypart, "imageSize", imagesize ? imagesize : json_null()); } if (jmap_wantprop(bodyprops, "isDeleted")) { diff --git a/imap/message.c b/imap/message.c index 37146c2beb..fad3d60f4f 100644 --- a/imap/message.c +++ b/imap/message.c @@ -1304,8 +1304,7 @@ EXPORTED void message_parse_type(const char *hdr, char **typep, char **subtypep, if (!attrlen || attr[attrlen-1] == '*') continue; /* Check if the parameter value has non-ASCII characters */ int has_highbit = 0; - const char *val = param->value; - for (val = param->value; *val && !has_highbit; val++) { + for (const char *val = param->value; *val && !has_highbit; val++) { has_highbit = *val & 0x80; } if (!has_highbit) continue; @@ -3282,13 +3281,12 @@ static void message_read_binarybody(struct body *body, const char **sect, uint32_t cache_version) { bit32 n, i; - const char *p = *sect; struct body *subpart; size_t len; uint32_t cte; n = CACHE_ITEM_BIT32(*sect); - p = *sect += CACHE_ITEM_SIZE_SKIP; + const char *p = *sect += CACHE_ITEM_SIZE_SKIP; if (!n) return; if (!strcmp(body->type, "MESSAGE") && !strcmp(body->subtype, "RFC822") && diff --git a/imap/search_squat.c b/imap/search_squat.c index 5eab917bb1..b1fe2226f7 100644 --- a/imap/search_squat.c +++ b/imap/search_squat.c @@ -397,10 +397,9 @@ static search_builder_t *begin_search(struct mailbox *mailbox, int opts) static int add_unindexed(SquatBuilderData *bb) { - struct opstack *top = opstack_top(bb); int r = 0; + struct opstack *top = opstack_push(bb, /*doesn't matter*/0); - top = opstack_push(bb, /*doesn't matter*/0); bv_setall(&top->msg_vector); bv_clear(&top->msg_vector, 0); /* UID 0 is not valid */ bb->part_types = "tfcbsmh"; diff --git a/imap/sievedir.c b/imap/sievedir.c index ada4faa55b..d2661e7c18 100644 --- a/imap/sievedir.c +++ b/imap/sievedir.c @@ -303,7 +303,8 @@ EXPORTED int sievedir_put_script(const char *sievedir, const char *name, /* parse the script */ sieve_script_t *s = NULL; char *myerrors = NULL; - int r = sieve_script_parse_string(NULL, content, &myerrors, &s); + int r; + sieve_script_parse_string(NULL, content, &myerrors, &s); if (errors) *errors = myerrors; else free(myerrors); From b0ac0fe938ac1a749aef2a2aadf47d8e4e4892fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=B8=D0=BB=D1=8F=D0=BD=20=D0=9F=D0=B0=D0=BB=D0=B0?= =?UTF-8?q?=D1=83=D0=B7=D0=BE=D0=B2?= Date: Thu, 27 May 2021 11:53:46 +0300 Subject: [PATCH 017/501] Typos --- imap/carddav_db.c | 2 +- imap/http_proxy.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/imap/carddav_db.c b/imap/carddav_db.c index 2aac4c73d7..e996a2c975 100644 --- a/imap/carddav_db.c +++ b/imap/carddav_db.c @@ -1290,7 +1290,7 @@ EXPORTED int carddav_store(struct mailbox *mailbox, struct vparse_card *vcard, goto done; } - /* Commit the append to the calendar mailbox */ + /* Commit the append to the addressbook mailbox */ r = append_commit(&as); if (r) { syslog(LOG_ERR, "append_commit() failed"); diff --git a/imap/http_proxy.c b/imap/http_proxy.c index 4065c07451..6a60af8dd8 100644 --- a/imap/http_proxy.c +++ b/imap/http_proxy.c @@ -652,7 +652,7 @@ static void send_response(struct transaction_t *txn, long code, spool_enum_hdrcache(hdrs, &write_cachehdr, txn); if (!body || !(len = buf_len(body))) { - /* Empty body -- use payload headers from response, if any */ + /* Empty body -- use payload headers from response, if any */ const char **hdr; if ((hdr = spool_getheader(hdrs, "Transfer-Encoding"))) { From b49020dd66075dbc153ed0e053ddd5c4b23745f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=B8=D0=BB=D1=8F=D0=BD=20=D0=9F=D0=B0=D0=BB=D0=B0?= =?UTF-8?q?=D1=83=D0=B7=D0=BE=D0=B2?= Date: Sun, 3 Sep 2023 12:32:52 +0000 Subject: [PATCH 018/501] spool.c:parseheader() - remove unnecessary checks for NULL The static function imap/spool.c:parseheader() is called from only one place. The passed parameters headname, contents and rawvalue from that place are not NULL, so there is no need to check within parseheader() if these parameters are NULL. --- imap/spool.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/imap/spool.c b/imap/spool.c index aa8ad487e8..dd48513a1a 100644 --- a/imap/spool.c +++ b/imap/spool.c @@ -110,6 +110,7 @@ typedef enum { on error, returns < 0 */ +__attribute__((nonnull(3,4,5))) static int parseheader(struct protstream *fin, FILE *fout, char **headname, char **contents, char **rawvalue, @@ -291,9 +292,9 @@ static int parseheader(struct protstream *fin, FILE *fout, if (c != EOF) prot_ungetc(c, fin); /* and we didn't get a header */ - if (headname != NULL) *headname = NULL; - if (contents != NULL) *contents = NULL; - if (rawvalue != NULL) *rawvalue = NULL; + *headname = NULL; + *contents = NULL; + *rawvalue = NULL; return r; @@ -303,9 +304,9 @@ static int parseheader(struct protstream *fin, FILE *fout, /* Note: xstrdup()ing the string ensures we return * a minimal length string with no allocation slack * at the end */ - if (headname != NULL) *headname = xstrdup(name.s); - if (contents != NULL) *contents = xstrdup(body.s); - if (rawvalue != NULL) *rawvalue = xstrdup(raw.s); + *headname = xstrdup(name.s); + *contents = xstrdup(body.s); + *rawvalue = xstrdup(raw.s); return 0; } From 505119a569f91f3c38f738db3becab019adbd4ca Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Thu, 19 Oct 2023 15:50:56 -0400 Subject: [PATCH 019/501] DList: non-destructive sort order code (from Rob M review) --- perl/imap/Cyrus/DList.pm | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/perl/imap/Cyrus/DList.pm b/perl/imap/Cyrus/DList.pm index 1aa737c452..18d4d29ef9 100644 --- a/perl/imap/Cyrus/DList.pm +++ b/perl/imap/Cyrus/DList.pm @@ -93,16 +93,19 @@ sub add_perl { elsif (ref($val) eq 'HASH') { my $child = $Self->add_kvlist($key); - my $order = delete $val->{__kvlist_order}; + my %keys = map { $_ => 1 } grep { not m/^_/ } keys %$val; - if ($order) { - die "Unknown order " . ref($order) if ref($order) ne 'ARRAY'; - foreach my $k (grep { exists $val->{$_} } @{$order}) { - $child->add_perl($k, delete $val->{$k}); - } + my $order = $val->{__kvlist_order} || []; + die "Unknown order " . ref($order) if ref($order) ne 'ARRAY'; + + # if there's a pre-defined key order, add all those keys first, in order + foreach my $k (grep { exists $val->{$_} } @$order) { + $child->add_perl($k, $val->{$k}); + delete $keys{$k}; } - $child->add_perl($_, $val->{$_}) for sort keys %$val; + # add the remaining keys in sort order + $child->add_perl($_, $val->{$_}) for sort keys %keys; } elsif (ref($val) eq 'REF') { @@ -327,7 +330,7 @@ sub as_perl { $kvlist->{$datum->{key}} = $datum->as_perl(); } - $kvlist->{__kvlist_order} = [ @order ]; + $kvlist->{__kvlist_order} = \@order; return $kvlist; } elsif ($Self->{type} eq 'list') { From 529c0b87fec486baa93fe6d585626b408cd6720f Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Thu, 19 Oct 2023 16:22:07 -0400 Subject: [PATCH 020/501] Cyrus::AccountSync - skip any mailbox that went away during fetch --- perl/imap/Cyrus/AccountSync.pm | 1 + 1 file changed, 1 insertion(+) diff --git a/perl/imap/Cyrus/AccountSync.pm b/perl/imap/Cyrus/AccountSync.pm index bf63aef454..fc3ba0fb71 100644 --- a/perl/imap/Cyrus/AccountSync.pm +++ b/perl/imap/Cyrus/AccountSync.pm @@ -92,6 +92,7 @@ sub dump_user { for my $folder (@{$info->{MAILBOX}}) { next if $folder->{MBOXTYPE}; # don't dump special folders my $fi = $self->{sync}->dlwrite("GET", "FULLMAILBOX", $folder->{MBOXNAME}); + next unless $fi->{MAILBOX}; # folder went away? my @emails; for my $record (@{$fi->{MAILBOX}[0]{RECORD}||[]}) { my $res = $self->{sync}->dlwrite("GET", "FETCH", { From a8692fa50c2ba63cd99691a652c9bff3ce4b9404 Mon Sep 17 00:00:00 2001 From: ellie timoney Date: Mon, 1 Jul 2024 11:52:24 +1000 Subject: [PATCH 021/501] RunnerXML: use our base class, not generic --- cassandane/Cassandane/Unit/RunnerXML.pm | 33 +++++++++++-------------- cassandane/testrunner.pl | 7 +++--- 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/cassandane/Cassandane/Unit/RunnerXML.pm b/cassandane/Cassandane/Unit/RunnerXML.pm index c2b54eb496..8cddee2cbc 100644 --- a/cassandane/Cassandane/Unit/RunnerXML.pm +++ b/cassandane/Cassandane/Unit/RunnerXML.pm @@ -38,39 +38,37 @@ # package Cassandane::Unit::RunnerXML; +use strict; +use warnings; +use vars qw($VERSION); use XML::Generator; use Time::HiRes qw(time); use Sys::Hostname; use POSIX qw(strftime); -use strict; -use warnings; -use vars qw($VERSION); -# XXX should this inherit from our own Cassandane::Unit::Runner? -use base qw(Test::Unit::Runner); +use lib '.'; +use base qw(Cassandane::Unit::Runner); # $Id: XML.pm 27 2004-08-24 11:22:24Z andrew $ $VERSION = '0.1'; sub new { - my ($class, $directory, $generator) = @_; + my ($class, $params, @args) = @_; - $generator ||= XML::Generator->new(escape => 'always', pretty => 2); + my $self = $class->SUPER::new(@args); - return bless({directory => $directory, gen => $generator, - all_tests_passed => 1, - classrecs => {}}, - $class); -} + $params->{generator} ||= XML::Generator->new(escape => 'always', + pretty => 2); -sub all_tests_passed { - my ($self) = @_; + $self->{directory} = $params->{directory}; + $self->{gen} = $params->{generator}; + $self->{classrecs} = {}; - return $self->{all_tests_passed}; + return $self; } -sub start { +sub do_run { my ($self, $suite) = @_; my $result = $self->create_test_result(); @@ -78,6 +76,7 @@ sub start { my $start_time = time(); $suite->run($result, $self); $self->_emit_xml(); + return $result->was_successful; } sub _classrec { @@ -115,7 +114,6 @@ sub add_failure { my $cr = $self->_classrec($test); my $tr = $self->_testrec($test); $cr->{failures}++; - $self->{all_tests_passed} = 0; push(@{$tr->{child_nodes}}, $self->{gen}->failure({type => _extype($exception), message => $exception->get_message()}, @@ -128,7 +126,6 @@ sub add_error { my $cr = $self->_classrec($test); my $tr = $self->_testrec($test); $cr->{errors}++; - $self->{all_tests_passed} = 0; push(@{$tr->{child_nodes}}, $self->{gen}->error({type => _extype($exception), message => $exception->get_message()}, diff --git a/cassandane/testrunner.pl b/cassandane/testrunner.pl index 6b39233e11..a88b279704 100755 --- a/cassandane/testrunner.pl +++ b/cassandane/testrunner.pl @@ -202,15 +202,16 @@ my ($plan, $fh) = @_; local *__ANON__ = "runner_xml"; - my $runner = Cassandane::Unit::RunnerXML->new($output_dir); + my $runner = Cassandane::Unit::RunnerXML->new({ + directory => $output_dir + }); my @filters = qw(x skip_version skip_missing_features skip_runtime_check enable_wanted_properties); push @filters, 'skip_slow' if $plan->{skip_slow}; push @filters, 'slow_only' if $plan->{slow_only}; $runner->filter(@filters); - $runner->start($plan); - return $runner->all_tests_passed(); + return $runner->do_run($plan, 0); }; }; if ($@) { From e6a702b0eae82e8279c6226d8f6e25a6540fa634 Mon Sep 17 00:00:00 2001 From: ellie timoney Date: Mon, 1 Jul 2024 14:43:54 +1000 Subject: [PATCH 022/501] Runner: decouple output formatting from running tests This disables all output formats except 'tap' for now. They'll be added back as the old RunnerFoos are converted to FormatFoos --- cassandane/Cassandane/Unit/FormatTAP.pm | 73 ++++++++++ cassandane/Cassandane/Unit/Formatter.pm | 185 ++++++++++++++++++++++++ cassandane/Cassandane/Unit/Runner.pm | 52 ++++++- cassandane/testrunner.pl | 61 +++++--- 4 files changed, 342 insertions(+), 29 deletions(-) create mode 100644 cassandane/Cassandane/Unit/FormatTAP.pm create mode 100644 cassandane/Cassandane/Unit/Formatter.pm diff --git a/cassandane/Cassandane/Unit/FormatTAP.pm b/cassandane/Cassandane/Unit/FormatTAP.pm new file mode 100644 index 0000000000..c6d9b220a7 --- /dev/null +++ b/cassandane/Cassandane/Unit/FormatTAP.pm @@ -0,0 +1,73 @@ +#!/usr/bin/perl +# +# Copyright (c) 2011-2017 FastMail Pty Ltd. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# +# 3. The name "Fastmail Pty Ltd" must not be used to +# endorse or promote products derived from this software without +# prior written permission. For permission or any legal +# details, please contact +# FastMail Pty Ltd +# PO Box 234 +# Collins St West 8007 +# Victoria +# Australia +# +# 4. Redistributions of any form whatsoever must retain the following +# acknowledgment: +# "This product includes software developed by Fastmail Pty. Ltd." +# +# FASTMAIL PTY LTD DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, +# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO +# EVENT SHALL OPERA SOFTWARE AUSTRALIA BE LIABLE FOR ANY SPECIAL, INDIRECT +# OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF +# USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +# TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE +# OF THIS SOFTWARE. +# + +package Cassandane::Unit::FormatTAP; +use strict; +use warnings; +use Data::Dumper; +use IO::File; + +use lib '.'; +use base qw(Cassandane::Unit::Formatter); + +sub new +{ + my ($class, $fh) = @_; + return $class->SUPER::new($fh); +} + +sub start_test +{ + my ($self, $test) = @_; + $self->_print('.'); +} + +sub add_error +{ + my ($self, $test, $exception) = @_; + $self->_print('E'); +} + +sub add_failure +{ + my ($self, $test, $exception) = @_; + $self->_print('F'); +} + +1; diff --git a/cassandane/Cassandane/Unit/Formatter.pm b/cassandane/Cassandane/Unit/Formatter.pm new file mode 100644 index 0000000000..f892c122be --- /dev/null +++ b/cassandane/Cassandane/Unit/Formatter.pm @@ -0,0 +1,185 @@ +#!/usr/bin/perl +# +# Copyright (c) 2011-2017 FastMail Pty Ltd. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# +# 3. The name "Fastmail Pty Ltd" must not be used to +# endorse or promote products derived from this software without +# prior written permission. For permission or any legal +# details, please contact +# FastMail Pty Ltd +# PO Box 234 +# Collins St West 8007 +# Victoria +# Australia +# +# 4. Redistributions of any form whatsoever must retain the following +# acknowledgment: +# "This product includes software developed by Fastmail Pty. Ltd." +# +# FASTMAIL PTY LTD DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, +# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO +# EVENT SHALL OPERA SOFTWARE AUSTRALIA BE LIABLE FOR ANY SPECIAL, INDIRECT +# OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF +# USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +# TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE +# OF THIS SOFTWARE. +# + +package Cassandane::Unit::Formatter; +use strict; +use warnings; + +use base 'Test::Unit::Listener'; +use Benchmark; +use IO::Handle; + +sub new +{ + my ($class, $fh) = @_; + + $fh //= \*STDOUT; + $fh->autoflush(1); + + return bless { + remove_me_in_cassandane_child => 1, + fh => $fh, + }, $class; +} + +sub _print +{ + my ($self, @args) = @_; + $self->{fh}->print(@args); +} + +# No-op implementations of Listener interface. To create a new output +# format, subclass from this and override the appropriate event handlers + +sub start_suite +{ + my ($self, $suite) = @_; +} + +sub start_test +{ + my ($self, $test) = @_; +} + +sub add_pass +{ + my ($self, $test) = @_; +} + +sub add_error +{ + my ($self, $test, $exception) = @_; +} + +sub add_failure +{ + my ($self, $test, $exception) = @_; +} + +sub end_test +{ + my ($self, $test) = @_; +} + +# Override this with your output format's end-of-tests handling. The +# default is to print a summary. +sub finished +{ + my ($self, $result, $start_time, $end_time) = @_; + $self->print_summary($result, $start_time, $end_time); +} + +# Override this, and/or subs print_header, print_errors, print_failures +# to change how the summary is presented. +sub print_summary +{ + my ($self, $result, $start_time, $end_time) = @_; + + my $run_time = timediff($end_time, $start_time); + print "\n", "Time: ", timestr($run_time), "\n"; + + $self->print_header($result); + $self->print_errors($result); + $self->print_failures($result); +} + +sub print_header +{ + my ($self, $result) = @_; + + if ($result->was_successful()) { + $self->_print("\n", "OK", " (", $result->run_count(), " tests)\n"); + } + else { + $self->_print("\n", "!!!FAILURES!!!", "\n", + "Test Results:\n", + "Run: ", $result->run_count(), + ", Failures: ", $result->failure_count(), + ", Errors: ", $result->error_count(), + "\n"); + } +} + +sub print_errors +{ + my ($self, $result) = @_; + + return unless my $error_count = $result->error_count(); + + my $msg = "\nThere " . + ($error_count == 1 ? + "was 1 error" + : "were $error_count errors") . + ":\n"; + $self->_print($msg); + + my $i = 0; + for my $e (@{$result->errors()}) { + chomp(my $e_to_str = $e); + $i++; + $self->_print("$i) $e_to_str\n"); + $self->_print("\nAnnotations:\n", $e->object->annotations()) + if $e->object->annotations(); + } +} + +sub print_failures +{ + my ($self, $result) = @_; + + return unless my $failure_count = $result->failure_count; + + my $msg = "\nThere " . + ($failure_count == 1 ? + "was 1 failure" + : "were $failure_count failures") . + ":\n"; + $self->_print($msg); + + my $i = 0; + for my $f (@{$result->failures()}) { + chomp(my $f_to_str = $f); + $self->_print("\n") if $i++; + $self->_print("$i) $f_to_str\n"); + $self->_print("\nAnnotations:\n", $f->object->annotations()) + if $f->object->annotations(); + } +} + +1; diff --git a/cassandane/Cassandane/Unit/Runner.pm b/cassandane/Cassandane/Unit/Runner.pm index 0ae79d11c0..770cc00e3b 100644 --- a/cassandane/Cassandane/Unit/Runner.pm +++ b/cassandane/Cassandane/Unit/Runner.pm @@ -40,8 +40,9 @@ package Cassandane::Unit::Runner; use strict; use warnings; -use base qw(Test::Unit::TestRunner); +use base qw(Test::Unit::Runner); use Test::Unit::Result; +use Benchmark; use IO::File; use lib '.'; @@ -49,17 +50,18 @@ use Cassandane::Cassini; sub new { - my $class = shift; - my $self = $class->SUPER::new(@_); - $self->{remove_me_in_cassandane_child} = 1; + my ($class) = @_; my $cassini = Cassandane::Cassini->instance(); my $rootdir = $cassini->val('cassandane', 'rootdir', '/var/tmp/cass'); my $failed_file = "$rootdir/failed"; - $self->{failed_fh} = IO::File->new($failed_file, 'w'); # if we can't write there, we just won't record failed tests! - return $self; + return bless { + remove_me_in_cassandane_child => 1, + formatters => [], + failed_fh => IO::File->new($failed_file, 'w'), + }, $class; } sub create_test_result @@ -69,6 +71,42 @@ sub create_test_result return $self->{_result}; } +sub add_formatter +{ + my ($self, $formatter) = @_; + + push @{$self->{formatters}}, $formatter; +} + +sub do_run +{ + my ($self, $suite) = @_; + my $result = $self->create_test_result(); + + $result->add_listener($self); + foreach my $f (@{$self->{formatters}}) { + $result->add_listener($f); + } + + my $start_time = new Benchmark(); + $suite->run($result, $self); + my $end_time = new Benchmark(); + + foreach my $f (@{$self->{formatters}}) { + $f->finished($result, $start_time, $end_time); + } + + return $result->was_successful; +} + +sub start_suite { } + +sub start_test { } + +sub end_test { } + +sub add_pass { } + sub record_failed { my ($self, $test) = @_; @@ -87,14 +125,12 @@ sub add_error { my ($self, $test) = @_; $self->record_failed($test); - $self->SUPER::add_error($test); } sub add_failure { my ($self, $test) = @_; $self->record_failed($test); - $self->SUPER::add_failure($test); } 1; diff --git a/cassandane/testrunner.pl b/cassandane/testrunner.pl index a88b279704..c241c76d1c 100755 --- a/cassandane/testrunner.pl +++ b/cassandane/testrunner.pl @@ -45,6 +45,7 @@ use lib '.'; use Cassandane::Util::Setup; +use Cassandane::Unit::FormatTAP; use Cassandane::Unit::Runner; use Cassandane::Unit::RunnerPretty; use Cassandane::Unit::TestPlan; @@ -58,7 +59,7 @@ $Data::Dumper::Sortkeys = 1; $Data::Dumper::Trailingcomma = 1; -my $format = 'prettier'; +my %want_formats = (); my $output_dir = 'reports'; my $do_list = 0; # The default really should be --no-keep-going like make @@ -137,21 +138,17 @@ }; }; +my %formatters = ( + tap => { + writes_to_stdout => 1, + formatter => sub { + my ($fh) = @_; + return Cassandane::Unit::FormatTAP->new($fh); + }, + }, +); my %runners = ( - tap => sub - { - my ($plan, $fh) = @_; - local *__ANON__ = "runner_tap"; - my $runner = Cassandane::Unit::Runner->new($fh); - my @filters = qw(x skip_version skip_missing_features - skip_runtime_check - enable_wanted_properties); - push @filters, 'skip_slow' if $plan->{skip_slow}; - push @filters, 'slow_only' if $plan->{slow_only}; - $runner->filter(@filters); - return $runner->do_run($plan, 0); - }, pretty => sub { my ($plan, $fh) = @_; @@ -249,13 +246,15 @@ sub usage } elsif ($a eq '-f') { - $format = shift; - usage unless defined $runners{$format}; + my $format = shift; + usage unless defined $formatters{$format}; + $want_formats{$format} = 1; } elsif ($a =~ m/^-f(\w+)$/) { - $format = $1; - usage unless defined $runners{$format}; + my $format = $1; + usage unless defined $formatters{$format}; + $want_formats{$format} = 1; } elsif ($a eq '-v' || $a eq '--verbose') { @@ -384,10 +383,30 @@ sub usage { # Build the schedule per commandline $plan->schedule(@names); + # Run the schedule - open my $fh, '>&', \*STDOUT - or die "Cannot save STDOUT as a runner print stream: $!"; - exit(! $runners{$format}->($plan, $fh)); + $want_formats{tap} = 1 if not scalar keys %want_formats; + my @writes_to_stdout = grep { + $formatters{$_}->{writes_to_stdout} + } keys %want_formats; + if (scalar @writes_to_stdout > 1) { + my $joined = join ', ', map { "'$_'" } @writes_to_stdout; + die "$joined formatters all want to write to stdout\n"; + } + + my @filters = qw(x skip_version skip_missing_features + skip_runtime_check + enable_wanted_properties); + push @filters, 'skip_slow' if $plan->{skip_slow}; + push @filters, 'slow_only' if $plan->{slow_only}; + + my $runner = Cassandane::Unit::Runner->new(); + foreach my $f (keys %want_formats) { + $runner->add_formatter($formatters{$f}->{formatter}->()); + } + $runner->filter(@filters); + + exit !$runner->do_run($plan); } sub _listitem { From 9bf013db17b1bbb4bfc0c4a527b5a51a507a5509 Mon Sep 17 00:00:00 2001 From: ellie timoney Date: Fri, 5 Jul 2024 09:58:12 +1000 Subject: [PATCH 023/501] Runner: add tell_formatters() for custom event handling --- cassandane/Cassandane/Unit/Runner.pm | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/cassandane/Cassandane/Unit/Runner.pm b/cassandane/Cassandane/Unit/Runner.pm index 770cc00e3b..65522a2d82 100644 --- a/cassandane/Cassandane/Unit/Runner.pm +++ b/cassandane/Cassandane/Unit/Runner.pm @@ -78,6 +78,19 @@ sub add_formatter push @{$self->{formatters}}, $formatter; } +# this is very similar to Test::Unit::Result's tell_listeners(), except +# without the annoying crash when the listener doesn't care about the event +sub tell_formatters +{ + my ($self, $method, @args) = @_; + + foreach my $formatter (@{$self->{formatters}}) { + if ($formatter->can($method)) { + $formatter->$method(@args); + } + } +} + sub do_run { my ($self, $suite) = @_; From dc912398f3e8f3e73a96c8d75e23c19db091038c Mon Sep 17 00:00:00 2001 From: ellie timoney Date: Mon, 1 Jul 2024 14:45:13 +1000 Subject: [PATCH 024/501] TestPlan: use tell_formatters() for fake_start_time event --- cassandane/Cassandane/Unit/TestPlan.pm | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/cassandane/Cassandane/Unit/TestPlan.pm b/cassandane/Cassandane/Unit/TestPlan.pm index 2837769b86..b2837a7989 100644 --- a/cassandane/Cassandane/Unit/TestPlan.pm +++ b/cassandane/Cassandane/Unit/TestPlan.pm @@ -871,10 +871,20 @@ sub _finish_workitem my ($self, $witem, $result, $runner) = @_; my ($suite, $test) = $self->_get_suite_and_test($witem); + # The test was actually started earlier by _run_workitem, but its + # start_test event wasn't sent. It might have got swallowed due to + # the output format listeners being removed in the workitem handling. + # Send the event again now, to make sure the formatters actually get + # it... $result->start_test($test); - if ($runner->can('fake_start_time')) + # But! If they're computing their own start time based on this event + # they'll get it wrong. We know the real start time, so tell the + # formatter to use that instead. + if ($runner->can('tell_formatters')) { - $runner->fake_start_time($test, $witem->{start_time}); + $runner->tell_formatters('fake_start_time', + $test, + $witem->{start_time}); } $test->annotate_from_file($witem->{logfile}); From 6c8cad8f1ae758255b0a51bf4bf62cde0ad75e5d Mon Sep 17 00:00:00 2001 From: ellie timoney Date: Fri, 5 Jul 2024 10:24:10 +1000 Subject: [PATCH 025/501] RunnerPretty: convert to FormatPretty --- .../Unit/{RunnerPretty.pm => FormatPretty.pm} | 28 ++++-------- cassandane/testrunner.pl | 44 +++++++------------ 2 files changed, 24 insertions(+), 48 deletions(-) rename cassandane/Cassandane/Unit/{RunnerPretty.pm => FormatPretty.pm} (90%) diff --git a/cassandane/Cassandane/Unit/RunnerPretty.pm b/cassandane/Cassandane/Unit/FormatPretty.pm similarity index 90% rename from cassandane/Cassandane/Unit/RunnerPretty.pm rename to cassandane/Cassandane/Unit/FormatPretty.pm index 94a20f4590..1c7f9c8c8b 100644 --- a/cassandane/Cassandane/Unit/RunnerPretty.pm +++ b/cassandane/Cassandane/Unit/FormatPretty.pm @@ -37,12 +37,12 @@ # OF THIS SOFTWARE. # -package Cassandane::Unit::RunnerPretty; +package Cassandane::Unit::FormatPretty; use strict; use warnings; use lib '.'; -use base qw(Cassandane::Unit::Runner); +use base qw(Cassandane::Unit::Formatter); sub new { @@ -66,7 +66,7 @@ sub new sub ansi { my ($self, $codes, @args) = @_; - my $isatty = -t $self->print_stream; + my $isatty = -t $self->{fh}; my $ansi; @@ -77,13 +77,6 @@ sub ansi return $ansi; } -sub start_test -{ - my $self = shift; - my $test = shift; - # prevent the default action which is to print "." -} - sub add_pass { my $self = shift; @@ -100,8 +93,6 @@ sub add_error my $self = shift; my $test = shift; - $self->record_failed($test); - my $line = sprintf "%s %s\n", $self->ansi([31], '[ERROR ]'), _getname($test); @@ -113,7 +104,6 @@ sub add_failure my $self = shift; my $test = shift; - $self->record_failed($test); my $line = sprintf "%s %s\n", $self->ansi([33], '[FAILED]'), _getname($test); @@ -149,8 +139,8 @@ sub print_errors my $saved_output_stream; if ($self->{_quiet}) { if ($self->{_quiet_report_fh}) { - $saved_output_stream = $self->{_Print_stream}; - $self->{_Print_stream} = $self->{_quiet_report_fh}; + $saved_output_stream = $self->{fh}; + $self->{fh} = $self->{_quiet_report_fh}; } else { return; @@ -178,7 +168,7 @@ sub print_errors } if ($saved_output_stream) { - $self->{_Print_stream} = $saved_output_stream; + $self->{fh} = $saved_output_stream; } } @@ -189,8 +179,8 @@ sub print_failures my $saved_output_stream; if ($self->{_quiet}) { if ($self->{_quiet_report_fh}) { - $saved_output_stream = $self->{_Print_stream}; - $self->{_Print_stream} = $self->{_quiet_report_fh}; + $saved_output_stream = $self->{fh}; + $self->{fh} = $self->{_quiet_report_fh}; } else { return; @@ -218,7 +208,7 @@ sub print_failures } if ($saved_output_stream) { - $self->{_Print_stream} = $saved_output_stream; + $self->{fh} = $saved_output_stream; } } diff --git a/cassandane/testrunner.pl b/cassandane/testrunner.pl index c241c76d1c..d08cdbd791 100755 --- a/cassandane/testrunner.pl +++ b/cassandane/testrunner.pl @@ -45,9 +45,9 @@ use lib '.'; use Cassandane::Util::Setup; +use Cassandane::Unit::FormatPretty; use Cassandane::Unit::FormatTAP; use Cassandane::Unit::Runner; -use Cassandane::Unit::RunnerPretty; use Cassandane::Unit::TestPlan; use Cassandane::Util::Log; use Cassandane::Cassini; @@ -146,36 +146,22 @@ return Cassandane::Unit::FormatTAP->new($fh); }, }, -); -my %runners = -( - pretty => sub - { - my ($plan, $fh) = @_; - local *__ANON__ = "runner_pretty"; - my $runner = Cassandane::Unit::RunnerPretty->new({}, $fh); - my @filters = qw(x skip_version skip_missing_features - skip_runtime_check - enable_wanted_properties); - push @filters, 'skip_slow' if $plan->{skip_slow}; - push @filters, 'slow_only' if $plan->{slow_only}; - $runner->filter(@filters); - return $runner->do_run($plan, 0); + pretty => { + writes_to_stdout => 1, + formatter => sub { + my ($fh) = @_; + return Cassandane::Unit::FormatPretty->new({}, $fh); + }, }, - prettier => sub - { - my ($plan, $fh) = @_; - local *__ANON__ = "runner_prettier"; - my $runner = Cassandane::Unit::RunnerPretty->new({quiet=>1}, $fh); - my @filters = qw(x skip_version skip_missing_features - skip_runtime_check - enable_wanted_properties); - push @filters, 'skip_slow' if $plan->{skip_slow}; - push @filters, 'slow_only' if $plan->{slow_only}; - $runner->filter(@filters); - return $runner->do_run($plan, 0); + prettier => { + writes_to_stdout => 1, + formatter => sub { + my ($fh) = @_; + return Cassandane::Unit::FormatPretty->new({quiet=>1}, $fh); + }, }, ); +my %runners = (); become_cyrus(); @@ -385,7 +371,7 @@ sub usage $plan->schedule(@names); # Run the schedule - $want_formats{tap} = 1 if not scalar keys %want_formats; + $want_formats{prettier} = 1 if not scalar keys %want_formats; my @writes_to_stdout = grep { $formatters{$_}->{writes_to_stdout} } keys %want_formats; From ad97843e4a56e264b83b3d271b1df9f9340a3c01 Mon Sep 17 00:00:00 2001 From: ellie timoney Date: Fri, 5 Jul 2024 10:50:23 +1000 Subject: [PATCH 026/501] RunnerXML: convert to FormatXML --- .../Unit/{RunnerXML.pm => FormatXML.pm} | 97 +++---------------- cassandane/testrunner.pl | 46 +++------ 2 files changed, 28 insertions(+), 115 deletions(-) rename cassandane/Cassandane/Unit/{RunnerXML.pm => FormatXML.pm} (77%) diff --git a/cassandane/Cassandane/Unit/RunnerXML.pm b/cassandane/Cassandane/Unit/FormatXML.pm similarity index 77% rename from cassandane/Cassandane/Unit/RunnerXML.pm rename to cassandane/Cassandane/Unit/FormatXML.pm index 8cddee2cbc..0770661cb2 100644 --- a/cassandane/Cassandane/Unit/RunnerXML.pm +++ b/cassandane/Cassandane/Unit/FormatXML.pm @@ -37,7 +37,7 @@ # OF THIS SOFTWARE. # -package Cassandane::Unit::RunnerXML; +package Cassandane::Unit::FormatXML; use strict; use warnings; use vars qw($VERSION); @@ -48,9 +48,8 @@ use Sys::Hostname; use POSIX qw(strftime); use lib '.'; -use base qw(Cassandane::Unit::Runner); +use base qw(Cassandane::Unit::Formatter); -# $Id: XML.pm 27 2004-08-24 11:22:24Z andrew $ $VERSION = '0.1'; sub new { @@ -68,17 +67,6 @@ sub new { return $self; } -sub do_run { - my ($self, $suite) = @_; - - my $result = $self->create_test_result(); - $result->add_listener($self); - my $start_time = time(); - $suite->run($result, $self); - $self->_emit_xml(); - return $result->was_successful; -} - sub _classrec { my ($self, $test) = @_; @@ -97,8 +85,6 @@ sub _testrec { { start_time => 0, node => undef, child_nodes => [] }; } -sub add_pass {} - sub _extype { my ($exception) = @_; @@ -194,6 +180,16 @@ sub _emit_xml { } } +sub finished +{ + my ($self, $result, $start_time, $end_time) = @_; + + # XXX This class does all its own accounting, which is probably + # XXX redundant since it doesn't report anything that it couldn't + # XXX just get from the usual $result/$start_time/$end_time args. + $self->_emit_xml(); +} + sub _xml_filename { my ($self, $class) = @_; @@ -202,72 +198,3 @@ sub _xml_filename { } 1; - -__END__ - - -=head1 NAME - -Test::Unit::Runner::XML - Generate XML reports from unit test results - -=head1 SYNOPSIS - - use Test::Unit::Runner::XML; - - mkdir("test_reports"); - my $runner = Test::Unit::Runner::XML->new("test-reports"); - $runner->start($test); - exit(!$runner->all_tests_passed()); - -=head1 DESCRIPTION - -Test::Unit::Runner::XML generates XML reports from unit test results. The -reports are in the same format as those produced by Ant's JUnit task, -allowing them to be used with Java continuous integration and reporting tools. - -=head1 CONSTRUCTOR - - Test::Unit::Runner::XML->new($directory) - -Construct a new runner that will write XML reports into $directory - -=head1 METHODS - -=head2 start - - $runner->start($test); - -Run the L $test and generate XML reports from the results. - -=head2 all_tests_passed - - exit(!$runner->all_tests_passed()); - -Return true if all tests executed by $runner since it was constructed passed. - -=head1 AUTHOR - -Copyright (c) 2004 Andrew Eland, Eandrew@andreweland.orgE. - -All rights reserved. This program is free software; you can -redistribute it and/or modify it under the same terms as Perl itself. - -=head1 SEE ALSO - -=over 4 - -=item * - -L - -=item * - -L - -=item * - -The Ant JUnit task, http://ant.apache.org/ - -=cut - - diff --git a/cassandane/testrunner.pl b/cassandane/testrunner.pl index d08cdbd791..716b1bc9ce 100755 --- a/cassandane/testrunner.pl +++ b/cassandane/testrunner.pl @@ -47,6 +47,7 @@ use Cassandane::Util::Setup; use Cassandane::Unit::FormatPretty; use Cassandane::Unit::FormatTAP; +use Cassandane::Unit::FormatXML; use Cassandane::Unit::Runner; use Cassandane::Unit::TestPlan; use Cassandane::Util::Log; @@ -160,49 +161,34 @@ return Cassandane::Unit::FormatPretty->new({quiet=>1}, $fh); }, }, + xml => { + writes_to_stdout => 0, + formatter => sub { + my ($fh) = @_; + return Cassandane::Unit::FormatXML->new({ + directory => $output_dir + }); + }, + }, ); -my %runners = (); become_cyrus(); -eval -{ - require Cassandane::Unit::RunnerXML; - - if ( ! -d $output_dir ) - { +eval { + if ( ! -d $output_dir ) { mkdir($output_dir) or die "Cannot make output directory \"$output_dir\": $!\n"; } - if (! -w $output_dir ) - { + if (! -w $output_dir ) { die "Cannot write to output directory \"$output_dir\"\n"; } - - $runners{xml} = sub - { - my ($plan, $fh) = @_; - local *__ANON__ = "runner_xml"; - - my $runner = Cassandane::Unit::RunnerXML->new({ - directory => $output_dir - }); - my @filters = qw(x skip_version skip_missing_features - skip_runtime_check - enable_wanted_properties); - push @filters, 'skip_slow' if $plan->{skip_slow}; - push @filters, 'slow_only' if $plan->{slow_only}; - $runner->filter(@filters); - return $runner->do_run($plan, 0); - }; }; if ($@) { my $eval_err = $@; - $runners{xml} = sub - { - print STDERR "Sorry, XML output format not available due to:\n=> $eval_err"; - return 0; + $formatters{xml}->{formatter} = sub { + die "Sorry, XML output format not available due to:\n", + "=> $eval_err"; }; } From 9e4a426dededd10171e6cb786657e64368f9eb17 Mon Sep 17 00:00:00 2001 From: ellie timoney Date: Mon, 24 Jun 2024 13:54:17 +1000 Subject: [PATCH 027/501] imapoptions: add fatals_abort option --- changes/next/fatals-abort | 18 ++++++++++++++++++ lib/imapoptions | 8 ++++++++ lib/libconfig.c | 5 +++++ lib/libconfig.h | 1 + 4 files changed, 32 insertions(+) create mode 100644 changes/next/fatals-abort diff --git a/changes/next/fatals-abort b/changes/next/fatals-abort new file mode 100644 index 0000000000..147f303498 --- /dev/null +++ b/changes/next/fatals-abort @@ -0,0 +1,18 @@ +Description: + +Adds an option for fatal errors to abort and produce a core dump. + + +Config changes: + +fatals_abort + + +Upgrade instructions: + +None + + +GitHub issue: + +None diff --git a/lib/imapoptions b/lib/imapoptions index 0fe0f58f5c..f1f7721a4f 100644 --- a/lib/imapoptions +++ b/lib/imapoptions @@ -998,6 +998,14 @@ Blank lines and lines beginning with ``#'' are ignored. For backward compatibility, if no unit is specified, seconds is assumed. */ +{ "fatals_abort", 0, SWITCH, "UNRELEASED" } +/* If enabled, when fatal errors are detected, Cyrus will call abort() + to produce a core dump, unless the error was due to a misbehaving + client. This is useful if you have the ability to debug a core dump. +.PP + If not enabled (the default), the process will exit normally with + an error code. */ + { "flushseenstate", 1, SWITCH, "2.5.0", "2.5.0" } /* Deprecated. No longer used. */ diff --git a/lib/libconfig.c b/lib/libconfig.c index 86c205000b..def68bb308 100644 --- a/lib/libconfig.c +++ b/lib/libconfig.c @@ -91,6 +91,7 @@ EXPORTED int config_qosmarking; EXPORTED int config_debug; EXPORTED toggle_debug_cb config_toggle_debug_cb = NULL; EXPORTED int config_debug_slowio = 0; +EXPORTED int config_fatals_abort = 0; static int config_loaded; @@ -663,6 +664,7 @@ EXPORTED void config_reset(void) config_debug = 0; config_toggle_debug_cb = NULL; config_debug_slowio = 0; + config_fatals_abort = 0; /* reset all the options */ for (opt = IMAPOPT_ZERO; opt < IMAPOPT_LAST; opt++) { @@ -893,6 +895,9 @@ EXPORTED void config_read(const char *alt_config, const int config_need_data) /* do we want artificially-slow I/O ops */ config_debug_slowio = config_getswitch(IMAPOPT_DEBUG_SLOWIO); + + /* do we want to abort() on fatal errors */ + config_fatals_abort = config_getswitch(IMAPOPT_FATALS_ABORT); } #define GROWSIZE 4096 diff --git a/lib/libconfig.h b/lib/libconfig.h index 71cf1b7731..5837970de6 100644 --- a/lib/libconfig.h +++ b/lib/libconfig.h @@ -103,6 +103,7 @@ extern unsigned config_maxword; extern int config_qosmarking; extern int config_debug; extern int config_debug_slowio; +extern int config_fatals_abort; /* for toggling config_debug and its behaviours at runtime */ typedef void (*toggle_debug_cb)(void); From 74ee258df84d8c69ed87ab1cdd569fd4dcafe915 Mon Sep 17 00:00:00 2001 From: ellie timoney Date: Wed, 26 Jun 2024 11:32:50 +1000 Subject: [PATCH 028/501] sieved: load config --- sieve/sieved.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/sieve/sieved.c b/sieve/sieved.c index 95b604ab6d..bac2084aa4 100644 --- a/sieve/sieved.c +++ b/sieve/sieved.c @@ -63,6 +63,7 @@ #include +#include "libconfig.h" #include "map.h" #include "times.h" @@ -74,7 +75,7 @@ EXPORTED void fatal(const char *s, int code) { fprintf(stderr, "Fatal error: %s (%d)\r\n", s, code); - exit(1); + exit(code); } static int load(int fd, bytecode_input_t ** d) @@ -104,13 +105,15 @@ int main(int argc, char * argv[]) bytecode_input_t *bc = NULL; int script_fd; int opt, usage_error = 0, gen_script = 0; + char *alt_config = NULL; unsigned long len; /* keep this in alphabetical order */ - static const char short_options[] = "s"; + static const char short_options[] = "C:s"; static const struct option long_options[] = { + /* n.b. no long option for -C */ { "as-sieve", no_argument, NULL, 's' }, { 0, 0, 0, 0 }, @@ -120,6 +123,9 @@ int main(int argc, char * argv[]) short_options, long_options, NULL))) { switch (opt) { + case 'C': /* alt config file */ + alt_config = optarg; + break; case 's': gen_script = 1; break; @@ -143,6 +149,9 @@ int main(int argc, char * argv[]) exit(1); } + /* Load configuration file. */ + config_read(alt_config, 0); + len=load(script_fd,&bc); close(script_fd); From 9f4602cc720a3feef1a11aaf2ba588de0b5c5d7e Mon Sep 17 00:00:00 2001 From: ellie timoney Date: Mon, 24 Jun 2024 14:05:08 +1000 Subject: [PATCH 029/501] various: add fatals_abort support (based on capstone commit) --- imap/calalarmd.c | 2 ++ imap/httpd.c | 3 +++ imap/idled.c | 2 ++ imap/imapd.c | 3 +++ imap/lmtpd.c | 4 ++-- imap/pop3d.c | 3 +++ imap/promstatsd.c | 2 ++ imap/sync_client.c | 3 +++ imap/sync_server.c | 1 + lib/assert.c | 3 +++ master/master.c | 1 + sieve/sieved.c | 3 +++ 12 files changed, 28 insertions(+), 2 deletions(-) diff --git a/imap/calalarmd.c b/imap/calalarmd.c index 4e5151caee..72ac749f89 100644 --- a/imap/calalarmd.c +++ b/imap/calalarmd.c @@ -76,6 +76,8 @@ EXPORTED void fatal(const char *msg, int err) cyrus_done(); + if (err != EX_PROTOCOL && config_fatals_abort) abort(); + exit(err); } diff --git a/imap/httpd.c b/imap/httpd.c index 9625c7c55d..e4385cd7f3 100644 --- a/imap/httpd.c +++ b/imap/httpd.c @@ -1235,6 +1235,9 @@ EXPORTED void fatal(const char* s, int code) } syslog(LOG_ERR, "%s%s", fatal, s); + + if (code != EX_PROTOCOL && config_fatals_abort) abort(); + shut_down(code); } diff --git a/imap/idled.c b/imap/idled.c index 2214af5de7..506a9c6175 100644 --- a/imap/idled.c +++ b/imap/idled.c @@ -113,6 +113,8 @@ EXPORTED void fatal(const char *msg, int err) cyrus_done(); + if (err != EX_PROTOCOL && config_fatals_abort) abort(); + exit(err); } diff --git a/imap/imapd.c b/imap/imapd.c index 18cc596ed8..3708cc1da0 100644 --- a/imap/imapd.c +++ b/imap/imapd.c @@ -1443,6 +1443,9 @@ EXPORTED void fatal(const char *s, int code) } syslog(LOG_ERR, "Fatal error: %s", s); + + if (code != EX_PROTOCOL && config_fatals_abort) abort(); + shut_down(code); } diff --git a/imap/lmtpd.c b/imap/lmtpd.c index 1b3d4b35d3..c322a7d3e2 100644 --- a/imap/lmtpd.c +++ b/imap/lmtpd.c @@ -1034,10 +1034,10 @@ EXPORTED void fatal(const char* s, int code) syslog(LOG_ERR, "FATAL: %s", s); + if (code != EX_PROTOCOL && config_fatals_abort) abort(); + /* shouldn't return */ shut_down(code); - - exit(code); } /* diff --git a/imap/pop3d.c b/imap/pop3d.c index eb764dd71b..941dc5738f 100644 --- a/imap/pop3d.c +++ b/imap/pop3d.c @@ -674,6 +674,9 @@ EXPORTED void fatal(const char* s, int code) prot_flush(popd_out); } syslog(LOG_ERR, "Fatal error: %s", s); + + if (code != EX_PROTOCOL && config_fatals_abort) abort(); + shut_down(code); } diff --git a/imap/promstatsd.c b/imap/promstatsd.c index 0a83a7f929..5d89c5a63e 100644 --- a/imap/promstatsd.c +++ b/imap/promstatsd.c @@ -87,6 +87,8 @@ EXPORTED void fatal(const char *msg, int err) syslog(LOG_CRIT, "%s", msg); syslog(LOG_NOTICE, "exiting"); + if (err != EX_PROTOCOL && config_fatals_abort) abort(); + shut_down(err); } diff --git a/imap/sync_client.c b/imap/sync_client.c index 94a0eaec19..5d69f72c3b 100644 --- a/imap/sync_client.c +++ b/imap/sync_client.c @@ -138,6 +138,9 @@ EXPORTED void fatal(const char *s, int code) { fprintf(stderr, "Fatal error: %s\n", s); syslog(LOG_ERR, "Fatal error: %s", s); + + if (code != EX_PROTOCOL && config_fatals_abort) abort(); + exit(code); } diff --git a/imap/sync_server.c b/imap/sync_server.c index c8f668a6f1..caa4784259 100644 --- a/imap/sync_server.c +++ b/imap/sync_server.c @@ -465,6 +465,7 @@ EXPORTED void fatal(const char* s, int code) prot_flush(sync_out); } syslog(LOG_ERR, "Fatal error: %s", s); + if (code != EX_PROTOCOL && config_fatals_abort) abort(); shut_down(code); } diff --git a/lib/assert.c b/lib/assert.c index 48d87b37ad..d1de82f6e9 100644 --- a/lib/assert.c +++ b/lib/assert.c @@ -47,6 +47,8 @@ #include "xmalloc.h" #include "assert.h" +extern int config_fatals_abort; + EXPORTED void assertionfailed(const char *file, int line, const char *expr) { @@ -54,5 +56,6 @@ assertionfailed(const char *file, int line, const char *expr) snprintf(buf, sizeof(buf), "Internal error: assertion failed: %s: %d%s%s", file, line, expr ? ": " : "", expr ? expr : ""); + if (config_fatals_abort) abort(); fatal(buf, EX_SOFTWARE); } diff --git a/master/master.c b/master/master.c index eb04de5b96..830d85a6f7 100644 --- a/master/master.c +++ b/master/master.c @@ -212,6 +212,7 @@ EXPORTED void fatal(const char *msg, int code) { syslog(LOG_CRIT, "%s", msg); syslog(LOG_NOTICE, "exiting"); + if (code != EX_PROTOCOL && config_fatals_abort) abort(); exit(code); } diff --git a/sieve/sieved.c b/sieve/sieved.c index bac2084aa4..dbb5be4c51 100644 --- a/sieve/sieved.c +++ b/sieve/sieved.c @@ -60,6 +60,7 @@ #include #include #include +#include #include @@ -75,6 +76,8 @@ EXPORTED void fatal(const char *s, int code) { fprintf(stderr, "Fatal error: %s (%d)\r\n", s, code); + if (code != EX_PROTOCOL && config_fatals_abort) abort(); + exit(code); } From c78a0159e4da426a3ed62b2aca24f588360b59fd Mon Sep 17 00:00:00 2001 From: ellie timoney Date: Mon, 24 Jun 2024 13:54:49 +1000 Subject: [PATCH 030/501] Simple: add tests for fatals_abort behaviour --- cassandane/Cassandane/Cyrus/Simple.pm | 59 +++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/cassandane/Cassandane/Cyrus/Simple.pm b/cassandane/Cassandane/Cyrus/Simple.pm index d07d1cf41c..002b3c5472 100644 --- a/cassandane/Cassandane/Cyrus/Simple.pm +++ b/cassandane/Cassandane/Cyrus/Simple.pm @@ -302,4 +302,63 @@ EOF $self->assert_str_equals("test\r\n", $res->{1}{binary}); } +sub test_fatals_abort_enabled + :NoStartInstances +{ + my ($self) = @_; + + $self->{instance}->{config}->set( + 'fatals_abort' => 'yes', + 'prometheus_enabled' => 'no', + ); + $self->_start_instances(); + + my $basedir = $self->{instance}->get_basedir(); + + # run `promstatsd -1` without having set up for prometheus, which should + # produce a "Prometheus metrics are not being tracked..." fatal error + eval { + $self->{instance}->run_command({ cyrus => 1 }, 'promstatsd', '-1'); + }; + my $e = $@; + $self->assert_not_null($e); + $self->assert_matches(qr{promstatsd pid \d+\) terminated by signal 6}, + $e->{'-text'}); + + my @cores = $self->{instance}->find_cores(); + if (@cores) { + # if we dumped core, there'd better only be one core file + $self->assert_num_equals(1, scalar @cores); + + # don't barf on it existing during shutdown + unlink $cores[0]; + } +} + +sub test_fatals_abort_disabled + :NoStartInstances +{ + my ($self) = @_; + + $self->{instance}->{config}->set( + 'fatals_abort' => 'no', + 'prometheus_enabled' => 'no', + ); + $self->_start_instances(); + + my $basedir = $self->{instance}->get_basedir(); + + # run `promstatsd -1` without having set up for prometheus, which should + # produce a "Prometheus metrics are not being tracked..." fatal error + eval { + $self->{instance}->run_command({ cyrus => 1 }, 'promstatsd', '-1'); + }; + my $e = $@; + $self->assert_not_null($e); + $self->assert_matches(qr{promstatsd pid \d+\) exited with code 78}, + $e->{'-text'}); + + # post-test sanity checks will complain for us if a core was left behind +} + 1; From ed5d2b903067cba0ad08df94d69d6ba12b96bd60 Mon Sep 17 00:00:00 2001 From: ellie timoney Date: Wed, 26 Jun 2024 12:05:54 +1000 Subject: [PATCH 031/501] various: add more fatals_abort support --- backup/backupd.c | 3 +++ backup/ctl_backups.c | 3 +++ backup/cyr_backup.c | 3 +++ backup/restore.c | 3 +++ imap/cli_fatal.c | 4 ++++ imap/ctl_conversationsdb.c | 4 +++- imap/cvt_xlist_specialuse.c | 3 +++ imap/deliver.c | 3 +++ imap/fud.c | 3 ++- imap/message_test.c | 3 +++ imap/mupdate.c | 6 +++--- imap/nntpd.c | 3 +++ imap/search_test.c | 3 +++ imap/smmapd.c | 3 ++- imap/sync_reset.c | 3 +++ netnews/remotepurge.c | 3 +++ notifyd/notifyd.c | 2 ++ ptclient/ptloader.c | 4 +++- sieve/sievec.c | 5 ++++- sieve/test.c | 4 ++++ sieve/test_mailbox.c | 3 +++ timsieved/timsieved.c | 2 ++ 22 files changed, 65 insertions(+), 8 deletions(-) diff --git a/backup/backupd.c b/backup/backupd.c index 919ed2c568..33a91fe018 100644 --- a/backup/backupd.c +++ b/backup/backupd.c @@ -173,6 +173,9 @@ EXPORTED void fatal(const char* s, int code) prot_flush(backupd_out); } syslog(LOG_ERR, "Fatal error: %s", s); + + if (code != EX_PROTOCOL && config_fatals_abort) abort(); + shut_down(code); } diff --git a/backup/ctl_backups.c b/backup/ctl_backups.c index 21e2cf3ddb..27942d9075 100644 --- a/backup/ctl_backups.c +++ b/backup/ctl_backups.c @@ -72,6 +72,9 @@ EXPORTED void fatal(const char *error, int code) { fprintf(stderr, "fatal error: %s\n", error); cyrus_done(); + + if (code != EX_PROTOCOL && config_fatals_abort) abort(); + exit(code); } diff --git a/backup/cyr_backup.c b/backup/cyr_backup.c index 7139bcad4b..f4f2672c4f 100644 --- a/backup/cyr_backup.c +++ b/backup/cyr_backup.c @@ -75,6 +75,9 @@ EXPORTED void fatal(const char *error, int code) { fprintf(stderr, "fatal error: %s\n", error); cyrus_done(); + + if (code != EX_PROTOCOL && config_fatals_abort) abort(); + exit(code); } diff --git a/backup/restore.c b/backup/restore.c index 51161c4b15..cdd40c3dd0 100644 --- a/backup/restore.c +++ b/backup/restore.c @@ -68,6 +68,9 @@ EXPORTED void fatal(const char *s, int code) { fprintf(stderr, "Fatal error: %s\n", s); syslog(LOG_ERR, "Fatal error: %s", s); + + if (code != EX_PROTOCOL && config_fatals_abort) abort(); + exit(code); } diff --git a/imap/cli_fatal.c b/imap/cli_fatal.c index 2442333a0c..9d655662ad 100644 --- a/imap/cli_fatal.c +++ b/imap/cli_fatal.c @@ -44,6 +44,7 @@ #include #include +#include #include "global.h" #include "xmalloc.h" @@ -60,5 +61,8 @@ EXPORTED void fatal(const char *message, int code) recurse_code = code; fprintf(stderr, "fatal error: %s\n", message); cyrus_done(); + + if (code != EX_PROTOCOL && config_fatals_abort) abort(); + exit(code); } diff --git a/imap/ctl_conversationsdb.c b/imap/ctl_conversationsdb.c index 61c109afd2..2bb881739a 100644 --- a/imap/ctl_conversationsdb.c +++ b/imap/ctl_conversationsdb.c @@ -1038,6 +1038,8 @@ EXPORTED void fatal(const char* s, int code) { fprintf(stderr, "ctl_conversationsdb: %s\n", s); cyrus_done(); + + if (code != EX_PROTOCOL && config_fatals_abort) abort(); + exit(code); } - diff --git a/imap/cvt_xlist_specialuse.c b/imap/cvt_xlist_specialuse.c index 91666fa268..ca534461e6 100644 --- a/imap/cvt_xlist_specialuse.c +++ b/imap/cvt_xlist_specialuse.c @@ -87,6 +87,9 @@ EXPORTED void fatal(const char *s, int code) { fprintf(stderr, "Fatal error: %s\n", s); syslog(LOG_ERR, "Fatal error: %s", s); + + if (code != EX_PROTOCOL && config_fatals_abort) abort(); + exit(code); } diff --git a/imap/deliver.c b/imap/deliver.c index 9fb66766d0..d3b32cec50 100644 --- a/imap/deliver.c +++ b/imap/deliver.c @@ -127,6 +127,9 @@ EXPORTED void fatal(const char* s, int code) prot_printf(deliver_out,"421 4.3.0 deliver: %s\r\n", s); prot_flush(deliver_out); cyrus_done(); + + if (code != EX_PROTOCOL && config_fatals_abort) abort(); + exit(code); } diff --git a/imap/fud.c b/imap/fud.c index 41e559361d..2d3489f4c5 100644 --- a/imap/fud.c +++ b/imap/fud.c @@ -481,6 +481,7 @@ EXPORTED void fatal(const char* s, int code) recurse_code = code; syslog(LOG_ERR, "Fatal error: %s", s); + if (code != EX_PROTOCOL && config_fatals_abort) abort(); + shut_down(code); } - diff --git a/imap/message_test.c b/imap/message_test.c index e6b7573161..38847e2584 100644 --- a/imap/message_test.c +++ b/imap/message_test.c @@ -311,6 +311,9 @@ EXPORTED void fatal(const char* s, int code) { fprintf(stderr, "message_test: %s\n", s); cyrus_done(); + + if (code != EX_PROTOCOL && config_fatals_abort) abort(); + exit(code); } diff --git a/imap/mupdate.c b/imap/mupdate.c index 6466200a64..cceec732d6 100644 --- a/imap/mupdate.c +++ b/imap/mupdate.c @@ -617,10 +617,10 @@ EXPORTED void fatal(const char *s, int code) else recurse_code = code; syslog(LOG_ERR, "%s", s); - shut_down(code); - /* NOTREACHED */ - exit(code); /* shut up GCC */ + if (code != EX_PROTOCOL && config_fatals_abort) abort(); + + shut_down(code); } #define CHECKNEWLINE(c, ch) do { if ((ch) == '\r') (ch)=prot_getc((c)->pin); \ diff --git a/imap/nntpd.c b/imap/nntpd.c index 7166b52870..eab9d93f19 100644 --- a/imap/nntpd.c +++ b/imap/nntpd.c @@ -646,6 +646,9 @@ EXPORTED void fatal(const char* s, int code) } if (stage) append_removestage(stage); syslog(LOG_ERR, "Fatal error: %s", s); + + if (code != EX_PROTOCOL && config_fatals_abort) abort(); + shut_down(code); } diff --git a/imap/search_test.c b/imap/search_test.c index 8b4fb0b84a..e0c6d200a3 100644 --- a/imap/search_test.c +++ b/imap/search_test.c @@ -332,6 +332,9 @@ EXPORTED void fatal(const char* s, int code) { fprintf(stderr, "search_test: %s\n", s); cyrus_done(); + + if (code != EX_PROTOCOL && config_fatals_abort) abort(); + exit(code); } diff --git a/imap/smmapd.c b/imap/smmapd.c index e6d4e6468f..a983bce686 100644 --- a/imap/smmapd.c +++ b/imap/smmapd.c @@ -166,7 +166,8 @@ EXPORTED void fatal(const char* s, int code) } recurse_code = code; syslog(LOG_ERR, "Fatal error: %s", s); - abort(); + + if (code != EX_PROTOCOL && config_fatals_abort) abort(); shut_down(code); } diff --git a/imap/sync_reset.c b/imap/sync_reset.c index 42f67a804e..cc7afa4fc0 100644 --- a/imap/sync_reset.c +++ b/imap/sync_reset.c @@ -116,6 +116,9 @@ static int usage(const char *name) EXPORTED void fatal(const char* s, int code) { fprintf(stderr, "sync_reset: %s\n", s); + + if (code != EX_PROTOCOL && config_fatals_abort) abort(); + exit(code); } diff --git a/netnews/remotepurge.c b/netnews/remotepurge.c index f4837ad2df..f1b1a9aeab 100644 --- a/netnews/remotepurge.c +++ b/netnews/remotepurge.c @@ -147,6 +147,9 @@ EXPORTED void fatal(const char *s, int code) syslog(LOG_ERR, "fatal error: %s", s); fprintf(stderr, "fatal error: %s\n", s); } + + if (code != EX_PROTOCOL && config_fatals_abort) abort(); + exit(code); } diff --git a/notifyd/notifyd.c b/notifyd/notifyd.c index bd30711f1a..1ddaf8e9ed 100644 --- a/notifyd/notifyd.c +++ b/notifyd/notifyd.c @@ -214,6 +214,8 @@ EXPORTED void fatal(const char *s, int code) syslog(LOG_ERR, "Fatal error: %s", s); + if (code != EX_PROTOCOL && config_fatals_abort) abort(); + shut_down(code); } diff --git a/ptclient/ptloader.c b/ptclient/ptloader.c index 9ebc897a2a..a8c15a8020 100644 --- a/ptclient/ptloader.c +++ b/ptclient/ptloader.c @@ -371,6 +371,8 @@ int service_main_fd(int c, int argc __attribute__((unused)), EXPORTED void fatal(const char *msg, int exitcode) { syslog(LOG_ERR, "%s", msg); + + if (exitcode != EX_PROTOCOL && config_fatals_abort) abort(); + exit(exitcode); } - diff --git a/sieve/sievec.c b/sieve/sievec.c index abecaa1601..27179d73f7 100644 --- a/sieve/sievec.c +++ b/sieve/sievec.c @@ -47,6 +47,7 @@ #include "sieve_interface.h" #include +#include #include "libconfig.h" #include "xmalloc.h" @@ -160,5 +161,7 @@ EXPORTED void fatal(const char *s, int code) { fprintf(stderr, "Fatal error: %s (%d)\r\n", s, code); - exit(1); + if (code != EX_PROTOCOL && config_fatals_abort) abort(); + + exit(code); } diff --git a/sieve/test.c b/sieve/test.c index 13330984ed..e960309ca7 100644 --- a/sieve/test.c +++ b/sieve/test.c @@ -58,6 +58,7 @@ #include #include #include +#include #include "libconfig.h" #include "assert.h" @@ -880,5 +881,8 @@ int main(int argc, char *argv[]) EXPORTED void fatal(const char* message, int rc) { fprintf(stderr, "fatal error: %s\n", message); + + if (rc != EX_PROTOCOL && config_fatals_abort) abort(); + exit(rc); } diff --git a/sieve/test_mailbox.c b/sieve/test_mailbox.c index 13af1ecb13..cb0372ce80 100644 --- a/sieve/test_mailbox.c +++ b/sieve/test_mailbox.c @@ -786,5 +786,8 @@ int main(int argc, char *argv[]) EXPORTED void fatal(const char* message, int rc) { fprintf(stderr, "fatal error: %s\n", message); + + if (rc != EX_PROTOCOL && config_fatals_abort) abort(); + exit(rc); } diff --git a/timsieved/timsieved.c b/timsieved/timsieved.c index 9cd0a4650f..c3f6802abd 100644 --- a/timsieved/timsieved.c +++ b/timsieved/timsieved.c @@ -198,6 +198,8 @@ EXPORTED void fatal(const char *s, int code) prot_printf(sieved_out, "NO Fatal error: %s\r\n", s); prot_flush(sieved_out); + if (code != EX_PROTOCOL && config_fatals_abort) abort(); + shut_down(EX_TEMPFAIL); } From 22efff1aa2aa48a71350edd9f926ddff9e109f9a Mon Sep 17 00:00:00 2001 From: Robert Stepanek Date: Tue, 30 Jul 2024 15:24:59 +0200 Subject: [PATCH 032/501] message.c: fix reading cached bodystructure for "message/global" Fixes a bug in which cached bodystructures having a cache version less than the current version 13 failed to parse. --- imap/message.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/imap/message.c b/imap/message.c index 42aae25ef1..2bbad2556e 100644 --- a/imap/message.c +++ b/imap/message.c @@ -4425,7 +4425,8 @@ static int parse_mime_params(struct protstream *prot, struct param **prev) return EOF; } -static int parse_bodystructure_part(struct protstream *prot, struct body *body, const char *part_id) +static int parse_bodystructure_part(struct protstream *prot, struct body *body, + const char *part_id, uint16_t cache_version) { int c; int r = 0; @@ -4453,7 +4454,7 @@ static int parse_bodystructure_part(struct protstream *prot, struct body *body, buf_printf(&buf, "%d", body->numparts); char *part_id = buf_release(&buf); struct body *subbody = &body->subpart[body->numparts-1]; - r = parse_bodystructure_part(prot, subbody, part_id); + r = parse_bodystructure_part(prot, subbody, part_id, cache_version); subbody->part_id = part_id; if (r) goto out; @@ -4511,7 +4512,11 @@ static int parse_bodystructure_part(struct protstream *prot, struct body *body, body->content_lines = atoi(buf_cstring(&buf)); } - else if (body_is_rfc822(body)) { + else if ((body_is_rfc822(body) && cache_version >= 13) || + // Cache versions < 13 only handled message/rfc822. + (!strcasecmp(body->type, "MESSAGE") && + !strcasecmp(body->subtype, "RFC822"))) { + body->numparts = 1; body->subpart = xzmalloc(sizeof(struct body)); @@ -4520,7 +4525,7 @@ static int parse_bodystructure_part(struct protstream *prot, struct body *body, if (r) goto out; /* process body */ - r = parse_bodystructure_part(prot, body->subpart, part_id); + r = parse_bodystructure_part(prot, body->subpart, part_id, cache_version); if (r) goto out; /* skip trailing space (parse_bs_part doesn't eat it) */ @@ -4584,7 +4589,10 @@ static int parse_bodystructure_sections(const char **cachestrp, const char *cach goto done; } - if (body_is_rfc822(body)) { + if ((body_is_rfc822(body) && cache_version >= 13) || + // Cache versions < 13 only handled message/rfc822. + (!strcasecmp(body->type, "MESSAGE") && + !strcasecmp(body->subtype, "RFC822"))) { if (strcmp(body->subpart->type, "MULTIPART") == 0) { @@ -4820,7 +4828,7 @@ static int message_parse_cbodystructure(message_t *m) return IMAP_MAILBOX_BADFORMAT; m->body = xzmalloc(sizeof(struct body)); - r = parse_bodystructure_part(prot, m->body, NULL); + r = parse_bodystructure_part(prot, m->body, NULL, m->record.cache_version); if (r) { xsyslog(LOG_ERR, "IOERROR: error parsing body structure", "mailbox=<%s> record_uid=<%u>, cacheitem=<%.*s>", From 5df641d8b1840fb28d979a78e4606afba40ed436 Mon Sep 17 00:00:00 2001 From: Robert Stepanek Date: Wed, 31 Jul 2024 09:21:08 +0200 Subject: [PATCH 033/501] Revert "jmap_contact.c: support X-PHONETIC extension properties" This reverts commit 5843077048ec24738cc896ed536496b73f2fb1c3. e353858b61b248c560f39ad5cc8bc57b161aea2b introduced a more general solution to preserve x-properties. --- .../JMAPContacts/contact_copy_phonetic | 104 ---------------- .../JMAPContacts/contact_copy_preserve_xprops | 4 + .../JMAPContacts/contact_get_phonetic | 50 -------- .../JMAPContacts/contact_set_phonetic | 116 ------------------ imap/jmap_contact.c | 67 ---------- 5 files changed, 4 insertions(+), 337 deletions(-) delete mode 100644 cassandane/tiny-tests/JMAPContacts/contact_copy_phonetic delete mode 100644 cassandane/tiny-tests/JMAPContacts/contact_get_phonetic delete mode 100644 cassandane/tiny-tests/JMAPContacts/contact_set_phonetic diff --git a/cassandane/tiny-tests/JMAPContacts/contact_copy_phonetic b/cassandane/tiny-tests/JMAPContacts/contact_copy_phonetic deleted file mode 100644 index d449cdd928..0000000000 --- a/cassandane/tiny-tests/JMAPContacts/contact_copy_phonetic +++ /dev/null @@ -1,104 +0,0 @@ -#!perl -use Cassandane::Tiny; - -sub test_contact_copy_phonetic - : needs_component_jmap { - my ($self) = @_; - my $jmap = $self->{jmap}; - my $carddav = $self->{carddav}; - my $admintalk = $self->{adminstore}->get_client(); - my $service = $self->{instance}->get_service("http"); - - xlog $self, "create shared account"; - $admintalk->create("user.other"); - - my $otherCarddav = Net::CardDAVTalk->new( - user => "other", - password => 'pass', - host => $service->host(), - port => $service->port(), - scheme => 'http', - url => '/', - expandurl => 1, - ); - - my $otherJmap = Mail::JMAPTalk->new( - user => 'other', - password => 'pass', - host => $service->host(), - port => $service->port(), - scheme => 'http', - url => '/jmap/', - ); - $otherJmap->DefaultUsing([ - 'urn:ietf:params:jmap:core', - 'https://cyrusimap.org/ns/jmap/contacts', - 'https://cyrusimap.org/ns/jmap/debug' - ]); - - xlog $self, "share addressbook"; - $admintalk->setacl( - "user.other.#addressbooks.Default", - "cassandane" => 'lrswipkxtecdn' - ) or die; - - my $card = decode( - 'utf-8', <Request('PUT', 'Default/test.vcf', $card, 'Content-Type' => 'text/vcard'); - - my $res = $jmap->CallMethods([ - [ 'Contact/query', {}, 'R1' ], - ]); - $self->assert_num_equals(1, scalar @{ $res->[0][1]{ids} }); - my $contactId = $res->[0][1]{ids}[0]; - $self->assert_not_null($contactId); - - $res = $jmap->CallMethods([ - [ - 'Contact/copy', - { - fromAccountId => 'cassandane', - accountId => 'other', - create => { - contact1 => { - addressbookId => 'Default', - id => $contactId - } - }, - onSuccessDestroyOriginal => JSON::false, - }, - 'R1' - ], - ]); - my $copiedContactId = $res->[0][1]{created}{contact1}{id}; - $self->assert_not_null($copiedContactId); - - $res = $otherJmap->CallMethods([ - [ - 'Contact/get', - { - accountId => 'other', - ids => [$copiedContactId], - properties => [ 'phoneticFirstName', 'phoneticMiddleName', 'phoneticLastName' ], - }, - 'R1' - ], - ]); - - $self->assert_str_equals(decode('utf-8', '/hæŋk/'), $res->[0][1]{list}[0]{phoneticFirstName}); - $self->assert_str_equals(decode('utf-8', '/fræŋk/'), $res->[0][1]{list}[0]{phoneticMiddleName}); - $self->assert_str_equals(decode('utf-8', '/smɪθ/'), $res->[0][1]{list}[0]{phoneticLastName}); -} diff --git a/cassandane/tiny-tests/JMAPContacts/contact_copy_preserve_xprops b/cassandane/tiny-tests/JMAPContacts/contact_copy_preserve_xprops index e3a086ca2a..3b4ce1ad62 100644 --- a/cassandane/tiny-tests/JMAPContacts/contact_copy_preserve_xprops +++ b/cassandane/tiny-tests/JMAPContacts/contact_copy_preserve_xprops @@ -50,6 +50,8 @@ UID:0A8F88DE-1073-4D47-926F-0D535523FD15 N:Smith;Hank;; FN:Hank Smith X-FOO;X-BAZ=Bam:Bar +X-PHONETIC-FIRST-NAME:phoneticFirst +X-PHONETIC-LAST-NAME:phoneticLast REV:2008-04-24T19:52:43Z END:VCARD EOF @@ -98,4 +100,6 @@ EOF $card = $otherCarddav->Request('GET', $res->[0][1]{list}[0]{'x-href'}); $self->assert_matches(qr/^X-FOO;X-BAZ=Bam:Bar\r$/m, $card->{content}); + $self->assert_matches(qr/^X-PHONETIC-FIRST-NAME:phoneticFirst\r$/m, $card->{content}); + $self->assert_matches(qr/^X-PHONETIC-LAST-NAME:phoneticLast\r$/m, $card->{content}); } diff --git a/cassandane/tiny-tests/JMAPContacts/contact_get_phonetic b/cassandane/tiny-tests/JMAPContacts/contact_get_phonetic deleted file mode 100644 index 3e93a4f546..0000000000 --- a/cassandane/tiny-tests/JMAPContacts/contact_get_phonetic +++ /dev/null @@ -1,50 +0,0 @@ -#!perl -use Cassandane::Tiny; - -sub test_contact_get_phonetic - : needs_component_jmap { - my ($self) = @_; - my $jmap = $self->{jmap}; - my $carddav = $self->{carddav}; - - my $card = decode('utf-8', <Request( - 'PUT', 'Default/test.vcf', $card, - 'Content-Type' => 'text/vcard; charset=utf-8' - ); - - my %vcardProps = ( - firstName => decode('utf-8', '一朗'), - lastName => decode('utf-8', '鈴木'), - company => decode('utf-8', '日本野球機構'), - phoneticFirstName => decode('utf-8', 'いちろう'), - phoneticLastName => decode('utf-8', 'すずき'), - phoneticCompany => decode('utf-8', 'にっぽんやきゅうきこう'), - ); - - my $res = $jmap->CallMethods([ [ - 'Contact/get', - { - properties => [ keys %vcardProps ], - }, - 'R1' - ] ]); - - keys %vcardProps; - while (my ($prop, $value) = each %vcardProps) { - $self->assert_str_equals($value, $res->[0][1]{list}[0]{$prop}); - } -} diff --git a/cassandane/tiny-tests/JMAPContacts/contact_set_phonetic b/cassandane/tiny-tests/JMAPContacts/contact_set_phonetic deleted file mode 100644 index abc864c0ae..0000000000 --- a/cassandane/tiny-tests/JMAPContacts/contact_set_phonetic +++ /dev/null @@ -1,116 +0,0 @@ -#!perl -use Cassandane::Tiny; -use Encode qw(decode); - -sub test_contact_set_phonetic - : needs_component_jmap { - my ($self) = @_; - my $jmap = $self->{jmap}; - my $carddav = $self->{carddav}; - - xlog $self, "Create Contact with phonetic properties"; - - my %jprops = ( - firstName => decode('utf-8', '一朗'), - lastName => decode('utf-8', '鈴木'), - company => decode('utf-8', '日本野球機構'), - phoneticFirstName => decode('utf-8', 'いちろう'), - phoneticLastName => decode('utf-8', 'すずき'), - phoneticCompany => decode('utf-8', 'にっぽんやきゅうきこう'), - ); - - my $res = $jmap->CallMethods([ - [ - 'Contact/set', - { - create => { - contact1 => \%jprops, - }, - }, - 'R1' - ], - [ - 'Contact/get', { - ids => ['#contact1'], properties => [ keys %jprops, "x-href" ] - }, 'R2' - ] - ]); - my $contactId = $res->[0][1]{created}{contact1}{id}; - $self->assert_not_null($contactId); - my $xhref = $res->[1][1]{list}[0]{'x-href'}; - $self->assert_not_null($xhref); - - xlog $self, "Assert JMAP and vCard properties"; - - keys %jprops; - while (my ($prop, $value) = each %jprops) { - $self->assert_str_equals($value, $res->[1][1]{list}[0]{$prop}); - } - - my %vcardProps = ( - N => decode('utf-8', '鈴木;一朗;'), - ORG => decode('utf-8', '日本野球機構'), - 'X-PHONETIC-FIRST-NAME' => decode('utf-8', 'いちろう'), - 'X-PHONETIC-LAST-NAME' => decode('utf-8', 'すずき'), - 'X-PHONETIC-ORG' => decode('utf-8', 'にっぽんやきゅうきこう'), - ); - - $res = $carddav->Request('GET', $xhref); - my $vcard = decode('utf-8', $res->{content}); - $self->assert_not_null($vcard); - - keys %vcardProps; - while (my ($prop, $value) = each %vcardProps) { - $self->assert_matches(qr/^$prop:$value\r?$/m, $vcard); - } - - xlog $self, "Update phonetic properties"; - - $jprops{company} = 'Acme Corporation'; - $jprops{phoneticCompany} = decode('utf-8', '/ˈæk.mi/ /ˌkɔːr.pəˈreɪ.ʃən/'); - delete $jprops{phoneticLastName}; - - $vcardProps{ORG} = $jprops{company}; - $vcardProps{'X-PHONETIC-ORG'} = $jprops{phoneticCompany}; - delete $vcardProps{'X-PHONETIC-LAST-NAME'}; - - my $res = $jmap->CallMethods([ - [ - 'Contact/set', - { - update => { - $contactId => { - company => $jprops{company}, - phoneticCompany => $jprops{phoneticCompany}, - phoneticLastName => undef, - } - } - }, - 'R1' - ], - [ - 'Contact/get', - { - ids => ['#contact1'], properties => [ keys %jprops ], - }, - 'R2' - ] - ]); - $self->assert(exists $res->[0][1]{updated}{$contactId}); - - xlog $self, "Assert JMAP and vCard properties"; - - keys %jprops; - while (my ($prop, $value) = each %jprops) { - $self->assert_str_equals($value, $res->[1][1]{list}[0]{$prop}); - } - - $res = $carddav->Request('GET', $xhref); - my $vcard = decode('utf-8', $res->{content}); - $self->assert_not_null($vcard); - - keys %vcardProps; - while (my ($prop, $value) = each %vcardProps) { - $self->assert_matches(qr/^$prop:$value\r?$/m, $vcard); - } -} diff --git a/imap/jmap_contact.c b/imap/jmap_contact.c index 12f5d95d5f..9fcbfc37e8 100644 --- a/imap/jmap_contact.c +++ b/imap/jmap_contact.c @@ -630,26 +630,6 @@ static const jmap_property_t contact_props[] = { NULL, 0 }, - { - "phoneticFirstName", - NULL, - 0 - }, - { - "phoneticMiddleName", - NULL, - 0 - }, - { - "phoneticLastName", - NULL, - 0 - }, - { - "phoneticCompany", - NULL, - 0 - }, { "vCardProps", NULL, @@ -1842,30 +1822,6 @@ static json_t *jmap_contact_from_vcard(const char *userid, json_object_set_new(item, "value", jmap_utf8string(entry->v.value)); json_array_append_new(online, item); } - else if (!strncasecmp(entry->name, "x-phonetic-", 11)) { - /* Apple supports phonetic properties for names and company */ - const char *pname = entry->name + 11; - if (!strcasecmp(pname, "first-name")) { - json_object_set_new(obj, "phoneticFirstName", - jmap_utf8string(entry->v.value)); - } - if (!strcasecmp(pname, "middle-name")) { - json_object_set_new(obj, "phoneticMiddleName", - jmap_utf8string(entry->v.value)); - } - else if (!strcasecmp(pname, "last-name")) { - json_object_set_new(obj, "phoneticLastName", - jmap_utf8string(entry->v.value)); - } - else if (!strcasecmp(pname, "org")) { - // Apple Contacts.app only supports setting - // a phonetic company, not a department. - // So despite this being an ORG property - // it is single-valued in practice. - json_object_set_new(obj, "phoneticCompany", - jmap_utf8string(entry->v.value)); - } - } else if (!strncasecmp(entry->name, "x-", 2)) { // Preserve unknown x-properties as RFC 7095 jCard. json_t *vcardprops = json_object_get(obj, "vCardProps"); @@ -4084,29 +4040,6 @@ static int _json_to_card(struct jmap_req *req, } record_is_dirty = 1; } - else if (!strncmp(key, "phonetic", 8)) { - const char *pname = NULL; - if (!strcmp(key + 8, "FirstName")) - pname = "X-PHONETIC-FIRST-NAME"; - else if (!strcmp(key + 8, "MiddleName")) - pname = "X-PHONETIC-MIDDLE-NAME"; - else if (!strcmp(key + 8, "LastName")) - pname = "X-PHONETIC-LAST-NAME"; - else if (!strcmp(key + 8, "Company")) - pname = "X-PHONETIC-ORG"; - - if (json_is_string(jval) && pname) { - vparse_delete_entries(card, NULL, pname); - buf_setcstr(&buf, json_string_value(jval)); - buf_trim(&buf); - if (buf_len(&buf)) { - vparse_add_entry(card, NULL, pname, buf_cstring(&buf)); - } - } - else if (!pname || JNOTNULL(jval)) { - json_array_append_new(invalid, json_string(key)); - } - } else if (!strcmp(key, "vCardProps")) { if (json_is_array(jval)) { size_t i; From 4bffb39cf0bdf58f6e75c12c4bbf05e542051dd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=B8=D0=BB=D1=8F=D0=BD=20=D0=9F=D0=B0=D0=BB=D0=B0?= =?UTF-8?q?=D1=83=D0=B7=D0=BE=D0=B2?= Date: Sat, 3 Aug 2024 14:05:15 +0200 Subject: [PATCH 034/501] =?UTF-8?q?docsrc/:=20Typos=20i=20=E2=86=92=20I?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- doc/legacy/changes.html | 4 ++-- docsrc/imap/download/release-notes/2.0/2.0.x.rst | 4 ++-- docsrc/imap/reference/manpages/systemcommands/mkimap.rst | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/legacy/changes.html b/doc/legacy/changes.html index 5d86b22626..70d094367c 100644 --- a/doc/legacy/changes.html +++ b/doc/legacy/changes.html @@ -2078,7 +2078,7 @@

Changes to the Cyrus IMAP Server since 2.0.6

names in cyrus.conf. also logs even more verbosely (see bug #115.) -
  • libwrap_init() is now inside the loop, since i don't quite +
  • libwrap_init() is now inside the loop, since I don't quite understand the semantics of libwrap calls.
  • setquota in cyradm now behaves more sanely (and gives correct @@ -2089,7 +2089,7 @@

    Changes to the Cyrus IMAP Server since 2.0.6

  • small fixes in timsieved.
    • -
  • added a "make dist" target so i won't dread releases as +
  • added a "make dist" target so I won't dread releases as much.
    • diff --git a/docsrc/imap/download/release-notes/2.0/2.0.x.rst b/docsrc/imap/download/release-notes/2.0/2.0.x.rst index afaf43ff10..1fec26b7cd 100644 --- a/docsrc/imap/download/release-notes/2.0/2.0.x.rst +++ b/docsrc/imap/download/release-notes/2.0/2.0.x.rst @@ -123,11 +123,11 @@ Changes to the Cyrus IMAP Server since 2.0.6 * off-by-one bug in seen_db fixed. * starting/committing/aborting transactions now logged more correctly in cyrsudb_db3 * master will now accept port numbers instead of just service names in cyrus.conf. also logs even more verbosely (see bug #115.) -* libwrap_init() is now inside the loop, since i don't quite understand the semantics of libwrap calls. +* libwrap_init() is now inside the loop, since I don't quite understand the semantics of libwrap calls. * setquota in cyradm now behaves more sanely (and gives correct usage message). * bugfixes to the managesieve client perl api. (still needs work.) * small fixes in timsieved. -* added a "make dist" target so i won't dread releases as much. +* added a "make dist" target so I won't dread releases as much. Changes to the Cyrus IMAP Server since 2.0.5 diff --git a/docsrc/imap/reference/manpages/systemcommands/mkimap.rst b/docsrc/imap/reference/manpages/systemcommands/mkimap.rst index ccb6031ebb..a2ffea76b9 100644 --- a/docsrc/imap/reference/manpages/systemcommands/mkimap.rst +++ b/docsrc/imap/reference/manpages/systemcommands/mkimap.rst @@ -66,9 +66,9 @@ Examples :: reading configure file /etc/imapd.conf... - i will configure directory /var/lib/imap. - i saw partition /var/spool/cyrus/mail. - i saw partition /var/spool/cyrus/news. + I will configure directory /var/lib/imap. + I saw partition /var/spool/cyrus/mail. + I saw partition /var/spool/cyrus/news. done configuring /var/lib/imap... creating /var/spool/cyrus/mail... From 551b5e5c42451a731c03ebd2b2626c2572aeb057 Mon Sep 17 00:00:00 2001 From: Siva Mahadevan Date: Tue, 6 Aug 2024 16:43:08 -0400 Subject: [PATCH 035/501] imap/telemetry: log rusage at LOG_INFO The syslog RFC 5424 states that LOG_NOTICE should be used for "normal but significant conditions". USAGE logs aren't significant conditions on the system since they aren't actually conditions at all; they are a way to view numerical metrics on resource usage on a particular action. --- imap/telemetry.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imap/telemetry.c b/imap/telemetry.c index 89570508f9..b5c6fe7f8d 100644 --- a/imap/telemetry.c +++ b/imap/telemetry.c @@ -136,7 +136,7 @@ EXPORTED void telemetry_rusage(char *userid) * Some systems provide significantly more data, but POSIX * guarantees user & sys CPU time. */ - syslog(LOG_NOTICE, "USAGE %s user: " TIME_T_FMT ".%.6d sys: " TIME_T_FMT ".%.6d", userid, + syslog(LOG_INFO, "USAGE %s user: " TIME_T_FMT ".%.6d sys: " TIME_T_FMT ".%.6d", userid, user.tv_sec, (int)user.tv_usec, sys.tv_sec, (int)sys.tv_usec); From 6635f402a42ac844f3a7e35e7b676e0a796eeeaa Mon Sep 17 00:00:00 2001 From: ellie timoney Date: Mon, 22 Jul 2024 14:05:34 +1000 Subject: [PATCH 036/501] prometheus: disentangle service and usage reports --- cassandane/Cassandane/Cyrus/Prometheus.pm | 8 +-- imap/prometheus.c | 65 ++++++++++--------- imap/prometheus.h | 3 +- imap/promstatsd.c | 77 +++++++++++++++-------- 4 files changed, 88 insertions(+), 65 deletions(-) diff --git a/cassandane/Cassandane/Cyrus/Prometheus.pm b/cassandane/Cassandane/Cyrus/Prometheus.pm index 4cb3456f33..f9bbb35021 100644 --- a/cassandane/Cassandane/Cyrus/Prometheus.pm +++ b/cassandane/Cassandane/Cyrus/Prometheus.pm @@ -133,7 +133,7 @@ sub test_aaasetup $self->assert(1); } -sub test_reportfile_exists +sub test_service_reportfile_exists :min_version_3_1 :needs_component_httpd { my ($self) = @_; @@ -144,11 +144,11 @@ sub test_reportfile_exists # and wait for a fresh report sleep 3; - my $reportfile_name = "$self->{instance}->{basedir}/conf/stats/report.txt"; + my $fname = "$self->{instance}->{basedir}/conf/stats/service.txt"; - $self->assert_file_test($reportfile_name, '-f'); + $self->assert_file_test($fname, '-f'); - my $report = parse_report(scalar read_file $reportfile_name); + my $report = parse_report(scalar read_file $fname); $self->assert(scalar keys %{$report}); $self->assert(exists $report->{cyrus_imap_connections_total}); diff --git a/imap/prometheus.c b/imap/prometheus.c index 519cca859b..7b4b633d12 100644 --- a/imap/prometheus.c +++ b/imap/prometheus.c @@ -319,51 +319,50 @@ EXPORTED void prometheus_apply_delta(enum prom_metric_id metric_id, EXPORTED int prometheus_text_report(struct buf *buf, const char **mimetype) { - char *report_fname = NULL; - struct mappedfile *mf = NULL; - int r; + const struct { + int required; + char *fname; + } reports[] = { + { 1, FNAME_PROM_SERVICE_REPORT }, + { 0, FNAME_PROM_USAGE_REPORT }, + { 0, FNAME_PROM_MASTER_REPORT }, + }; + const size_t n_reports = sizeof(reports) / sizeof(reports[0]); + unsigned i; + int r = 0; if (!prometheus_enabled) return IMAP_INTERNAL; - report_fname = strconcat(prometheus_stats_dir(), FNAME_PROM_REPORT, NULL); - - r = mappedfile_open(&mf, report_fname, 0); - if (r) { - free(report_fname); - return r; - } + buf_reset(buf); - r = mappedfile_readlock(mf); - if (!r) { - buf_setmap(buf, mappedfile_base(mf), mappedfile_size(mf)); - if (mimetype) - *mimetype = "text/plain; version=0.0.4"; - } + for (i = 0; i < n_reports; i++) { + char *report_fname = NULL; + struct mappedfile *mf = NULL; - mappedfile_unlock(mf); - mappedfile_close(&mf); - free(report_fname); + report_fname = strconcat(prometheus_stats_dir(), + reports[i].fname, + NULL); - if (r) return r; + r = mappedfile_open(&mf, report_fname, 0); + if (r && reports[i].required) { + free(report_fname); + return r; + } - report_fname = strconcat(prometheus_stats_dir(), FNAME_PROM_MASTER_REPORT, NULL); + r = mappedfile_readlock(mf); + if (!r) { + buf_appendmap(buf, mappedfile_base(mf), mappedfile_size(mf)); + } - r = mappedfile_open(&mf, report_fname, 0); - if (r) { + mappedfile_unlock(mf); + mappedfile_close(&mf); free(report_fname); - return 0; /* no master.txt yet? no worries */ - } - - r = mappedfile_readlock(mf); - if (!r) { - buf_appendmap(buf, mappedfile_base(mf), mappedfile_size(mf)); } - mappedfile_unlock(mf); - mappedfile_close(&mf); - free(report_fname); + if (!r && mimetype) + *mimetype = "text/plain; version=0.0.4"; - return 0; + return r; } EXPORTED enum prom_metric_id prometheus_lookup_label(enum prom_labelled_metric metric, diff --git a/imap/prometheus.h b/imap/prometheus.h index ce566ca3a8..56f19d6a70 100644 --- a/imap/prometheus.h +++ b/imap/prometheus.h @@ -52,8 +52,9 @@ #include "imap/promdata.h" -#define FNAME_PROM_REPORT "report.txt" +#define FNAME_PROM_SERVICE_REPORT "service.txt" #define FNAME_PROM_MASTER_REPORT "master.txt" +#define FNAME_PROM_USAGE_REPORT "usage.txt" #define FNAME_PROM_DONEPROCS "doneprocs" #define FNAME_PROM_STATS_DIR "/stats" diff --git a/imap/promstatsd.c b/imap/promstatsd.c index 0a83a7f929..9b2b8037d9 100644 --- a/imap/promstatsd.c +++ b/imap/promstatsd.c @@ -69,15 +69,33 @@ #include "imap/prometheus.h" #include "imap/quota.h" +static void do_collate_service_report(struct buf *buf); +static void do_collate_usage_report(struct buf *buf); + /* globals so that shut_down() can clean up */ -static struct buf report_buf = BUF_INITIALIZER; -static struct mappedfile *report_file = NULL; +static struct { + const char *fname; + const char *desc; + void (*collate_fn)(struct buf *); + struct mappedfile *mf; + struct buf buf; +} reports[] = { + { FNAME_PROM_SERVICE_REPORT, "service", &do_collate_service_report, + NULL, BUF_INITIALIZER }, + { FNAME_PROM_USAGE_REPORT, "usage", &do_collate_usage_report, + NULL, BUF_INITIALIZER }, +}; +const size_t n_reports = sizeof(reports) / sizeof(reports[0]); static void shut_down(int ec) __attribute__((noreturn)); static void shut_down(int ec) { - mappedfile_close(&report_file); - buf_free(&report_buf); + unsigned i; + + for (i = 0; i < n_reports; i++) { + mappedfile_close(&reports[i].mf); + buf_free(&reports[i].buf); + } cyrus_done(); exit(ec); } @@ -145,7 +163,9 @@ enum promdir_foreach_mode { PROMDIR_FOREACH_DONEPROCS, }; typedef int promdir_foreach_cb(const struct prom_stats *stats, void *rock); -static int promdir_foreach(promdir_foreach_cb *proc, enum promdir_foreach_mode mode, void *rock) +static int promdir_foreach(promdir_foreach_cb *proc, + enum promdir_foreach_mode mode, + void *rock) { const char *basedir; DIR *dh; @@ -242,7 +262,7 @@ static void format_metric(const char *key __attribute__((unused)), stats->metrics[fmrock->metric].last_updated); } -static void do_collate_report(struct buf *buf) +static void do_collate_service_report(struct buf *buf) { hash_table all_stats = HASH_TABLE_INITIALIZER; char *doneprocs_lock_fname; @@ -552,13 +572,14 @@ static void format_usage_quota_commitment(struct buf *buf, } } -static void do_collate_usage(struct buf *buf) +static void do_collate_usage_report(struct buf *buf) { hash_table h = HASH_TABLE_INITIALIZER; strarray_t *partition_names = NULL; int r; int64_t starttime; + buf_reset(buf); construct_hash_table(&h, 10, 0); /* 10 partitions is probably enough right */ starttime = now_ms(); @@ -629,8 +650,6 @@ int main(int argc, char **argv) const char *alt_config = NULL; int call_debugger = 0; - char *report_fname = NULL; - struct mappedfile *report_file = NULL; const char *p; int cleanup = 0; int debugmode = 0; @@ -639,6 +658,7 @@ int main(int argc, char **argv) int oneshot = 0; int opt; int r; + unsigned i; p = getenv("CYRUS_VERBOSE"); if (p) verbose = atoi(p) + 1; @@ -725,17 +745,22 @@ int main(int argc, char **argv) if (frequency <= 0) frequency = 10; - report_fname = strconcat(prometheus_stats_dir(), FNAME_PROM_REPORT, NULL); - syslog(LOG_DEBUG, "updating %s every %d seconds", report_fname, frequency); - - xunlink(report_fname); - r = mappedfile_open(&report_file, report_fname, MAPPEDFILE_CREATE | MAPPEDFILE_RW); - free(report_fname); - if (r) fatal("couldn't open report file", EX_IOERR); + for (i = 0; i < n_reports; i++) { + char *fname = strconcat(prometheus_stats_dir(), + reports[i].fname, + NULL); + syslog(LOG_DEBUG, "updating %s every %d seconds", + fname, frequency); + + xunlink(fname); + r = mappedfile_open(&reports[i].mf, fname, + MAPPEDFILE_CREATE | MAPPEDFILE_RW); + free(fname); + if (r) fatal("couldn't open report file", EX_IOERR); + } for (;;) { int sig; - int64_t starttime; sig = signals_poll(); if (sig == SIGHUP && getenv("CYRUS_ISDAEMON")) { @@ -750,17 +775,15 @@ int main(int argc, char **argv) shut_down(0); } - starttime = now_ms(); - do_collate_report(&report_buf); - syslog(LOG_DEBUG, "collated service report in %f seconds", - (now_ms() - starttime) / 1000.0); + for (i = 0; i < n_reports; i++) { + int64_t starttime = now_ms(); - starttime = now_ms(); - do_collate_usage(&report_buf); - syslog(LOG_DEBUG, "collated usage report in %f seconds", - (now_ms() - starttime) / 1000.0); - - do_write_report(report_file, &report_buf); + reports[i].collate_fn(&reports[i].buf); + syslog(LOG_DEBUG, "collated %s report in %f seconds", + reports[i].desc, + (now_ms() - starttime) / 1000.0); + do_write_report(reports[i].mf, &reports[i].buf); + } if (oneshot) { shut_down(0); From 45cbd26490f84fca80181aa200a610cc2c377983 Mon Sep 17 00:00:00 2001 From: ellie timoney Date: Wed, 7 Aug 2024 11:08:12 +1000 Subject: [PATCH 037/501] prometheus: per-report update frequencies, step one Deprecates prometheus_update_freq in favour of prometheus_service_update_freq and prometheus_master_update_freq. Fow now, the usage report hangs on prometheus_service_update_freq. --- cassandane/Cassandane/Cyrus/Prometheus.pm | 3 ++- changes/next/prom-usage-stats | 20 ++++++++++++++++++ docsrc/imap/reference/admin/monitoring.rst | 13 ++++++++---- .../manpages/systemcommands/promstatsd.rst | 20 +++++++++--------- imap/promstatsd.c | 2 +- lib/imapoptions | 21 +++++++++++++------ master/master.c | 4 +++- 7 files changed, 60 insertions(+), 23 deletions(-) create mode 100644 changes/next/prom-usage-stats diff --git a/cassandane/Cassandane/Cyrus/Prometheus.pm b/cassandane/Cassandane/Cyrus/Prometheus.pm index f9bbb35021..dcb5c6ce90 100644 --- a/cassandane/Cassandane/Cyrus/Prometheus.pm +++ b/cassandane/Cassandane/Cyrus/Prometheus.pm @@ -59,7 +59,8 @@ sub new $config->set(prometheus_enabled => "yes"); $config->set(httpmodules => "prometheus"); $config->set(prometheus_need_auth => "none"); - $config->set(prometheus_update_freq => 2); + $config->set(prometheus_service_update_freq => 2); + $config->set(prometheus_master_update_freq => 2); return $class->SUPER::new( { adminstore => 1, diff --git a/changes/next/prom-usage-stats b/changes/next/prom-usage-stats new file mode 100644 index 0000000000..6f9ad50db2 --- /dev/null +++ b/changes/next/prom-usage-stats @@ -0,0 +1,20 @@ +Description: + +Increased granularity of Prometheus report frequency configuration. + + +Config changes: + +prometheus_update_freq +prometheus_service_update_freq +prometheus_master_update_freq + + +Upgrade instructions: + +None yet (FIXME) + + +GitHub issue: + +None diff --git a/docsrc/imap/reference/admin/monitoring.rst b/docsrc/imap/reference/admin/monitoring.rst index cef2836ee7..74670a9a41 100644 --- a/docsrc/imap/reference/admin/monitoring.rst +++ b/docsrc/imap/reference/admin/monitoring.rst @@ -16,8 +16,9 @@ Setup * Set the `prometheus_enabled` setting in :cyrusman:`imapd.conf(5)` to "yes" * Add the `prometheus` module to your `httpmodules` in :cyrusman:`imapd.conf(5)` - * Set the `prometheus_need_auth`, `prometheus_update_freq` and `prometheus_stats_dir` - settings in :cyrusman:`imapd.conf(5)` to taste + * Set the `prometheus_need_auth`, `prometheus_service_update_freq`, + `prometheus_master_update_freq`, and `prometheus_stats_dir` settings in + :cyrusman:`imapd.conf(5)` to taste * Add a job to run :cyrusman:`promstatsd(8)` to the DAEMON section of :cyrusman:`cyrus.conf(5)` (the actual daemon process) * Add a job to run ``promstatsd -c`` to the START section of :cyrusman:`cyrus.conf(5)` @@ -36,8 +37,12 @@ Configuration options :end-before: endblob prometheus_need_auth .. include:: /imap/reference/manpages/configs/imapd.conf.rst - :start-after: startblob prometheus_update_freq - :end-before: endblob prometheus_update_freq + :start-after: startblob prometheus_service_update_freq + :end-before: endblob prometheus_service_update_freq + + .. include:: /imap/reference/manpages/configs/imapd.conf.rst + :start-after: startblob prometheus_master_update_freq + :end-before: endblob prometheus_master_update_freq .. include:: /imap/reference/manpages/configs/imapd.conf.rst :start-after: startblob prometheus_stats_dir diff --git a/docsrc/imap/reference/manpages/systemcommands/promstatsd.rst b/docsrc/imap/reference/manpages/systemcommands/promstatsd.rst index 212277aacc..bd37ce5db5 100644 --- a/docsrc/imap/reference/manpages/systemcommands/promstatsd.rst +++ b/docsrc/imap/reference/manpages/systemcommands/promstatsd.rst @@ -13,7 +13,7 @@ Synopsis .. parsed-literal:: - **promstatsd** [ **-C** *config-file* ] [ **-v** ] [ **-f** *frequency* ] [ **-d** ] + **promstatsd** [ **-C** *config-file* ] [ **-v** ] [ **-f** *service-frequency* ] [ **-d** ] **promstatsd** [ **-C** *config-file* ] [ **-v** ] **-c** @@ -36,11 +36,11 @@ the "/metrics" URL, if "prometheus" has been set in **httpmodules** in **promstatsd** |default-conf-text| In the first synopsis, **promstatsd** will run as a daemon, updating the -report at the specified *frequency*. If the optional **-f** *frequency* -argument is not provided, the **prometheus_update_freq** from -:cyrusman:`imapd.conf(5)` will be used, which defaults to 10 seconds. This -invocation should be run from the DAEMON section of :cyrusman:`cyrus.conf(5)` -(see :ref:`promstatsd-examples` below). +service report at the specified *service-frequency*. If the optional +**-f** *service-frequency* argument is not provided, the +**prometheus_service_update_freq** from :cyrusman:`imapd.conf(5)` will be used, +which defaults to 10 seconds. This invocation should be run from the DAEMON +section of :cyrusman:`cyrus.conf(5)` (see :ref:`promstatsd-examples` below). In the second synopsis, **promstatsd** will clean up all statistics files and exit. The statistics Cyrus maintains are only valid while Cyrus is running, @@ -80,11 +80,11 @@ Options Debug mode -- **promstatsd** will not background itself, for aid in debugging. -.. option:: -f frequency +.. option:: -f service-frequency - Update the report every *frequency* seconds. If not specified, the - **prometheus_update_freq** from :cyrusman:`imapd.conf(5)` will be used, - which defaults to 10 seconds. + Update the service report every *service-frequency* seconds. If not + specified, the **prometheus_service_update_freq** from + :cyrusman:`imapd.conf(5)` will be used, which defaults to 10 seconds. .. option:: -v diff --git a/imap/promstatsd.c b/imap/promstatsd.c index 9b2b8037d9..a5c359e24c 100644 --- a/imap/promstatsd.c +++ b/imap/promstatsd.c @@ -741,7 +741,7 @@ int main(int argc, char **argv) } if (frequency <= 0) - frequency = config_getduration(IMAPOPT_PROMETHEUS_UPDATE_FREQ, 's'); + frequency = config_getduration(IMAPOPT_PROMETHEUS_SERVICE_UPDATE_FREQ, 's'); if (frequency <= 0) frequency = 10; diff --git a/lib/imapoptions b/lib/imapoptions index 152dc5d41d..47c7989fea 100644 --- a/lib/imapoptions +++ b/lib/imapoptions @@ -2156,17 +2156,26 @@ If all partitions are over that limit, this feature is not used anymore. used. */ { "prometheus_enabled", 0, SWITCH, "3.1.2" } -/* Whether tracking of service metrics for Prometheus is enabled. */ +/* Whether tracking of metrics for Prometheus is enabled. */ { "prometheus_need_auth", "admin", STRINGLIST("none", "user", "admin"), "3.1.2" } /* Authentication level required to fetch Prometheus metrics. */ -{ "prometheus_update_freq", "10s", DURATION, "3.1.8" } -/* Frequency in at which promstatsd should re-collate its statistics - report. The minimum value is 1 second. +{ "prometheus_update_freq", "10s", DURATION, "3.1.8", "UNRELEASED", "prometheus_service_update_freq" } +/* Deprecated in favour of \fIprometheus_service_update_freq\fR + and \fIprometheus_master_update_freq\fR */ + +{ "prometheus_service_update_freq", "10s", DURATION, "UNRELEASED" } +/* Frequency in at which promstatsd should re-collate its service + statistics report. The minimum value is 1 second. .PP - For backward compatibility, if no unit is specified, seconds is - assumed. */ + If no unit is specified, seconds is assumed. */ + +{ "prometheus_master_update_freq", NULL, DURATION, "UNRELEASED" } +/* Frequency in at which master should re-collate its statistics report. + If not set, \fIprometheus_service_update_freq\fR is used. +.PP + If no unit is specified, seconds is assumed. */ { "prometheus_stats_dir", NULL, STRING, "3.1.2" } /* Directory to use for gathering prometheus statistics. If specified, diff --git a/master/master.c b/master/master.c index eb04de5b96..db962822a7 100644 --- a/master/master.c +++ b/master/master.c @@ -2479,7 +2479,9 @@ static void init_prom_report(struct timeval now) const char *tmp; prom_enabled = config_getswitch(IMAPOPT_PROMETHEUS_ENABLED); - prom_frequency = config_getduration(IMAPOPT_PROMETHEUS_UPDATE_FREQ, 's'); + prom_frequency = config_getduration(IMAPOPT_PROMETHEUS_MASTER_UPDATE_FREQ, 's'); + if (!prom_frequency) + prom_frequency = config_getduration(IMAPOPT_PROMETHEUS_SERVICE_UPDATE_FREQ, 's'); if (prom_frequency < 1) prom_enabled = 0; if (!prom_enabled) return; From 4df1359522a5ab3403e603f0b4fb8d651b3a86cc Mon Sep 17 00:00:00 2001 From: ellie timoney Date: Wed, 7 Aug 2024 12:17:33 +1000 Subject: [PATCH 038/501] prometheus: per-report update frequencies, step two Adds prometheus_usage_update_freq, usage report no longer enabled by default --- cassandane/Cassandane/Cyrus/Prometheus.pm | 1 + changes/next/prom-usage-stats | 5 +- docsrc/imap/reference/admin/monitoring.rst | 8 +- .../manpages/systemcommands/promstatsd.rst | 16 ++-- imap/promstatsd.c | 79 +++++++++++++------ lib/imapoptions | 14 +++- 6 files changed, 88 insertions(+), 35 deletions(-) diff --git a/cassandane/Cassandane/Cyrus/Prometheus.pm b/cassandane/Cassandane/Cyrus/Prometheus.pm index dcb5c6ce90..f660d99bff 100644 --- a/cassandane/Cassandane/Cyrus/Prometheus.pm +++ b/cassandane/Cassandane/Cyrus/Prometheus.pm @@ -61,6 +61,7 @@ sub new $config->set(prometheus_need_auth => "none"); $config->set(prometheus_service_update_freq => 2); $config->set(prometheus_master_update_freq => 2); + $config->set(prometheus_usage_update_freq => 2); return $class->SUPER::new( { adminstore => 1, diff --git a/changes/next/prom-usage-stats b/changes/next/prom-usage-stats index 6f9ad50db2..679a6251d3 100644 --- a/changes/next/prom-usage-stats +++ b/changes/next/prom-usage-stats @@ -8,11 +8,14 @@ Config changes: prometheus_update_freq prometheus_service_update_freq prometheus_master_update_freq +prometheus_usage_update_freq Upgrade instructions: -None yet (FIXME) +The Prometheus usage report will be turned off by default after upgrade. +This means that the "cyrus_usage_..." metrics will no longer be reported. +To turn this back on, set prometheus_usage_update_freq to a suitable duration. GitHub issue: diff --git a/docsrc/imap/reference/admin/monitoring.rst b/docsrc/imap/reference/admin/monitoring.rst index 74670a9a41..cfc9d6d0db 100644 --- a/docsrc/imap/reference/admin/monitoring.rst +++ b/docsrc/imap/reference/admin/monitoring.rst @@ -17,8 +17,8 @@ Setup * Set the `prometheus_enabled` setting in :cyrusman:`imapd.conf(5)` to "yes" * Add the `prometheus` module to your `httpmodules` in :cyrusman:`imapd.conf(5)` * Set the `prometheus_need_auth`, `prometheus_service_update_freq`, - `prometheus_master_update_freq`, and `prometheus_stats_dir` settings in - :cyrusman:`imapd.conf(5)` to taste + `prometheus_master_update_freq`, `prometheus_usage_update_freq`, and + `prometheus_stats_dir` settings in :cyrusman:`imapd.conf(5)` to taste * Add a job to run :cyrusman:`promstatsd(8)` to the DAEMON section of :cyrusman:`cyrus.conf(5)` (the actual daemon process) * Add a job to run ``promstatsd -c`` to the START section of :cyrusman:`cyrus.conf(5)` @@ -44,6 +44,10 @@ Configuration options :start-after: startblob prometheus_master_update_freq :end-before: endblob prometheus_master_update_freq + .. include:: /imap/reference/manpages/configs/imapd.conf.rst + :start-after: startblob prometheus_usage_update_freq + :end-before: endblob prometheus_usage_update_freq + .. include:: /imap/reference/manpages/configs/imapd.conf.rst :start-after: startblob prometheus_stats_dir :end-before: endblob prometheus_stats_dir diff --git a/docsrc/imap/reference/manpages/systemcommands/promstatsd.rst b/docsrc/imap/reference/manpages/systemcommands/promstatsd.rst index bd37ce5db5..1fcd0c5e16 100644 --- a/docsrc/imap/reference/manpages/systemcommands/promstatsd.rst +++ b/docsrc/imap/reference/manpages/systemcommands/promstatsd.rst @@ -36,11 +36,13 @@ the "/metrics" URL, if "prometheus" has been set in **httpmodules** in **promstatsd** |default-conf-text| In the first synopsis, **promstatsd** will run as a daemon, updating the -service report at the specified *service-frequency*. If the optional -**-f** *service-frequency* argument is not provided, the -**prometheus_service_update_freq** from :cyrusman:`imapd.conf(5)` will be used, -which defaults to 10 seconds. This invocation should be run from the DAEMON -section of :cyrusman:`cyrus.conf(5)` (see :ref:`promstatsd-examples` below). +service and (optionally) usage reports at the frequencies set by the +**prometheus_service_update_freq** and **prometheus_usage_update_freq** +:cyrusman:`imapd.conf(5)` options, which default to 10s and disabled, +respectively. The optional **-f** *service-frequency* argument can be used to +override **prometheus_service_update_freq**. This invocation should be run +from the DAEMON section of :cyrusman:`cyrus.conf(5)` (see +:ref:`promstatsd-examples` below). In the second synopsis, **promstatsd** will clean up all statistics files and exit. The statistics Cyrus maintains are only valid while Cyrus is running, @@ -48,7 +50,7 @@ so this invocation must be run from the START section of :cyrusman:`cyrus.conf(5)` (see :ref:`promstatsd-examples` below) to clean up after the previous run, before new service processes are started. -In the third synopsis, **promstatsd** will immediately update the report +In the third synopsis, **promstatsd** will immediately update the report(s) once, and then exit. This can be safely used while another **promstatsd** process runs in daemon form. It is useful if you need to update the report *now* for some reason, rather than waiting for the daemon's next update. @@ -69,7 +71,7 @@ Options .. option:: -1 - Update the report once and exit. + Update the report(s) once and exit. .. option:: -c diff --git a/imap/promstatsd.c b/imap/promstatsd.c index a5c359e24c..5ddcb45cb3 100644 --- a/imap/promstatsd.c +++ b/imap/promstatsd.c @@ -73,17 +73,23 @@ static void do_collate_service_report(struct buf *buf); static void do_collate_usage_report(struct buf *buf); /* globals so that shut_down() can clean up */ -static struct { +static struct report { const char *fname; const char *desc; void (*collate_fn)(struct buf *); + enum imapopt freq_opt; + int default_frequency; struct mappedfile *mf; struct buf buf; + int frequency; + int64_t prev_report_time; } reports[] = { { FNAME_PROM_SERVICE_REPORT, "service", &do_collate_service_report, - NULL, BUF_INITIALIZER }, + IMAPOPT_PROMETHEUS_SERVICE_UPDATE_FREQ, 10, + NULL, BUF_INITIALIZER, 0, 0 }, { FNAME_PROM_USAGE_REPORT, "usage", &do_collate_usage_report, - NULL, BUF_INITIALIZER }, + IMAPOPT_PROMETHEUS_USAGE_UPDATE_FREQ, 0, + NULL, BUF_INITIALIZER, 0, 0 }, }; const size_t n_reports = sizeof(reports) / sizeof(reports[0]); @@ -112,7 +118,7 @@ static const char *argv0 = NULL; static void usage(void) { fprintf(stderr, "Usage:\n"); - fprintf(stderr, " %s [-C alt_config] [-v] [-f frequency] [-d]\n", argv0); + fprintf(stderr, " %s [-C alt_config] [-v] [-f service_frequency] [-d]\n", argv0); fprintf(stderr, " %s [-C alt_config] [-v] -c\n", argv0); exit(EX_USAGE); } @@ -644,6 +650,11 @@ static void do_write_report(struct mappedfile *mf, const struct buf *report) mappedfile_unlock(mf); } +static inline int report_due(const struct report *report, int64_t tick) +{ + return (report->prev_report_time + 1000 * report->frequency <= tick); +} + int main(int argc, char **argv) { save_argv0(argv[0]); @@ -654,8 +665,8 @@ int main(int argc, char **argv) int cleanup = 0; int debugmode = 0; int verbose = 0; - int frequency = 0; int oneshot = 0; + int min_frequency = INT_MAX; int opt; int r; unsigned i; @@ -685,9 +696,9 @@ int main(int argc, char **argv) debugmode = 1; break; - case 'f': /* set frequency */ - frequency = atoi(optarg); - if (frequency <= 0) usage(); + case 'f': /* set service report frequency */ + reports[0].frequency = atoi(optarg); + if (reports[0].frequency <= 0) usage(); break; case 'v': /* verbose */ @@ -740,17 +751,25 @@ int main(int argc, char **argv) } } - if (frequency <= 0) - frequency = config_getduration(IMAPOPT_PROMETHEUS_SERVICE_UPDATE_FREQ, 's'); - if (frequency <= 0) - frequency = 10; - for (i = 0; i < n_reports; i++) { char *fname = strconcat(prometheus_stats_dir(), reports[i].fname, NULL); - syslog(LOG_DEBUG, "updating %s every %d seconds", - fname, frequency); + + if (reports[i].frequency <= 0) + reports[i].frequency = config_getduration(reports[i].freq_opt, 's'); + if (reports[i].frequency <= 0) + reports[i].frequency = reports[i].default_frequency; + + if (reports[i].frequency) { + syslog(LOG_DEBUG, "updating %s every %d seconds", + fname, reports[i].frequency); + if (reports[i].frequency < min_frequency) + min_frequency = reports[i].frequency; + } + else { + syslog(LOG_DEBUG, "not updating %s due to frequency 0", fname); + } xunlink(fname); r = mappedfile_open(&reports[i].mf, fname, @@ -758,9 +777,11 @@ int main(int argc, char **argv) free(fname); if (r) fatal("couldn't open report file", EX_IOERR); } + assert(min_frequency > 0 && min_frequency < INT_MAX); for (;;) { int sig; + int64_t tick, elapsed; sig = signals_poll(); if (sig == SIGHUP && getenv("CYRUS_ISDAEMON")) { @@ -775,14 +796,19 @@ int main(int argc, char **argv) shut_down(0); } + tick = now_ms(); for (i = 0; i < n_reports; i++) { - int64_t starttime = now_ms(); - - reports[i].collate_fn(&reports[i].buf); - syslog(LOG_DEBUG, "collated %s report in %f seconds", - reports[i].desc, - (now_ms() - starttime) / 1000.0); - do_write_report(reports[i].mf, &reports[i].buf); + if (reports[i].frequency + && (oneshot || report_due(&reports[i], tick))) + { + int64_t profile_starttime = now_ms(); + reports[i].collate_fn(&reports[i].buf); + syslog(LOG_DEBUG, "collated %s report in %f seconds", + reports[i].desc, + (now_ms() - profile_starttime) / 1000.0); + do_write_report(reports[i].mf, &reports[i].buf); + reports[i].prev_report_time = tick; + } } if (oneshot) { @@ -790,7 +816,14 @@ int main(int argc, char **argv) } /* then wait around a bit */ - sleep(frequency); /* XXX substract elapsed time? */ + /* XXX This isn't exactly right: if service is 10s and usage is 15s + * XXX we'll end up waking at 10s, 20s, 30s, 40s, and the usage report + * XXX will be on a lurching clock since 10 doesn't evenly divide 15. + * XXX Probably want to use greatest common divisor, but that would + * XXX become annoying to compute if we add a third report to the mix. + */ + elapsed = now_ms() - tick; + sleep(min_frequency - elapsed / 1000); } /* NOTREACHED */ diff --git a/lib/imapoptions b/lib/imapoptions index 47c7989fea..136c01c773 100644 --- a/lib/imapoptions +++ b/lib/imapoptions @@ -2162,8 +2162,9 @@ If all partitions are over that limit, this feature is not used anymore. /* Authentication level required to fetch Prometheus metrics. */ { "prometheus_update_freq", "10s", DURATION, "3.1.8", "UNRELEASED", "prometheus_service_update_freq" } -/* Deprecated in favour of \fIprometheus_service_update_freq\fR - and \fIprometheus_master_update_freq\fR */ +/* Deprecated in favour of \fIprometheus_service_update_freq\fR, + \fIprometheus_master_update_freq\fR, and + \fIprometheus_usage_update_freq\fR. */ { "prometheus_service_update_freq", "10s", DURATION, "UNRELEASED" } /* Frequency in at which promstatsd should re-collate its service @@ -2177,6 +2178,15 @@ If all partitions are over that limit, this feature is not used anymore. .PP If no unit is specified, seconds is assumed. */ +{ "prometheus_usage_update_freq", NULL, DURATION, "UNRELEASED" } +/* Frequency in at which promstatsd should re-collate its usage + statistics report. Note that this report is relatively expensive to + produce. If not set, usage statistics are not reported. +.PP + Best to make this a multiple of \fIprometheus_service_update_freq\fR. +.PP + If no unit is specified, seconds is assumed. */ + { "prometheus_stats_dir", NULL, STRING, "3.1.2" } /* Directory to use for gathering prometheus statistics. If specified, must be an absolute path. If not specified, the default path From 6e899c822345ad45560c6ce6d01384de4c15f863 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=B8=D0=BB=D1=8F=D0=BD=20=D0=9F=D0=B0=D0=BB=D0=B0?= =?UTF-8?q?=D1=83=D0=B7=D0=BE=D0=B2?= Date: Tue, 13 Aug 2024 18:14:10 +0200 Subject: [PATCH 039/501] notify/nodifyd.c: update synopsis --- notifyd/notifyd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notifyd/notifyd.c b/notifyd/notifyd.c index bd30711f1a..545741011f 100644 --- a/notifyd/notifyd.c +++ b/notifyd/notifyd.c @@ -219,7 +219,7 @@ EXPORTED void fatal(const char *s, int code) static void usage(void) { - syslog(LOG_ERR, "usage: notifyd [-C ]"); + syslog(LOG_ERR, "usage: notifyd [-C ] [-U ] [-T timeout] [-D] [-X] [-m method]. method defaults to null"); exit(EX_USAGE); } From 262a1fc9638b56d6ac3a1f654b1d6f4cd68a5256 Mon Sep 17 00:00:00 2001 From: Ken Murchison Date: Wed, 14 Aug 2024 11:28:21 -0400 Subject: [PATCH 040/501] Sieve: update to latest processcalendar spec - will continue to execute processimp bytecode - upgrades processimip to processcalendar at parse/compile time --- cassandane/Cassandane/Cyrus/Sieve.pm | 2 +- cassandane/tiny-tests/Sieve/imip_add | 7 +- cassandane/tiny-tests/Sieve/imip_allow_public | 65 +++++++ cassandane/tiny-tests/Sieve/imip_cancel | 7 +- .../tiny-tests/Sieve/imip_cancel_delete | 7 +- .../tiny-tests/Sieve/imip_cancel_instance | 7 +- .../tiny-tests/Sieve/imip_cancel_to_organizer | 7 +- .../tiny-tests/Sieve/imip_disallow_public | 64 +++++++ cassandane/tiny-tests/Sieve/imip_invite | 7 +- .../tiny-tests/Sieve/imip_invite_base64 | 7 +- .../tiny-tests/Sieve/imip_invite_calendarid | 7 +- .../tiny-tests/Sieve/imip_invite_funky_uid | 7 +- .../Sieve/imip_invite_known_organizer | 88 ++++++++++ .../tiny-tests/Sieve/imip_invite_multipart | 7 +- .../Sieve/imip_invite_single_then_master | 7 +- .../Sieve/imip_invite_unknown_organizer | 87 +++++++++ .../tiny-tests/Sieve/imip_invite_updatesonly | 7 +- .../tiny-tests/Sieve/imip_invite_wrong_addr | 70 ++++++++ cassandane/tiny-tests/Sieve/imip_move_event | 7 +- cassandane/tiny-tests/Sieve/imip_override | 7 +- .../tiny-tests/Sieve/imip_preserve_alerts | 7 +- cassandane/tiny-tests/Sieve/imip_publish | 7 +- .../tiny-tests/Sieve/imip_publish_legacy | 70 ++++++++ .../Sieve/imip_publish_no_organizer | 68 +++++++ ...nizer => imip_publish_no_organizer_legacy} | 5 +- cassandane/tiny-tests/Sieve/imip_reply | 7 +- .../tiny-tests/Sieve/imip_reply_no_organizer | 9 +- .../Sieve/imip_reply_one_recurrence | 17 +- .../tiny-tests/Sieve/imip_reply_override | 7 +- .../Sieve/imip_reply_override_google | 7 +- .../Sieve/imip_reply_override_invalid | 7 +- .../Sieve/imip_reply_override_rdate | 7 +- .../tiny-tests/Sieve/imip_reply_with_alarm | 7 +- .../tiny-tests/Sieve/imip_strip_personal_data | 7 +- cassandane/tiny-tests/Sieve/imip_update | 7 +- .../Sieve/imip_update_master_and_add_override | 7 +- .../Sieve/remove_itip_on_jmap_update | 6 +- imap/itip_support.c | 5 +- imap/itip_support.h | 6 +- imap/lmtp_sieve.c | 115 +++++++----- imap/promdata.p | 2 +- lib/imapoptions | 2 +- sieve/bc_emit.c | 2 +- sieve/bc_eval.c | 40 +++-- sieve/bc_parse.c | 25 ++- sieve/bytecode.h | 22 ++- sieve/interp.c | 21 +-- sieve/interp.h | 8 +- sieve/script.c | 29 ++- sieve/sieve-lex.l | 24 ++- sieve/sieve.y | 166 ++++++++++++------ sieve/sieve_interface.h | 17 +- sieve/sieved.c | 51 ++++-- sieve/tree.c | 16 +- sieve/tree.h | 11 +- 55 files changed, 976 insertions(+), 312 deletions(-) create mode 100644 cassandane/tiny-tests/Sieve/imip_allow_public create mode 100644 cassandane/tiny-tests/Sieve/imip_disallow_public create mode 100644 cassandane/tiny-tests/Sieve/imip_invite_known_organizer create mode 100644 cassandane/tiny-tests/Sieve/imip_invite_unknown_organizer create mode 100644 cassandane/tiny-tests/Sieve/imip_invite_wrong_addr create mode 100644 cassandane/tiny-tests/Sieve/imip_publish_legacy create mode 100644 cassandane/tiny-tests/Sieve/imip_publish_no_organizer rename cassandane/tiny-tests/Sieve/{imip_publish_no_from_no_organizer => imip_publish_no_organizer_legacy} (93%) diff --git a/cassandane/Cassandane/Cyrus/Sieve.pm b/cassandane/Cassandane/Cyrus/Sieve.pm index b2cb8062d4..0ba95ce6cb 100644 --- a/cassandane/Cassandane/Cyrus/Sieve.pm +++ b/cassandane/Cassandane/Cyrus/Sieve.pm @@ -78,7 +78,7 @@ sub new } $config->set(sievenotifier => 'mailto'); $config->set(caldav_realm => 'Cassandane'); - $config->set(httpmodules => ['caldav', 'jmap']); + $config->set(httpmodules => ['caldav', 'carddav', 'jmap']); $config->set(calendar_user_address_set => 'example.com'); $config->set(httpallowcompress => 'no'); $config->set(caldav_historical_age => -1); diff --git a/cassandane/tiny-tests/Sieve/imip_add b/cassandane/tiny-tests/Sieve/imip_add index f263318928..2413918066 100644 --- a/cassandane/tiny-tests/Sieve/imip_add +++ b/cassandane/tiny-tests/Sieve/imip_add @@ -2,8 +2,7 @@ use Cassandane::Tiny; sub test_imip_add - :needs_component_sieve :needs_component_httpd :min_version_3_7 - :want_service_http + :needs_component_sieve :needs_component_httpd :want_service_http { my ($self) = @_; @@ -19,9 +18,9 @@ sub test_imip_add xlog $self, "Install a sieve script to process iMIP"; $self->{instance}->install_sieve_script(<{store}->get_client(); + $self->{store}->_select(); + $self->assert_num_equals(1, $IMAP->uid()); + $self->{store}->set_fetch_attributes(qw(uid flags)); + + xlog $self, "Create calendar user"; + my $CalDAV = $self->{caldav}; + my $CalendarId = 'Default'; + my $uuid = "6de280c9-edff-4019-8ebd-cfebc73f8201"; + + xlog $self, "Install a sieve script to process iMIP"; + $self->{instance}->install_sieve_script(< +Message-ID: <$uuid\@example.net> +Content-Type: text/calendar; component=VEVENT +X-Cassandane-Unique: $uuid + +BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Apple Inc.//Mac OS X 10.10.4//EN +BEGIN:VEVENT +CREATED:20210923T034327Z +UID:$uuid +DTEND;TZID=America/New_York:20210923T183000 +TRANSP:OPAQUE +SUMMARY:An Event +DTSTART;TZID=America/New_York:20210923T153000 +DTSTAMP:20210923T034327Z +SEQUENCE:0 +END:VEVENT +END:VCALENDAR +EOF + + xlog $self, "Deliver iMIP invite"; + my $msg = Cassandane::Message->new(raw => $imip); + $msg->set_attribute(uid => 1, + flags => [ '\\Recent', '\\Flagged' ]); + $self->{instance}->deliver($msg); + + xlog $self, "Check that the message made it to INBOX"; + $self->check_messages({ 1 => $msg }, check_guid => 0); + + xlog $self, "Check that the event made it to calendar"; + my $events = $CalDAV->GetEvents($CalendarId); + $self->assert_equals(1, scalar @$events); + $self->assert_str_equals($uuid, $events->[0]{uid}); +} diff --git a/cassandane/tiny-tests/Sieve/imip_cancel b/cassandane/tiny-tests/Sieve/imip_cancel index c436ebe684..2a21a210a7 100644 --- a/cassandane/tiny-tests/Sieve/imip_cancel +++ b/cassandane/tiny-tests/Sieve/imip_cancel @@ -2,8 +2,7 @@ use Cassandane::Tiny; sub test_imip_cancel - :needs_component_sieve :needs_component_httpd :min_version_3_5 - :want_service_http + :needs_component_sieve :needs_component_httpd :want_service_http { my ($self) = @_; @@ -19,9 +18,9 @@ sub test_imip_cancel xlog $self, "Install a sieve script to process iMIP"; $self->{instance}->install_sieve_script(<{instance}->install_sieve_script(<{instance}->install_sieve_script(<{instance}->install_sieve_script(<{store}->get_client(); + $self->{store}->_select(); + $self->assert_num_equals(1, $IMAP->uid()); + $self->{store}->set_fetch_attributes(qw(uid flags)); + + xlog $self, "Create calendar user"; + my $CalDAV = $self->{caldav}; + my $CalendarId = 'Default'; + my $uuid = "6de280c9-edff-4019-8ebd-cfebc73f8201"; + + xlog $self, "Install a sieve script to process iMIP"; + $self->{instance}->install_sieve_script(< +Message-ID: <$uuid\@example.net> +Content-Type: text/calendar; component=VEVENT +X-Cassandane-Unique: $uuid + +BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Apple Inc.//Mac OS X 10.10.4//EN +BEGIN:VEVENT +CREATED:20210923T034327Z +UID:$uuid +DTEND;TZID=America/New_York:20210923T183000 +TRANSP:OPAQUE +SUMMARY:An Event +DTSTART;TZID=America/New_York:20210923T153000 +DTSTAMP:20210923T034327Z +SEQUENCE:0 +END:VEVENT +END:VCALENDAR +EOF + + xlog $self, "Deliver iMIP invite"; + my $msg = Cassandane::Message->new(raw => $imip); + $msg->set_attribute(uid => 1, + flags => [ '\\Recent', '\\Flagged' ]); + $self->{instance}->deliver($msg); + + xlog $self, "Check that the message made it to INBOX"; + $self->check_messages({ 1 => $msg }, check_guid => 0); + + xlog $self, "Check that the event DID NOT make it to calendar"; + my $events = $CalDAV->GetEvents($CalendarId); + $self->assert_equals(0, scalar @$events); +} diff --git a/cassandane/tiny-tests/Sieve/imip_invite b/cassandane/tiny-tests/Sieve/imip_invite index 72bff7e9ee..8dc9a49ba2 100644 --- a/cassandane/tiny-tests/Sieve/imip_invite +++ b/cassandane/tiny-tests/Sieve/imip_invite @@ -2,8 +2,7 @@ use Cassandane::Tiny; sub test_imip_invite - :needs_component_sieve :needs_component_httpd :min_version_3_5 - :want_service_http + :needs_component_sieve :needs_component_httpd :want_service_http { my ($self) = @_; @@ -19,9 +18,9 @@ sub test_imip_invite xlog $self, "Install a sieve script to process iMIP"; $self->{instance}->install_sieve_script(<{instance}->install_sieve_script(<{instance}->install_sieve_script(<{instance}->install_sieve_script(<{store}->get_client(); + $self->{store}->_select(); + $self->assert_num_equals(1, $IMAP->uid()); + $self->{store}->set_fetch_attributes(qw(uid flags)); + + xlog $self, "Create calendar user"; + my $CalDAV = $self->{caldav}; + my $CalendarId = 'Default'; + my $uuid = "6de280c9-edff-4019-8ebd-cfebc73f8201"; + + xlog $self, "Create addressbook user"; + my $CardDAV = $self->{carddav}; + + xlog $self, "Add a card for known organizer"; + my $card = <NewContact('Default', + Net::CardDAVTalk::VCard->new_fromstring($card)); + + xlog $self, "Install a sieve script to process iMIP"; + $self->{instance}->install_sieve_script(< +To: Cassandane +Message-ID: <$uuid\@example.net> +Content-Type: text/calendar; method=REQUEST; component=VEVENT +X-Cassandane-Unique: $uuid + +BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Apple Inc.//Mac OS X 10.10.4//EN +METHOD:REQUEST +BEGIN:VEVENT +CREATED:20210923T034327Z +UID:$uuid +DTEND;TZID=America/New_York:20210923T183000 +TRANSP:OPAQUE +SUMMARY:An Event +DTSTART;TZID=America/New_York:20210923T153000 +DTSTAMP:20210923T034327Z +SEQUENCE:0 +ORGANIZER;CN=Test User:MAILTO:foo\@example.net +ATTENDEE;CN=Test User;PARTSTAT=ACCEPTED;RSVP=TRUE:MAILTO:foo\@example.net +ATTENDEE;PARTSTAT=NEEDS-ACTION;RSVP=TRUE:MAILTO:cassandane\@example.com +END:VEVENT +END:VCALENDAR +EOF + + xlog $self, "Deliver iMIP invite"; + my $msg = Cassandane::Message->new(raw => $imip); + $msg->set_attribute(uid => 1, + flags => [ '\\Recent', '\\Flagged' ]); + $self->{instance}->deliver($msg); + + xlog $self, "Check that the message made it to INBOX"; + $self->check_messages({ 1 => $msg }, check_guid => 0); + + xlog $self, "Check that the event made it to calendar"; + my $events = $CalDAV->GetEvents($CalendarId); + $self->assert_equals(1, scalar @$events); + $self->assert_str_equals($uuid, $events->[0]{uid}); +} diff --git a/cassandane/tiny-tests/Sieve/imip_invite_multipart b/cassandane/tiny-tests/Sieve/imip_invite_multipart index 9d20122c25..90f6a70da6 100644 --- a/cassandane/tiny-tests/Sieve/imip_invite_multipart +++ b/cassandane/tiny-tests/Sieve/imip_invite_multipart @@ -2,8 +2,7 @@ use Cassandane::Tiny; sub test_imip_invite_multipart - :needs_component_sieve :needs_component_httpd :min_version_3_5 - :want_service_http + :needs_component_sieve :needs_component_httpd :want_service_http { my ($self) = @_; @@ -19,9 +18,9 @@ sub test_imip_invite_multipart xlog $self, "Install a sieve script to process iMIP"; $self->{instance}->install_sieve_script(<{instance}->install_sieve_script(<{store}->get_client(); + $self->{store}->_select(); + $self->assert_num_equals(1, $IMAP->uid()); + $self->{store}->set_fetch_attributes(qw(uid flags)); + + xlog $self, "Create calendar user"; + my $CalDAV = $self->{caldav}; + my $CalendarId = 'Default'; + my $uuid = "6de280c9-edff-4019-8ebd-cfebc73f8201"; + + xlog $self, "Create addressbook user"; + my $CardDAV = $self->{carddav}; + + xlog $self, "Add a card for known organizer"; + my $card = <NewContact('Default', + Net::CardDAVTalk::VCard->new_fromstring($card)); + + xlog $self, "Install a sieve script to process iMIP"; + $self->{instance}->install_sieve_script(< +To: Cassandane +Message-ID: <$uuid\@example.net> +Content-Type: text/calendar; method=REQUEST; component=VEVENT +X-Cassandane-Unique: $uuid + +BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Apple Inc.//Mac OS X 10.10.4//EN +METHOD:REQUEST +BEGIN:VEVENT +CREATED:20210923T034327Z +UID:$uuid +DTEND;TZID=America/New_York:20210923T183000 +TRANSP:OPAQUE +SUMMARY:An Event +DTSTART;TZID=America/New_York:20210923T153000 +DTSTAMP:20210923T034327Z +SEQUENCE:0 +ORGANIZER;CN=Test User:MAILTO:bar\@example.net +ATTENDEE;CN=Test User;PARTSTAT=ACCEPTED;RSVP=TRUE:MAILTO:bar\@example.net +ATTENDEE;PARTSTAT=NEEDS-ACTION;RSVP=TRUE:MAILTO:cassandane\@example.com +END:VEVENT +END:VCALENDAR +EOF + + xlog $self, "Deliver iMIP invite"; + my $msg = Cassandane::Message->new(raw => $imip); + $msg->set_attribute(uid => 1, + flags => [ '\\Recent', '\\Flagged' ]); + $self->{instance}->deliver($msg); + + xlog $self, "Check that the message made it to INBOX"; + $self->check_messages({ 1 => $msg }, check_guid => 0); + + xlog $self, "Check that the event DID NOT make it to calendar"; + my $events = $CalDAV->GetEvents($CalendarId); + $self->assert_equals(0, scalar @$events); +} diff --git a/cassandane/tiny-tests/Sieve/imip_invite_updatesonly b/cassandane/tiny-tests/Sieve/imip_invite_updatesonly index f831ea8892..9549202e3a 100644 --- a/cassandane/tiny-tests/Sieve/imip_invite_updatesonly +++ b/cassandane/tiny-tests/Sieve/imip_invite_updatesonly @@ -2,8 +2,7 @@ use Cassandane::Tiny; sub test_imip_invite_updatesonly - :needs_component_sieve :needs_component_httpd :min_version_3_5 - :want_service_http + :needs_component_sieve :needs_component_httpd :want_service_http { my ($self) = @_; @@ -19,9 +18,9 @@ sub test_imip_invite_updatesonly xlog $self, "Install a sieve script to process iMIP"; $self->{instance}->install_sieve_script(<{store}->get_client(); + $self->{store}->_select(); + $self->assert_num_equals(1, $IMAP->uid()); + $self->{store}->set_fetch_attributes(qw(uid flags)); + + xlog $self, "Create calendar user"; + my $CalDAV = $self->{caldav}; + my $CalendarId = 'Default'; + my $uuid = "6de280c9-edff-4019-8ebd-cfebc73f8201"; + + xlog $self, "Install a sieve script to process iMIP"; + $self->{instance}->install_sieve_script(< +To: Cassandane +Message-ID: <$uuid\@example.net> +Content-Type: text/calendar; method=REQUEST; component=VEVENT +X-Cassandane-Unique: $uuid + +BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Apple Inc.//Mac OS X 10.10.4//EN +METHOD:REQUEST +BEGIN:VEVENT +CREATED:20210923T034327Z +UID:$uuid +DTEND;TZID=America/New_York:20210923T183000 +TRANSP:OPAQUE +SUMMARY:An Event +DTSTART;TZID=America/New_York:20210923T153000 +DTSTAMP:20210923T034327Z +SEQUENCE:0 +ORGANIZER;CN=Test User:MAILTO:foo\@example.net +ATTENDEE;CN=Test User;PARTSTAT=ACCEPTED;RSVP=TRUE:MAILTO:foo\@example.net +ATTENDEE;PARTSTAT=NEEDS-ACTION;RSVP=TRUE:MAILTO:not-cassandane\@example.com +END:VEVENT +END:VCALENDAR +EOF + + xlog $self, "Deliver iMIP invite"; + my $msg = Cassandane::Message->new(raw => $imip); + $msg->set_attribute(uid => 1, + flags => [ '\\Recent', '\\Flagged' ]); + $self->{instance}->deliver($msg); + + xlog $self, "Check that the message made it to INBOX"; + $self->check_messages({ 1 => $msg }, check_guid => 0); + + xlog $self, "Check that the event made it to calendar"; + my $events = $CalDAV->GetEvents($CalendarId); + $self->assert_equals(1, scalar @$events); + $self->assert_str_equals($uuid, $events->[0]{uid}); +} diff --git a/cassandane/tiny-tests/Sieve/imip_move_event b/cassandane/tiny-tests/Sieve/imip_move_event index 114a57424b..4cfbb81b95 100644 --- a/cassandane/tiny-tests/Sieve/imip_move_event +++ b/cassandane/tiny-tests/Sieve/imip_move_event @@ -2,8 +2,7 @@ use Cassandane::Tiny; sub test_imip_move_event - :min_version_3_7 :needs_component_jmap :needs_component_sieve - :want_service_http + :needs_component_jmap :needs_component_sieve :want_service_http { my ($self) = @_; @@ -13,9 +12,9 @@ sub test_imip_move_event xlog $self, "Install a sieve script to process iMIP"; $self->{instance}->install_sieve_script(<{instance}->install_sieve_script(<{instance}->install_sieve_script(<{instance}->install_sieve_script(<{store}->get_client(); + $self->{store}->_select(); + $self->assert_num_equals(1, $IMAP->uid()); + $self->{store}->set_fetch_attributes(qw(uid flags)); + + xlog $self, "Create calendar user"; + my $CalDAV = $self->{caldav}; + my $CalendarId = 'Default'; + my $uuid = "6de280c9-edff-4019-8ebd-cfebc73f8201"; + + xlog $self, "Install a sieve script to process iMIP"; + $self->{instance}->install_sieve_script(< +To: Cassandane +Message-ID: <$uuid\@example.net> +Content-Type: text/calendar; method=PUBLISH; component=VEVENT +X-Cassandane-Unique: $uuid + +BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Apple Inc.//Mac OS X 10.10.4//EN +METHOD:PUBLISH +BEGIN:VEVENT +CREATED:20210923T034327Z +UID:$uuid +DTEND;TZID=America/New_York:20210923T183000 +TRANSP:OPAQUE +SUMMARY:An Event +DTSTART;TZID=America/New_York:20210923T153000 +DTSTAMP:20210923T034327Z +SEQUENCE:0 +ORGANIZER: +END:VEVENT +END:VCALENDAR +EOF + + xlog $self, "Deliver iMIP invite"; + my $msg = Cassandane::Message->new(raw => $imip); + $msg->set_attribute(uid => 1, + flags => [ '\\Recent', '\\Flagged' ]); + $self->{instance}->deliver($msg); + + xlog $self, "Check that the message made it to INBOX"; + $self->check_messages({ 1 => $msg }, check_guid => 0); + + xlog $self, "Check that the event made it to calendar"; + my $events = $CalDAV->GetEvents($CalendarId); + $self->assert_equals(1, scalar @$events); + $self->assert_str_equals($uuid, $events->[0]{uid}); +} diff --git a/cassandane/tiny-tests/Sieve/imip_publish_no_organizer b/cassandane/tiny-tests/Sieve/imip_publish_no_organizer new file mode 100644 index 0000000000..e9ebd28e11 --- /dev/null +++ b/cassandane/tiny-tests/Sieve/imip_publish_no_organizer @@ -0,0 +1,68 @@ +#!perl +use Cassandane::Tiny; + +sub test_imip_publish_no_organizer + :needs_component_sieve :needs_component_httpd :want_service_http +{ + my ($self) = @_; + + my $IMAP = $self->{store}->get_client(); + $self->{store}->_select(); + $self->assert_num_equals(1, $IMAP->uid()); + $self->{store}->set_fetch_attributes(qw(uid flags)); + + xlog $self, "Create calendar user"; + my $CalDAV = $self->{caldav}; + my $CalendarId = 'Default'; + my $uuid = "6de280c9-edff-4019-8ebd-cfebc73f8201"; + + xlog $self, "Install a sieve script to process iMIP"; + $self->{instance}->install_sieve_script(< +Message-ID: <$uuid\@example.net> +Content-Type: text/calendar; method=PUBLISH; component=VEVENT +X-Cassandane-Unique: $uuid + +BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Apple Inc.//Mac OS X 10.10.4//EN +METHOD:PUBLISH +BEGIN:VEVENT +CREATED:20210923T034327Z +UID:$uuid +DTEND;TZID=America/New_York:20210923T183000 +TRANSP:OPAQUE +SUMMARY:An Event +DTSTART;TZID=America/New_York:20210923T153000 +DTSTAMP:20210923T034327Z +SEQUENCE:0 +END:VEVENT +END:VCALENDAR +EOF + + xlog $self, "Deliver iMIP invite"; + my $msg = Cassandane::Message->new(raw => $imip); + $msg->set_attribute(uid => 1, + flags => [ '\\Recent', '\\Flagged' ]); + $self->{instance}->deliver($msg); + + xlog $self, "Check that the message made it to INBOX"; + $self->check_messages({ 1 => $msg }, check_guid => 0); + + xlog $self, "Check that the event made it to calendar"; + my $events = $CalDAV->GetEvents($CalendarId); + $self->assert_equals(1, scalar @$events); + $self->assert_str_equals($uuid, $events->[0]{uid}); +} diff --git a/cassandane/tiny-tests/Sieve/imip_publish_no_from_no_organizer b/cassandane/tiny-tests/Sieve/imip_publish_no_organizer_legacy similarity index 93% rename from cassandane/tiny-tests/Sieve/imip_publish_no_from_no_organizer rename to cassandane/tiny-tests/Sieve/imip_publish_no_organizer_legacy index 3e3f71b8f1..c77b0388dd 100644 --- a/cassandane/tiny-tests/Sieve/imip_publish_no_from_no_organizer +++ b/cassandane/tiny-tests/Sieve/imip_publish_no_organizer_legacy @@ -1,9 +1,8 @@ #!perl use Cassandane::Tiny; -sub test_imip_publish_no_from_no_organizer - :needs_component_sieve :needs_component_httpd :min_version_3_9 - :want_service_http +sub test_imip_publish_no_organizer_legacy + :needs_component_sieve :needs_component_httpd :want_service_http { my ($self) = @_; diff --git a/cassandane/tiny-tests/Sieve/imip_reply b/cassandane/tiny-tests/Sieve/imip_reply index 3ca2aa9f91..409fe3e00f 100644 --- a/cassandane/tiny-tests/Sieve/imip_reply +++ b/cassandane/tiny-tests/Sieve/imip_reply @@ -2,8 +2,7 @@ use Cassandane::Tiny; sub test_imip_reply - :needs_component_sieve :needs_component_httpd :min_version_3_5 - :want_service_http + :needs_component_sieve :needs_component_httpd :want_service_http { my ($self) = @_; @@ -20,9 +19,9 @@ sub test_imip_reply xlog $self, "Install a sieve script to process iMIP"; $self->{instance}->install_sieve_script(<{instance}->install_sieve_script(<{instance}->install_sieve_script(<{instance}->install_sieve_script(<{instance}->install_sieve_script(<{instance}->install_sieve_script(<{instance}->install_sieve_script(<{instance}->install_sieve_script(<{instance}->install_sieve_script(<{instance}->install_sieve_script(<{instance}->install_sieve_script(<{instance}->install_sieve_script(<