From 8b63558686d5c0b4e61539f62b1ed3df44b56e74 Mon Sep 17 00:00:00 2001 From: ellie timoney Date: Mon, 22 Apr 2024 10:46:32 +1000 Subject: [PATCH] backup: remove source files --- .gitignore | 4 - backup/backup.h | 404 ----- backup/backupd.c | 1572 ----------------- backup/ctl_backups.c | 1119 ------------ backup/cyr_backup.c | 1009 ----------- backup/lcb.c | 844 --------- backup/lcb_append.c | 326 ---- backup/lcb_backupdb.c | 67 - backup/lcb_compact.c | 622 ------- backup/lcb_indexr.c | 1236 ------------- backup/lcb_indexw.c | 708 -------- backup/lcb_internal.c | 122 -- backup/lcb_internal.h | 123 -- backup/lcb_partlist.c | 103 -- backup/lcb_read.c | 267 --- backup/lcb_sqlconsts.c | 604 ------- backup/lcb_sqlconsts.h | 103 -- backup/lcb_verify.c | 652 ------- backup/restore.c | 942 ---------- doc/internal/backup | 903 ---------- doc/legacy/install-backups.html | 400 ----- docsrc/imap/concepts/deployment/databases.rst | 12 - docsrc/imap/developer/thoughts/backup.rst | 925 ---------- docsrc/imap/reference/admin.rst | 1 - docsrc/imap/reference/admin/backups.rst | 498 ------ docsrc/imap/reference/admin/locations.rst | 1 - .../manpages/systemcommands/backupd.rst | 98 - .../manpages/systemcommands/ctl_backups.rst | 272 --- .../manpages/systemcommands/cyr_backup.rst | 148 -- .../manpages/systemcommands/restore.rst | 249 --- lib/gzuncat.c | 353 ---- lib/gzuncat.h | 68 - 32 files changed, 14755 deletions(-) delete mode 100644 backup/backup.h delete mode 100644 backup/backupd.c delete mode 100644 backup/ctl_backups.c delete mode 100644 backup/cyr_backup.c delete mode 100644 backup/lcb.c delete mode 100644 backup/lcb_append.c delete mode 100644 backup/lcb_backupdb.c delete mode 100644 backup/lcb_compact.c delete mode 100644 backup/lcb_indexr.c delete mode 100644 backup/lcb_indexw.c delete mode 100644 backup/lcb_internal.c delete mode 100644 backup/lcb_internal.h delete mode 100644 backup/lcb_partlist.c delete mode 100644 backup/lcb_read.c delete mode 100644 backup/lcb_sqlconsts.c delete mode 100644 backup/lcb_sqlconsts.h delete mode 100644 backup/lcb_verify.c delete mode 100644 backup/restore.c delete mode 100644 doc/internal/backup delete mode 100644 doc/legacy/install-backups.html delete mode 100644 docsrc/imap/developer/thoughts/backup.rst delete mode 100644 docsrc/imap/reference/admin/backups.rst delete mode 100644 docsrc/imap/reference/manpages/systemcommands/backupd.rst delete mode 100644 docsrc/imap/reference/manpages/systemcommands/ctl_backups.rst delete mode 100644 docsrc/imap/reference/manpages/systemcommands/cyr_backup.rst delete mode 100644 docsrc/imap/reference/manpages/systemcommands/restore.rst delete mode 100644 lib/gzuncat.c delete mode 100644 lib/gzuncat.h diff --git a/.gitignore b/.gitignore index b3d7b458cb3..f40b0bb7060 100644 --- a/.gitignore +++ b/.gitignore @@ -29,10 +29,6 @@ Makefile.bak Makefile.old aclocal.m4 autom4te.cache/ -backup/backupd -backup/ctl_backups -backup/cyr_backup -backup/restore bench/cyrdbbench cmulocal/libtool.m4 cmulocal/ltoptions.m4 diff --git a/backup/backup.h b/backup/backup.h deleted file mode 100644 index c5d48a8a3a5..00000000000 --- a/backup/backup.h +++ /dev/null @@ -1,404 +0,0 @@ -/* backup.h -- replication-based backup api - * - * Copyright (c) 1994-2015 Carnegie Mellon University. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * 3. The name "Carnegie Mellon University" must not be used to - * endorse or promote products derived from this software without - * prior written permission. For permission or any legal - * details, please contact - * Carnegie Mellon University - * Center for Technology Transfer and Enterprise Creation - * 4615 Forbes Avenue - * Suite 302 - * Pittsburgh, PA 15213 - * (412) 268-7393, fax: (412) 268-7395 - * innovation@andrew.cmu.edu - * - * 4. Redistributions of any form whatsoever must retain the following - * acknowledgment: - * "This product includes software developed by Computing Services - * at Carnegie Mellon University (http://www.cmu.edu/computing/)." - * - * CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO - * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY - * AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE - * FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN - * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING - * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - * - */ - -#ifndef BACKUP_BACKUP_H -#define BACKUP_BACKUP_H - -#include "imap/dlist.h" -#include "imap/mboxname.h" -#include "imap/sync_support.h" - -#include "lib/gzuncat.h" - -struct backup; - -void backup_cleanup_staging_path(void); - -/* opening the backups database */ -#define FNAME_BACKUPDB "/backups.db" -int backupdb_open(struct db **backup_dbp, struct txn **tidp); - - -/* opening and closing backups */ -enum backup_open_nonblock { - BACKUP_OPEN_BLOCK = 0, - BACKUP_OPEN_NONBLOCK = 1, -}; - -enum backup_open_create { - BACKUP_OPEN_NOCREATE = 0, - BACKUP_OPEN_CREATE = 1, - BACKUP_OPEN_CREATE_EXCL = 2, -}; - -int backup_open(struct backup **backupp, - const mbname_t *mbname, - enum backup_open_nonblock nonblock, - enum backup_open_create create); - -int backup_get_paths(const mbname_t *mbname, - struct buf *data_fname, struct buf *index_fname, - enum backup_open_create create); - -int backup_open_paths(struct backup **backupp, - const char *data_fname, - const char *index_fname, - enum backup_open_nonblock nonblock, - enum backup_open_create create); - -int backup_close(struct backup **backupp); // also ends index/append ops -int backup_unlink(struct backup **backupp); - - -/* verifying backups */ -enum { - BACKUP_VERIFY_LAST_CHECKSUM = (1 << 0), - BACKUP_VERIFY_ALL_CHECKSUMS = (1 << 1), - BACKUP_VERIFY_MESSAGE_LINKS = (1 << 2), - BACKUP_VERIFY_MAILBOX_LINKS = (1 << 3), - BACKUP_VERIFY_MESSAGE_GUIDS = (1 << 4), -}; -#define BACKUP_VERIFY_QUICK BACKUP_VERIFY_LAST_CHECKSUM -#define BACKUP_VERIFY_MESSAGES (BACKUP_VERIFY_MESSAGE_LINKS | BACKUP_VERIFY_MESSAGE_GUIDS) -#define BACKUP_VERIFY_FULL ((unsigned) -1) -int backup_verify(struct backup *backup, unsigned level, int verbose, FILE *out); - - -/* accessing backup properties */ -const char *backup_get_data_fname(const struct backup *backup); -const char *backup_get_index_fname(const struct backup *backup); -int backup_stat(const struct backup *backup, - struct stat *data_statp, - struct stat *index_statp); - - -/* reading backup chunk data */ -struct backup_chunk { - struct backup_chunk *next; - int id; - time_t ts_start; - time_t ts_end; - off_t offset; - size_t length; - char *file_sha1; - char *data_sha1; -}; - -struct backup_chunk_list { - struct backup_chunk *head; - struct backup_chunk *tail; - size_t count; -}; - -void backup_chunk_list_add(struct backup_chunk_list *list, - struct backup_chunk *chunk); -void backup_chunk_list_empty(struct backup_chunk_list *list); -void backup_chunk_list_free(struct backup_chunk_list **chunk_listp); - -struct backup_chunk_list *backup_get_chunks(struct backup *backup); -struct backup_chunk_list *backup_get_live_chunks(struct backup *backup, - time_t since); - -struct backup_chunk *backup_get_chunk(struct backup *backup, int chunk_id); -struct backup_chunk *backup_get_latest_chunk(struct backup *backup); - -void backup_chunk_free(struct backup_chunk **chunkp); - -/* reading backup mailbox data */ -struct backup_mailbox_message { - struct backup_mailbox_message *next; - int id; - int mailbox_id; - char *mailbox_uniqueid; - int message_id; - int last_chunk_id; - int uid; - modseq_t modseq; - time_t last_updated; - char *flags; - time_t internaldate; - struct message_guid guid; - size_t size; - char *annotations; - time_t expunged; -}; - -struct backup_mailbox_message_list { - struct backup_mailbox_message *head; - struct backup_mailbox_message *tail; - size_t count; -}; - -struct backup_mailbox { - struct backup_mailbox *next; - int id; - int last_chunk_id; - char *uniqueid; - char *mboxname; - char *mboxtype; - uint32_t last_uid; - modseq_t highestmodseq; - uint32_t recentuid; - time_t recenttime; - time_t last_appenddate; - time_t pop3_last_login; - time_t pop3_show_after; - uint32_t uidvalidity; - char *partition; - char *acl; - char *options; - uint32_t sync_crc; - uint32_t sync_crc_annot; - char *quotaroot; - modseq_t xconvmodseq; - char *annotations; - time_t deleted; - struct backup_mailbox_message_list *records; -}; - -struct backup_mailbox_list { - struct backup_mailbox *head; - struct backup_mailbox *tail; - size_t count; -}; - -enum backup_mailbox_want_records { - BACKUP_MAILBOX_NO_RECORDS = 0, - BACKUP_MAILBOX_ALL_RECORDS = 1, - BACKUP_MAILBOX_MATCH_RECORDS = 2, -}; - -int backup_get_mailbox_id(struct backup *backup, const char *uniqueid); - -typedef int (*backup_mailbox_foreach_cb)(const struct backup_mailbox *mailbox, - void *rock); -int backup_mailbox_foreach(struct backup *backup, int chunk_id, - enum backup_mailbox_want_records want_records, - backup_mailbox_foreach_cb cb, void *rock); - -struct backup_mailbox_list *backup_get_mailboxes( - struct backup *backup, - int chunk_id, - enum backup_mailbox_want_records want_records); - -struct backup_message; -struct backup_mailbox_list *backup_get_mailboxes_by_message( - struct backup *backup, - const struct backup_message *message, - enum backup_mailbox_want_records want_records); - -struct backup_mailbox *backup_get_mailbox_by_uniqueid( - struct backup *backup, - const char *uniqueid, - enum backup_mailbox_want_records want_records); - -struct backup_mailbox *backup_get_mailbox_by_name( - struct backup *backup, - const mbname_t *mbname, - enum backup_mailbox_want_records want_records); - -struct dlist *backup_mailbox_to_dlist(const struct backup_mailbox *mailbox); - -struct backup_mailbox *backup_mailbox_clone(const struct backup_mailbox *mailbox); - -void backup_mailbox_free(struct backup_mailbox **mailboxp); - -struct backup_mailbox_message_list *backup_get_mailbox_messages(struct backup *backup, - int chunk_id); - -struct backup_mailbox_message *backup_get_mailbox_message( - struct backup *backup, - const char *uniqueid, - const char *guid); - -struct backup_mailbox_message *backup_mailbox_message_clone( - const struct backup_mailbox_message *orig); - -void backup_mailbox_message_free(struct backup_mailbox_message **mailbox_messagep); - -void backup_mailbox_list_add(struct backup_mailbox_list *list, - struct backup_mailbox *mailbox); - -struct backup_mailbox *backup_mailbox_list_remove(struct backup_mailbox_list *list, - struct backup_mailbox *mailbox); - -struct backup_mailbox_message *backup_mailbox_message_list_remove( - struct backup_mailbox_message_list *list, - struct backup_mailbox_message *mailbox_message); - -void backup_mailbox_list_empty(struct backup_mailbox_list *list); -void backup_mailbox_message_list_empty(struct backup_mailbox_message_list *list); - - -/* reading backup message data */ -struct backup_message { - int id; - struct message_guid *guid; - char *partition; - int chunk_id; - off_t offset; - size_t length; -}; - -int backup_get_message_id(struct backup *backup, const char *guid); -// FIXME do i even need these? -struct backup_message *backup_get_message(struct backup *backup, - const struct message_guid *guid); -char *backup_get_message_content(struct backup *backup, - const struct backup_message *message); -void backup_message_free(struct backup_message **message); - -typedef int (*backup_message_foreach_cb)(const struct backup_message *message, - void *rock); -int backup_message_foreach(struct backup *backup, - int chunk_id, const time_t *sincep, - backup_message_foreach_cb cb, void *rock); - - -/* reading backup seen data */ -struct backup_seen { - int id; - int last_chunk_id; - char *uniqueid; - time_t lastread; - uint32_t lastuid; - time_t lastchange; - char *seenuids; -}; - -void backup_seen_free(struct backup_seen **seen); - -typedef int (*backup_seen_foreach_cb)(const struct backup_seen *seen, - void *rock); -int backup_seen_foreach(struct backup *backup, - int chunk_id, - backup_seen_foreach_cb cb, - void *rock); - - -/* reading backup subscription data */ -struct backup_subscription { - int id; - int last_chunk_id; - char *mboxname; - time_t unsubscribed; -}; - -void backup_subscription_free(struct backup_subscription **sub); - -typedef int (*backup_subscription_foreach_cb)(const struct backup_subscription *sub, - void *rock); -int backup_subscription_foreach(struct backup *backup, - int chunk_id, - backup_subscription_foreach_cb cb, - void *rock); - - -/* reading backup sieve data */ -struct backup_sieve { - int id; - int chunk_id; - time_t last_update; - char *filename; - struct message_guid guid; - off_t offset; - time_t deleted; -}; - -void backup_sieve_free(struct backup_sieve **sieve); - -typedef int (*backup_sieve_foreach_cb)(const struct backup_sieve *sieve, - void *rock); -int backup_sieve_foreach(struct backup *backup, - int chunk_id, - backup_sieve_foreach_cb cb, - void *rock); - - -/* writing backup data */ -enum backup_append_flush { - BACKUP_APPEND_NOFLUSH = 0, - BACKUP_APPEND_FLUSH = 1, -}; - -int backup_append_start(struct backup *backup, - const time_t *tsp, - enum backup_append_flush flush); -int backup_append(struct backup *backup, - struct dlist *dlist, - const time_t *tsp, - enum backup_append_flush flush); -int backup_append_end(struct backup *backup, - const time_t *tsp); -int backup_append_abort(struct backup *backup); - - -/* reading backup data */ -typedef int (*backup_read_data_cb)(const struct buf *buf, void *rock); - -int backup_read_chunk_data(struct backup *backup, - const struct backup_chunk *chunk, - backup_read_data_cb proc, void *rock); -int backup_read_message_data(struct backup *backup, - const struct backup_message *message, - backup_read_data_cb proc, void *rock); - -typedef struct sync_msgid *(*sync_msgid_lookup_func)( - const struct sync_msgid_list *list, - const struct message_guid *guid); - -int backup_prepare_message_upload(struct backup *backup, - const char *partition, - struct sync_msgid_list *msgid_list, - sync_msgid_lookup_func msgid_lookup, - struct dlist **uploadp); -/* miscellaneous */ -int backup_reindex(const char *name, - enum backup_open_nonblock nonblock, - int verbose, FILE *out); -int backup_rename(const mbname_t *old_mbname, const mbname_t *new_mbname); -int backup_compact(const char *name, - enum backup_open_nonblock nonblock, - int force, int verbose, FILE *out); - -#endif diff --git a/backup/backupd.c b/backup/backupd.c deleted file mode 100644 index 7b502d92bf5..00000000000 --- a/backup/backupd.c +++ /dev/null @@ -1,1572 +0,0 @@ -/* backupd.c -- replication-based backup daemon - * - * Copyright (c) 1994-2015 Carnegie Mellon University. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * 3. The name "Carnegie Mellon University" must not be used to - * endorse or promote products derived from this software without - * prior written permission. For permission or any legal - * details, please contact - * Carnegie Mellon University - * Center for Technology Transfer and Enterprise Creation - * 4615 Forbes Avenue - * Suite 302 - * Pittsburgh, PA 15213 - * (412) 268-7393, fax: (412) 268-7395 - * innovation@andrew.cmu.edu - * - * 4. Redistributions of any form whatsoever must retain the following - * acknowledgment: - * "This product includes software developed by Computing Services - * at Carnegie Mellon University (http://www.cmu.edu/computing/)." - * - * CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO - * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY - * AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE - * FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN - * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING - * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - * - */ - -#include - -#include -#include -#include -#include -#include -#include - -#include -#include - -#include "lib/bsearch.h" -#include "lib/imparse.h" -#include "lib/map.h" -#include "lib/proc.h" -#include "lib/signals.h" -#include "lib/slowio.h" -#include "lib/strarray.h" -#include "lib/util.h" -#include "lib/xmalloc.h" - -#include "imap/global.h" -#include "imap/imap_err.h" -#include "imap/sync_support.h" -#include "imap/telemetry.h" -#include "imap/tls.h" -#include "imap/version.h" - -#include "backup/backup.h" - -const int config_need_data = 0; -static sasl_ssf_t extprops_ssf = 0; - -static struct auth_state *backupd_authstate = 0; -static int backupd_userisadmin = 0; -static char *backupd_userid = NULL; -static struct protstream *backupd_out = NULL; -static struct protstream *backupd_in = NULL; -static const char *backupd_clienthost = "[local]"; -static sasl_conn_t *backupd_saslconn = NULL; -static int backupd_starttls_done = 0; -static int backupd_compress_done = 0; -static int backupd_logfd = -1; -static struct proc_handle *proc_handle = NULL; - -struct open_backup { - char *name; - struct backup *backup; - time_t timestamp; - struct open_backup *next; - struct sync_msgid_list *reserved_guids; -}; - -static struct open_backups_list { - struct open_backup *head; - size_t count; -} backupd_open_backups = {0}; - -static struct { - char *ipremoteport; - char *iplocalport; - sasl_ssf_t ssf; - char *authid; -} saslprops = { NULL, NULL, 0, NULL}; - -static struct proxy_context backupd_proxyctx = { - 0, 1, &backupd_authstate, &backupd_userisadmin, NULL -}; - -static struct sasl_callback mysasl_cb[] = { - { SASL_CB_GETOPT, (mysasl_cb_ft *) &mysasl_config, NULL }, - { SASL_CB_PROXY_POLICY, (mysasl_cb_ft *) &mysasl_proxy_policy, (void*) &backupd_proxyctx }, - { SASL_CB_CANON_USER, (mysasl_cb_ft *) &mysasl_canon_user, NULL }, - { SASL_CB_LIST_END, NULL, NULL } -}; - -extern int saslserver(sasl_conn_t *conn, const char *mech, - const char *init_resp, const char *resp_prefix, - const char *continuation, const char *empty_resp, - struct protstream *pin, struct protstream *pout, - int *sasl_result, char **success_data); - -static void backupd_reset(void); -static void dobanner(void); -static int reset_saslconn(sasl_conn_t **conn); -static void shut_down(int code) __attribute__((noreturn)); -static void usage(void); - -static struct open_backup *open_backups_list_add(struct open_backups_list *list, - const char *name, - struct backup *backup); -static struct open_backup *open_backups_list_find(struct open_backups_list *list, - const char *name); -static void open_backups_list_close(struct open_backups_list *list, time_t age); - -static int backupd_print_mailbox(const struct backup_mailbox *mailbox, - void *rock __attribute__((__unused__))); -static int backupd_print_seen(const struct backup_seen *seen, - void *rock __attribute__((__unused__))); -static int backupd_print_subscriptions(struct backup *backup); -static int backupd_print_sieve(const struct backup_sieve *sieve, - void *rock __attribute__((__unused__))); -static const char *backupd_response(int r); - -static void cmdloop(void); -static void cmd_authenticate(char *mech, char *resp); -static void cmd_apply(struct dlist *dl); -static void cmd_compress(const char *alg); -static void cmd_get(struct dlist *dl); -static void cmd_restart(void); - -/****************************************************************************/ - -EXPORTED void fatal(const char* s, int code) -{ - static int recurse_code = 0; - - if (recurse_code) { - /* We were called recursively. Just give up */ - proc_cleanup(&proc_handle); - exit(recurse_code); - } - recurse_code = code; - - open_backups_list_close(&backupd_open_backups, 0); - - if (backupd_out) { - prot_printf(backupd_out, "* Fatal error: %s\r\n", s); - prot_flush(backupd_out); - } - syslog(LOG_ERR, "Fatal error: %s", s); - shut_down(code); -} - -/* - * run once when process is forked; - * MUST NOT exit directly; must return with non-zero error code - */ -EXPORTED int service_init(int argc __attribute__((unused)), - char **argv __attribute__((unused)), - char **envp __attribute__((unused))) -{ - // FIXME should this be calling fatal? fatal exits directly - if (geteuid() == 0) fatal("must run as the Cyrus user", EX_USAGE); - proc_settitle_init(argc, argv, envp); - - /* set signal handlers */ - signals_set_shutdown(&shut_down); - signal(SIGPIPE, SIG_IGN); - - /* load the SASL plugins */ - global_sasl_init(1, 1, mysasl_cb); - - int opt; - while ((opt = getopt(argc, argv, "p:f")) != EOF) { - switch(opt) { - case 'p': /* external protection */ - extprops_ssf = atoi(optarg); - break; - default: - usage(); - } - } - - return 0; -} - -/* Called by service API to shut down the service */ -EXPORTED void service_abort(int error) -{ - shut_down(error); -} - -/* - * run for each accepted connection - */ -EXPORTED int service_main(int argc __attribute__((unused)), - char **argv __attribute__((unused)), - char **envp __attribute__((unused))) -{ - const char *localip, *remoteip; - sasl_security_properties_t *secprops = NULL; - int r, timeout; - - signals_poll(); - - backupd_in = prot_new(0, 0); - backupd_out = prot_new(1, 1); - - /* Force use of LITERAL+ so we don't need two way communications */ - prot_setisclient(backupd_in, 1); - prot_setisclient(backupd_out, 1); - - /* Find out name of client host */ - backupd_clienthost = get_clienthost(0, &localip, &remoteip); - if (!strcmp(backupd_clienthost, UNIX_SOCKET)) { - /* we're not connected to an internet socket! */ - backupd_userid = xstrdup(cyrus_user()); - backupd_userisadmin = 1; - } - else { - /* other params should be filled in */ - if (sasl_server_new("csync", config_servername, NULL, NULL, NULL, - NULL, 0, &backupd_saslconn) != SASL_OK) - fatal("SASL failed initializing: sasl_server_new()",EX_TEMPFAIL); - - /* will always return something valid */ - secprops = mysasl_secprops(SASL_SEC_NOANONYMOUS); - if (sasl_setprop(backupd_saslconn, SASL_SEC_PROPS, secprops) != SASL_OK) - fatal("Failed to set SASL property", EX_TEMPFAIL); - - if (sasl_setprop(backupd_saslconn, SASL_SSF_EXTERNAL, &extprops_ssf) != SASL_OK) - fatal("Failed to set SASL property", EX_TEMPFAIL); - - if (localip) { - sasl_setprop(backupd_saslconn, SASL_IPLOCALPORT, localip); - saslprops.iplocalport = xstrdup(localip); - } - - if (remoteip) { - if (sasl_setprop(backupd_saslconn, SASL_IPREMOTEPORT, remoteip) != SASL_OK) - fatal("failed to set sasl property", EX_TEMPFAIL); - saslprops.ipremoteport = xstrdup(remoteip); - } - - tcp_disable_nagle(1); /* XXX magic fd */ - } - - r = proc_register(&proc_handle, 0, - config_ident, backupd_clienthost, NULL, NULL, NULL); - if (r) fatal("unable to register process", EX_IOERR); - proc_settitle(config_ident, backupd_clienthost, NULL, NULL, NULL); - - /* Set inactivity timer */ - timeout = config_getduration(IMAPOPT_SYNC_TIMEOUT, 's'); - if (timeout < 3) timeout = 3; - prot_settimeout(backupd_in, timeout); - - prot_setflushonread(backupd_in, backupd_out); - - dobanner(); - - cmdloop(); - - /* EXIT executed */ - - /* cleanup */ - backupd_reset(); - - return 0; -} - -/****************************************************************************/ - -static void backupd_reset(void) -{ - open_backups_list_close(&backupd_open_backups, 0); - - proc_cleanup(&proc_handle); - - if (backupd_in) { - prot_NONBLOCK(backupd_in); - prot_fill(backupd_in); - prot_free(backupd_in); - backupd_in = NULL; - } - - if (backupd_out) { - prot_flush(backupd_out); - prot_free(backupd_out); - backupd_out = NULL; - } - -//#ifdef HAVE_SSL -// if (tls_conn) { -// tls_reset_servertls(&tls_conn); -// tls_conn = NULL; -// } -//#endif - - cyrus_reset_stdio(); - - backupd_clienthost = "[local]"; - if (backupd_logfd != -1) { - close(backupd_logfd); - backupd_logfd = -1; - } - if (backupd_userid != NULL) { - free(backupd_userid); - backupd_userid = NULL; - } - if (backupd_authstate) { - auth_freestate(backupd_authstate); - backupd_authstate = NULL; - } - if (backupd_saslconn) { - sasl_dispose(&backupd_saslconn); - backupd_saslconn = NULL; - } - - backupd_starttls_done = 0; - backupd_compress_done = 0; - - if(saslprops.iplocalport) { - free(saslprops.iplocalport); - saslprops.iplocalport = NULL; - } - if(saslprops.ipremoteport) { - free(saslprops.ipremoteport); - saslprops.ipremoteport = NULL; - } - if(saslprops.authid) { - free(saslprops.authid); - saslprops.authid = NULL; - } - saslprops.ssf = 0; - - backup_cleanup_staging_path(); - - slowio_reset(); -} - -static void dobanner(void) -{ - const char *mechlist; - int mechcount; - - if (!backupd_userid) { - if (sasl_listmech(backupd_saslconn, NULL, - "* SASL ", " ", "\r\n", - &mechlist, NULL, &mechcount) == SASL_OK - && mechcount > 0) { - prot_printf(backupd_out, "%s", mechlist); - } - - if (tls_enabled() && !backupd_starttls_done) { - prot_printf(backupd_out, "* STARTTLS\r\n"); - } - -#ifdef HAVE_ZLIB - if (!backupd_compress_done && !backupd_starttls_done) { - prot_printf(backupd_out, "* COMPRESS DEFLATE\r\n"); - } -#endif - } - - prot_printf(backupd_out, - "* OK %s Cyrus backup server %s\r\n", - config_servername, CYRUS_VERSION); - - prot_flush(backupd_out); -} - -/* Reset the given sasl_conn_t to a sane state */ -static int reset_saslconn(sasl_conn_t **conn) -{ - int ret; - sasl_security_properties_t *secprops = NULL; - - sasl_dispose(conn); - /* do initialization typical of service_main */ - ret = sasl_server_new("csync", config_servername, - NULL, NULL, NULL, - NULL, 0, conn); - if (ret != SASL_OK) return ret; - - if (saslprops.ipremoteport) - ret = sasl_setprop(*conn, SASL_IPREMOTEPORT, - saslprops.ipremoteport); - if (ret != SASL_OK) return ret; - - if (saslprops.iplocalport) - ret = sasl_setprop(*conn, SASL_IPLOCALPORT, - saslprops.iplocalport); - if (ret != SASL_OK) return ret; - secprops = mysasl_secprops(SASL_SEC_NOANONYMOUS); - ret = sasl_setprop(*conn, SASL_SEC_PROPS, secprops); - if (ret != SASL_OK) return ret; - /* end of service_main initialization excepting SSF */ - - /* If we have TLS/SSL info, set it */ - if (saslprops.ssf) { - ret = sasl_setprop(*conn, SASL_SSF_EXTERNAL, &saslprops.ssf); - } else { - ret = sasl_setprop(*conn, SASL_SSF_EXTERNAL, &extprops_ssf); - } - - if (ret != SASL_OK) return ret; - - if (saslprops.authid) { - ret = sasl_setprop(*conn, SASL_AUTH_EXTERNAL, saslprops.authid); - if(ret != SASL_OK) return ret; - } - /* End TLS/SSL Info */ - - return SASL_OK; -} - -static void shut_down(int code) -{ - in_shutdown = 1; - - backupd_reset(); - - cyrus_done(); - - exit(code); -} - -static void usage(void) -{ - // FIXME -} - -/****************************************************************************/ - -static struct open_backup *open_backups_list_add(struct open_backups_list *list, - const char *name, struct backup *backup) -{ - struct open_backup *open = xzmalloc(sizeof(struct open_backup)); - - open->next = list->head; - list->head = open; - list->count++; - - open->name = xstrdupnull(name); - open->backup = backup; - open->timestamp = time(0); - - /* XXX pick a suitable msgid list hash size: - * 0 (default) => 256 - * SYNC_MSGID_LIST_HASH_SIZE (used by sync_server) => 65536 - */ - open->reserved_guids = sync_msgid_list_create(0); - - return open; -} - -static struct open_backup *open_backups_list_find(struct open_backups_list *list, - const char *name) -{ - struct open_backup *open; - - for (open = list->head; open; open = open->next) { - if (strcmpnull(name, open->name) == 0) - return open; - } - - return NULL; -} - -static int backupd_open_backup(struct open_backup **openp, const mbname_t *mbname) -{ - const char *key = mbname_userid(mbname); - - struct open_backup *open = open_backups_list_find(&backupd_open_backups, key); - - time_t now = time(0); - - if (!open) { - struct backup *backup = NULL; - int r = backup_open(&backup, mbname, - BACKUP_OPEN_NONBLOCK, BACKUP_OPEN_CREATE); - if (r) return r; - - r = backup_verify(backup, BACKUP_VERIFY_QUICK, 0, NULL); - - if (!r) - r = backup_append_start(backup, NULL, BACKUP_APPEND_FLUSH); - - if (r) { - backup_close(&backup); - return r; - } - - open = open_backups_list_add(&backupd_open_backups, key, backup); - } - - open->timestamp = now; - *openp = open; - - return 0; -} - - -// FIXME do i even need this - yes if a user gets renamed, need to close the old handle -#if 0 -static struct open_backup *open_backups_list_remove(struct open_backups_list *list, - const char *name) -{ - struct open_backup *open, *prev = NULL; - - for (open = list->head; open; open = open->next) { - if (strcmp(name, open->name) == 0) - break; - - prev = open; - } - - if (!open) return NULL; - - if (prev) - prev->next = open->next; - else - list->head = open->next; - - list->count --; - open->next = NULL; - return open; -} -#endif - -static void open_backups_list_close(struct open_backups_list *list, time_t age) -{ - time_t now = time(0); - - struct open_backup *current = list->head, *prev = NULL; - - while (current) { - struct open_backup *next = current->next; - - if (!age || current->timestamp < now - age) { - current->next = NULL; - backup_close(¤t->backup); - sync_msgid_list_free(¤t->reserved_guids); - free(current->name); - free(current); - - if (prev) { - prev->next = next; - } - else { - list->head = next; - } - - list->count --; - } - else { - prev = current; - } - - current = next; - } -} - -/****************************************************************************/ - -static void cmdloop(void) -{ - int c; - char *p; - static struct buf cmd; - static struct buf arg1, arg2; - - /* we don't expect there to be backups open already */ - assert(backupd_open_backups.head == NULL); - assert(backupd_open_backups.count == 0); - - for (;;) { - prot_flush(backupd_out); - open_backups_list_close(&backupd_open_backups, 5 * 60); /* 5 mins FIXME */ - - /* Parse command name */ - if ((c = getword(backupd_in, &cmd)) == EOF) - break; - - if (!cmd.s[0]) { - prot_printf(backupd_out, "BAD Null command\r\n"); - eatline(backupd_in, c); - continue; - } - - if (Uislower(cmd.s[0])) - cmd.s[0] = toupper((unsigned char) cmd.s[0]); - for (p = &cmd.s[1]; *p; p++) { - if (Uisupper(*p)) *p = tolower((unsigned char) *p); - } - - /* Must be an admin */ - if (backupd_userid && !backupd_userisadmin) goto noperm; - - switch (cmd.s[0]) { - case 'A': - if (!strcmp(cmd.s, "Authenticate")) { - int haveinitresp = 0; - if (c != ' ') goto missingargs; - c = getword(backupd_in, &arg1); - if (!imparse_isatom(arg1.s)) { - prot_printf(backupd_out, "BAD Invalid mechanism\r\n"); - eatline(backupd_in, c); - continue; - } - if (c == ' ') { - haveinitresp = 1; - c = getword(backupd_in, &arg2); - if (c == EOF) goto missingargs; - } - if (c == '\r') c = prot_getc(backupd_in); - if (c != '\n') goto extraargs; - - if (backupd_userid) { - prot_printf(backupd_out, "BAD Already authenticated\r\n"); - continue; - } - cmd_authenticate(arg1.s, haveinitresp ? arg2.s : NULL); - continue; - } - if (!backupd_userid) goto nologin; - if (!strcmp(cmd.s, "Apply")) { - struct dlist *dl = NULL; - c = dlist_parse(&dl, /*parsekeys*/ 1, /*isarchive*/ 0, /*isbackup*/ 1, backupd_in); - if (c == EOF) goto missingargs; - if (c == '\r') c = prot_getc(backupd_in); - if (c != '\n') goto extraargs; - cmd_apply(dl); - dlist_unlink_files(dl); - dlist_free(&dl); - continue; - } - break; - - case 'C': - if (!strcmp(cmd.s, "Compress")) { - if (c != ' ') goto missingargs; - c = getword(backupd_in, &arg1); - if (c == '\r') c = prot_getc(backupd_in); - if (c != '\n') goto extraargs; - cmd_compress(arg1.s); - continue; - } - break; - - case 'E': - if (!strcmp(cmd.s, "Exit")) { - if (c == '\r') c = prot_getc(backupd_in); - if (c != '\n') goto extraargs; - prot_printf(backupd_out, "OK Finished\r\n"); - prot_flush(backupd_out); - goto exit; - } - break; - - case 'G': - if (!backupd_userid) goto nologin; - if (!strcmp(cmd.s, "Get")) { - struct dlist *dl = NULL; - c = dlist_parse(&dl, /*parsekeys*/ 1, /*isarchive*/ 0, /*isbackup*/ 1, backupd_in); - if (c == EOF) goto missingargs; - if (c == '\r') c = prot_getc(backupd_in); - if (c != '\n') goto extraargs; - cmd_get(dl); - dlist_unlink_files(dl); - dlist_free(&dl); - continue; - } - break; - - case 'N': - if (!strcmp(cmd.s, "Noop")) { - if (c == '\r') c = prot_getc(backupd_in); - if (c != '\n') goto extraargs; - prot_printf(backupd_out, "OK Noop completed\r\n"); - continue; - } - break; - - case 'R': - if (!strcmp(cmd.s, "Restart")) { - if (c == '\r') c = prot_getc(backupd_in); - if (c != '\n') goto extraargs; - cmd_restart(); - prot_printf(backupd_out, "OK Restarting\r\n"); - continue; - } - break; - - case 'S': - if (!strcmp(cmd.s, "Starttls") && tls_enabled()) { - prot_printf(backupd_out, "NO command not implemented\r\n"); - eatline(backupd_in, c); - continue; - } - break; - - } - - xsyslog(LOG_ERR, "IOERROR: received bad command", - "command=<%s>", cmd.s); - prot_printf(backupd_out, "BAD IMAP_PROTOCOL_ERROR Unrecognized command\r\n"); - eatline(backupd_in, c); - continue; - - nologin: - prot_printf(backupd_out, "NO Please authenticate first\r\n"); - eatline(backupd_in, c); - continue; - - noperm: - prot_printf(backupd_out, "NO %s\r\n", - error_message(IMAP_PERMISSION_DENIED)); - eatline(backupd_in, c); - continue; - - missingargs: - prot_printf(backupd_out, "BAD Missing required argument to %s\r\n", cmd.s); - eatline(backupd_in, c); - continue; - - extraargs: - prot_printf(backupd_out, "BAD Unexpected extra arguments to %s\r\n", cmd.s); - eatline(backupd_in, c); - continue; - } - -exit: - cmd_restart(); -} - -static void cmd_authenticate(char *mech, char *resp) -{ - int r, sasl_result; - sasl_ssf_t ssf; - const char *ssfmsg = NULL; - const void *val; - int failedloginpause; - - if (backupd_userid) { - prot_printf(backupd_out, "BAD Already authenticated\r\n"); - return; - } - - r = saslserver(backupd_saslconn, mech, resp, "", "+ ", "", - backupd_in, backupd_out, &sasl_result, NULL); - - if (r) { - const char *errorstring = NULL; - - switch (r) { - case IMAP_SASL_CANCEL: - prot_printf(backupd_out, - "BAD Client canceled authentication\r\n"); - break; - case IMAP_SASL_PROTERR: - errorstring = prot_error(backupd_in); - - prot_printf(backupd_out, - "NO Error reading client response: %s\r\n", - errorstring ? errorstring : ""); - break; - default: - /* failed authentication */ - errorstring = sasl_errstring(sasl_result, NULL, NULL); - - syslog(LOG_NOTICE, "badlogin: %s %s [%s]", - backupd_clienthost, mech, sasl_errdetail(backupd_saslconn)); - - failedloginpause = config_getduration(IMAPOPT_FAILEDLOGINPAUSE, 's'); - if (failedloginpause != 0) { - sleep(failedloginpause); - } - - if (errorstring) { - prot_printf(backupd_out, "NO %s\r\n", errorstring); - } else { - prot_printf(backupd_out, "NO Error authenticating\r\n"); - } - } - - reset_saslconn(&backupd_saslconn); - return; - } - - /* successful authentication */ - - /* get the userid from SASL --- already canonicalized from - * mysasl_proxy_policy() - */ - sasl_result = sasl_getprop(backupd_saslconn, SASL_USERNAME, &val); - if (sasl_result != SASL_OK) { - prot_printf(backupd_out, "NO weird SASL error %d SASL_USERNAME\r\n", - sasl_result); - syslog(LOG_ERR, "weird SASL error %d getting SASL_USERNAME", - sasl_result); - reset_saslconn(&backupd_saslconn); - return; - } - - backupd_userid = xstrdup((const char *) val); - r = proc_register(&proc_handle, 0, - config_ident, backupd_clienthost, backupd_userid, - NULL, NULL); - if (r) fatal("unable to register process", EX_IOERR); - proc_settitle(config_ident, backupd_clienthost, backupd_userid, NULL, NULL); - - syslog(LOG_NOTICE, "login: %s %s %s%s %s", backupd_clienthost, backupd_userid, - mech, backupd_starttls_done ? "+TLS" : "", "User logged in"); - - sasl_getprop(backupd_saslconn, SASL_SSF, &val); - ssf = *((sasl_ssf_t *) val); - - /* really, we should be doing a sasl_getprop on SASL_SSF_EXTERNAL, - but the current libsasl doesn't allow that. */ - if (backupd_starttls_done) { - switch(ssf) { - case 0: ssfmsg = "tls protection"; break; - case 1: ssfmsg = "tls plus integrity protection"; break; - default: ssfmsg = "tls plus privacy protection"; break; - } - } else { - switch(ssf) { - case 0: ssfmsg = "no protection"; break; - case 1: ssfmsg = "integrity protection"; break; - default: ssfmsg = "privacy protection"; break; - } - } - - prot_printf(backupd_out, "OK Success (%s)\r\n", ssfmsg); - - prot_setsasl(backupd_in, backupd_saslconn); - prot_setsasl(backupd_out, backupd_saslconn); - - /* Create telemetry log */ - backupd_logfd = telemetry_log(backupd_userid, backupd_in, backupd_out, 0); -} - -static int cmd_apply_mailbox(struct dlist *dl) -{ - const char *mboxname = NULL; - struct open_backup *open = NULL; - int r; - - if (!dlist_getatom(dl, "MBOXNAME", &mboxname)) return IMAP_PROTOCOL_ERROR; - - mbname_t *mbname = mbname_from_intname(mboxname); - r = backupd_open_backup(&open, mbname); - mbname_free(&mbname); - - if (!r) - r = backup_append(open->backup, dl, NULL, BACKUP_APPEND_FLUSH); - - return r; -} - -static int cmd_apply_unmailbox(struct dlist *dl) -{ - const char *mboxname = dl->sval; - struct open_backup *open = NULL; - int r; - - mbname_t *mbname = mbname_from_intname(mboxname); - r = backupd_open_backup(&open, mbname); - mbname_free(&mbname); - - if (!r) - r = backup_append(open->backup, dl, NULL, BACKUP_APPEND_FLUSH); - - return r; -} - -static int cmd_apply_message(struct dlist *dl) -{ - struct sync_msgid_list *guids = sync_msgid_list_create(0); - struct dlist *di; - int r = 0; - - /* dig out each guid */ - for (di = dl->head; di; di = di->next) { - struct message_guid computed_guid, *guid = NULL; - const char *fname = NULL; - const char *msg_base = NULL; - size_t msg_len = 0; - int fd; - - if (!dlist_tofile(di, NULL, &guid, NULL, &fname)) - continue; - - /* bail out if it doesn't match the data */ - fd = open(fname, O_RDWR); - if (fd != -1) { - map_refresh(fd, 1, &msg_base, &msg_len, MAP_UNKNOWN_LEN, fname, NULL); - - message_guid_generate(&computed_guid, msg_base, msg_len); - if (!message_guid_equal(guid, &computed_guid)) { - syslog(LOG_ERR, "%s: guid mismatch: header %s, derived %s", - __func__, message_guid_encode(guid), - message_guid_encode(&computed_guid)); - r = IMAP_PROTOCOL_ERROR; - } - - map_free(&msg_base, &msg_len); - close(fd); - } - else { - xsyslog(LOG_ERR, "IOERROR: open failed", - "filename=<%s>", fname); - r = IMAP_IOERROR; - } - - if (r) goto done; - sync_msgid_insert(guids, guid); - } - - /* bail out if there's no messages */ - if (!guids->head) { - r = IMAP_PROTOCOL_ERROR; - goto done; - } - - /* find each open backup that wants a copy of any of these guids, - * and append the entire MESSAGE line to it - */ - struct open_backup *open; - int appended = 0; - for (open = backupd_open_backups.head; open; open = open->next) { - struct sync_msgid *msgid; - int want_append = 0; - - for (msgid = guids->head; msgid; msgid = msgid->next) { - if (sync_msgid_lookup(open->reserved_guids, &msgid->guid)) { - want_append++; - sync_msgid_remove(open->reserved_guids, &msgid->guid); - } - } - - if (want_append) { - r = backup_append(open->backup, dl, NULL, BACKUP_APPEND_FLUSH); - if (r) break; - appended++; - } - } - - /* for mailboxes that have never been seen before, the sync client - * will send APPLY MESSAGE commands without a corresponding reserve, - * which means none of the open backups will accept the message. if - * we get here and haven't appended the line to any backup, then - * we've found this case. so append it to all of the open backups, - * and let compact sort it out. - */ - if (!r && appended == 0) { - if (backupd_open_backups.count) { - syslog(LOG_DEBUG, - "received unreserved messages, applying to all (" - SIZE_T_FMT ") open backups...", - backupd_open_backups.count); - for (open = backupd_open_backups.head; open; open = open->next) { - syslog(LOG_DEBUG, "applying unreserved messages to %s", open->name); - r = backup_append(open->backup, dl, NULL, BACKUP_APPEND_FLUSH); - if (r) break; - } - } - else { - syslog(LOG_DEBUG, - "received unreserved messages, but no open backups to apply to"); - r = IMAP_PROTOCOL_ERROR; - } - } - -done: - sync_msgid_list_free(&guids); - return r; -} - -static int reserve_one(const mbname_t *mbname, - struct dlist *dl, struct dlist *gl, - struct sync_msgid_list *missing) -{ - struct open_backup *open = NULL; - struct dlist *di; - int r; - - r = backupd_open_backup(&open, mbname); - if (r) return r; - - r = backup_append(open->backup, dl, NULL, BACKUP_APPEND_FLUSH); - if (r) return r; - - for (di = gl->head; di; di = di->next) { - struct message_guid *guid = NULL; - const char *guid_str; - - if (!dlist_toguid(di, &guid)) continue; - guid_str = message_guid_encode(guid); - - int message_id = backup_get_message_id(open->backup, guid_str); - - if (message_id <= 0) { - syslog(LOG_DEBUG, "%s: %s wants message %s", - __func__, mbname_intname(mbname), guid_str); - - /* add it to the reserved guids list */ - sync_msgid_insert(open->reserved_guids, guid); - - /* add it to the missing list */ - sync_msgid_insert(missing, guid); - } - } - - return 0; -} - -static int cmd_apply_reserve(struct dlist *dl) -{ - const char *partition = NULL; - struct dlist *ml = NULL; - struct dlist *gl = NULL; - struct dlist *di; - strarray_t userids = STRARRAY_INITIALIZER; - mbname_t *shared_mbname = NULL; - int i, r = 0; - - if (!dlist_getatom(dl, "PARTITION", &partition)) return IMAP_PROTOCOL_ERROR; - if (!dlist_getlist(dl, "MBOXNAME", &ml)) return IMAP_PROTOCOL_ERROR; - if (!dlist_getlist(dl, "GUID", &gl)) return IMAP_PROTOCOL_ERROR; - - /* find the list of users this reserve applies to */ - for (di = ml->head; di; di = di->next) { - mbname_t *mbname = mbname_from_intname(di->sval); - if (mbname_userid(mbname)) { - strarray_append(&userids, mbname_userid(mbname)); - mbname_free(&mbname); - } - else if (!shared_mbname) { - shared_mbname = mbname; - } - else { - mbname_free(&mbname); - } - } - strarray_sort(&userids, cmpstringp_raw); - strarray_uniq(&userids); - - /* track the missing guids */ - struct sync_msgid_list *missing = sync_msgid_list_create(0); - - /* log the entire reserve to all relevant backups, and accumulate missing list */ - for (i = 0; i < strarray_size(&userids); i++) { - mbname_t *mbname = mbname_from_userid(strarray_nth(&userids, i)); - r = reserve_one(mbname, dl, gl, missing); - mbname_free(&mbname); - - if (r) goto done; - } - - /* and the shared mailboxes backup, if there were any */ - if (shared_mbname) { - r = reserve_one(shared_mbname, dl, gl, missing); - if (r) goto done; - } - - if (missing->head) { - struct dlist *kout = dlist_newlist(NULL, "MISSING"); - struct sync_msgid *msgid; - - for (msgid = missing->head; msgid; msgid = msgid->next) { - dlist_setguid(kout, "GUID", &msgid->guid); - } - - prot_printf(backupd_out, "* "); - dlist_print(kout, 1, backupd_out); - prot_printf(backupd_out, "\r\n"); - dlist_free(&kout); - } - -done: - mbname_free(&shared_mbname); - strarray_fini(&userids); - sync_msgid_list_free(&missing); - return r; -} - -static int cmd_apply_rename(struct dlist *dl) -{ - int r; - const char *old_mboxname = NULL; - const char *new_mboxname = NULL; - - if (!dlist_getatom(dl, "OLDMBOXNAME", &old_mboxname)) return IMAP_PROTOCOL_ERROR; - if (!dlist_getatom(dl, "NEWMBOXNAME", &new_mboxname)) return IMAP_PROTOCOL_ERROR; - - mbname_t *old = mbname_from_intname(old_mboxname); - mbname_t *new = mbname_from_intname(old_mboxname); - - if (strcmpnull(mbname_userid(old), mbname_userid(new)) == 0) { - // same user, unremarkable folder rename *whew* - struct open_backup *open = NULL; - r = backupd_open_backup(&open, old); - if (!r) - r = backup_append(open->backup, dl, NULL, BACKUP_APPEND_FLUSH); - } - else { - // user name has changed! - // FIXME implement this - syslog(LOG_ERR, "rename of user not yet supported: %s -> %s", - old_mboxname, new_mboxname); - r = IMAP_INTERNAL; - } - - mbname_free(&old); - mbname_free(&new); - - return r; -} - -static int cmd_apply_seen(struct dlist *dl) -{ - const char *userid = NULL; - mbname_t *mbname = NULL; - struct open_backup *open = NULL; - int r; - - if (!dlist_getatom(dl, "USERID", &userid)) return IMAP_PROTOCOL_ERROR; - - mbname = mbname_from_userid(userid); - r = backupd_open_backup(&open, mbname); - mbname_free(&mbname); - - if (r) return r; - - r = backup_append(open->backup, dl, NULL, BACKUP_APPEND_FLUSH); - - return r; -} - -static int cmd_apply_sub(struct dlist *dl) -{ - const char *userid = NULL; - const char *mboxname = NULL; - mbname_t *mbname = NULL; - struct open_backup *open = NULL; - int r; - - if (!dlist_getatom(dl, "USERID", &userid)) return IMAP_PROTOCOL_ERROR; - if (!dlist_getatom(dl, "MBOXNAME", &mboxname)) return IMAP_PROTOCOL_ERROR; - - mbname = mbname_from_userid(userid); - r = backupd_open_backup(&open, mbname); - mbname_free(&mbname); - - if (r) return r; - - r = backup_append(open->backup, dl, NULL, BACKUP_APPEND_FLUSH); - - return r; -} - -static int cmd_apply_sieve(struct dlist *dl) -{ - const char *userid = NULL; - mbname_t *mbname = NULL; - struct open_backup *open = NULL; - int r; - - if (!dlist_getatom(dl, "USERID", &userid)) return IMAP_PROTOCOL_ERROR; - - mbname = mbname_from_userid(userid); - r = backupd_open_backup(&open, mbname); - mbname_free(&mbname); - - if (r) return r; - - r = backup_append(open->backup, dl, NULL, BACKUP_APPEND_FLUSH); - - return r; -} - -static void cmd_apply(struct dlist *dl) -{ - int r; - - if (strcmp(dl->name, "MAILBOX") == 0) { - r = cmd_apply_mailbox(dl); - } - else if (strcmp(dl->name, "UNMAILBOX") == 0) { - r = cmd_apply_unmailbox(dl); - } - else if (strcmp(dl->name, "MESSAGE") == 0) { - r = cmd_apply_message(dl); - } - else if (strcmp(dl->name, "QUOTA") == 0) { - /* ignore and succeed */ - r = 0; - } - else if (strcmp(dl->name, "RENAME") == 0) { - r = cmd_apply_rename(dl); - } - else if (strcmp(dl->name, "RESERVE") == 0) { - r = cmd_apply_reserve(dl); - } - else if (strcmp(dl->name, "SEEN") == 0) { - r = cmd_apply_seen(dl); - } - else if (strcmp(dl->name, "SIEVE") == 0) { - r = cmd_apply_sieve(dl); - } - else if (strcmp(dl->name, "UNSIEVE") == 0) { - r = cmd_apply_sieve(dl); - } - else if (strcmp(dl->name, "ACTIVATE_SIEVE") == 0) { - /* ignore and succeed */ - r = 0; - } - else if (strcmp(dl->name, "UNACTIVATE_SIEVE") == 0) { - /* ignore and succeed */ - r = 0; - } - else if (strcmp(dl->name, "SUB") == 0) { - r = cmd_apply_sub(dl); - } - else if (strcmp(dl->name, "UNSUB") == 0) { - r = cmd_apply_sub(dl); - } - else if (strcmp(dl->name, "UNUSER") == 0) { - /* ignore and succeed */ - r = 0; - } - else { - r = IMAP_PROTOCOL_ERROR; - } - - syslog(LOG_DEBUG, "sending response to %s: %i (%s)", - dl->name, r, error_message(r)); - prot_printf(backupd_out, "%s\r\n", backupd_response(r)); -} - -static void cmd_compress(const char *alg) -{ - if (backupd_compress_done) { - prot_printf(backupd_out, "NO Compression already active: %s\r\n", alg); - return; - } - if (strcasecmp(alg, "DEFLATE")) { - prot_printf(backupd_out, "NO Unknown compression algorithm: %s\r\n", alg); - return; - } - if (ZLIB_VERSION[0] != zlibVersion()[0]) { - prot_printf(backupd_out, "NO Error initializing %s " - "(incompatible zlib version)\r\n", alg); - return; - } - prot_printf(backupd_out, "OK %s active\r\n", alg); - prot_flush(backupd_out); - prot_setcompress(backupd_in); - prot_setcompress(backupd_out); - backupd_compress_done = 1; -} - -static int backupd_print_mailbox(const struct backup_mailbox *mailbox, - void *rock __attribute__((__unused__))) -{ - if (mailbox->deleted) return 0; - - struct dlist *dlist = backup_mailbox_to_dlist(mailbox); - if (!dlist) return IMAP_INTERNAL; - - prot_puts(backupd_out, "* "); - dlist_print(dlist, /* printkeys */ 1, backupd_out); - prot_puts(backupd_out, "\r\n"); - dlist_free(&dlist); - - return 0; -} - -static int backupd_print_seen(const struct backup_seen *seen, - void *rock __attribute__((__unused__))) -{ - struct dlist *kl = NULL; - - kl = dlist_newkvlist(NULL, "SEEN"); - dlist_setatom(kl, "UNIQUEID", seen->uniqueid); - dlist_setdate(kl, "LASTREAD", seen->lastread); - dlist_setnum32(kl, "LASTUID", seen->lastuid); - dlist_setdate(kl, "LASTCHANGE", seen->lastchange); - dlist_setatom(kl, "SEENUIDS", seen->seenuids); - - prot_puts(backupd_out, "* "); - dlist_print(kl, 1, backupd_out); - prot_puts(backupd_out, "\r\n"); - - if (kl) dlist_free(&kl); - - return 0; -} - -static int _sublist_add(const struct backup_subscription *sub, void *rock) -{ - strarray_t *list = (strarray_t *) rock; - - if (sub->unsubscribed) return 0; - - strarray_append(list, sub->mboxname); - - return 0; -} - -static int backupd_print_subscriptions(struct backup *backup) -{ - int i, r; - strarray_t list = STRARRAY_INITIALIZER; - struct dlist *kl = NULL; - - r = backup_subscription_foreach(backup, 0, _sublist_add, &list); - if (r) goto done; - - kl = dlist_newlist(NULL, "LSUB"); - - for (i = 0; i < list.count; i++) { - const char *mboxname = strarray_nth(&list, i); - dlist_setatom(kl, "MBOXNAME", mboxname); - } - - if (kl->head) { - prot_puts(backupd_out, "* "); - dlist_print(kl, 1, backupd_out); - prot_puts(backupd_out, "\r\n"); - } - -done: - if (kl) dlist_free(&kl); - strarray_fini(&list); - return r; -} - -static int backupd_print_sieve(const struct backup_sieve *sieve, - void *rock __attribute__((__unused__))) -{ - struct dlist *kl = NULL; - - if (sieve->deleted) return 0; - - kl = dlist_newkvlist(NULL, "SIEVE"); - dlist_setatom(kl, "FILENAME", sieve->filename); - dlist_setdate(kl, "LAST_UPDATE", sieve->last_update); - dlist_setatom(kl, "GUID", message_guid_encode(&sieve->guid)); - dlist_setnum32(kl, "ISACTIVE", 0); - - prot_puts(backupd_out, "* "); - dlist_print(kl, 1, backupd_out); - prot_puts(backupd_out, "\r\n"); - - if (kl) dlist_free(&kl); - - return 0; -} - -static const char *backupd_response(int r) -{ - switch (r) { - case 0: - return "OK Success"; - case IMAP_MAILBOX_LOCKED: - return "NO IMAP_MAILBOX_LOCKED Mailbox locked"; - case IMAP_MAILBOX_NONEXISTENT: - return "NO IMAP_MAILBOX_NONEXISTENT No Such Mailbox"; - case IMAP_PROTOCOL_ERROR: - return "NO IMAP_PROTOCOL_ERROR Protocol error"; - case IMAP_PROTOCOL_BAD_PARAMETERS: - return "NO IMAP_PROTOCOL_BAD_PARAMETERS Bad parameters"; - default: - return "NO Unknown error"; - } -} - -static int cmd_get_mailbox(struct dlist *dl, int want_records) -{ - struct open_backup *open = NULL; - struct backup_mailbox *mb = NULL; - mbname_t *mbname = NULL; - int r; - - if (!dl->sval) return IMAP_PROTOCOL_BAD_PARAMETERS; - - mbname = mbname_from_intname(dl->sval); - if (!mbname) return IMAP_INTERNAL; - - r = backupd_open_backup(&open, mbname); - if (r) goto done; - - mb = backup_get_mailbox_by_name(open->backup, mbname, want_records); - if (!mb) { - r = IMAP_MAILBOX_NONEXISTENT; - goto done; - } - - backupd_print_mailbox(mb, NULL); - r = 0; - -done: - if (mb) backup_mailbox_free(&mb); - if (mbname) mbname_free(&mbname); - - return r; -} - -static int cmd_get_meta(struct dlist *dl) -{ - struct open_backup *open = NULL; - mbname_t *mbname = NULL; - int r; - - if (!dl->sval) return IMAP_PROTOCOL_BAD_PARAMETERS; - - mbname = mbname_from_userid(dl->sval); - if (!mbname) return IMAP_INTERNAL; - - r = backupd_open_backup(&open, mbname); - if (r) goto done; - - r = backup_seen_foreach(open->backup, 0, - backupd_print_seen, NULL); - if (r) goto done; - - r = backupd_print_subscriptions(open->backup); - if (r) goto done; - - r = backup_sieve_foreach(open->backup, 0, - backupd_print_sieve, NULL); - if (r) goto done; - -done: - if (mbname) mbname_free(&mbname); - return r; -} - -static int is_mailboxes_single_user(struct dlist *dl) -{ - char *userid = NULL; - struct dlist *di; - - for (di = dl->head; di; di = di->next) { - if (!di->sval) continue; - - mbname_t *mbname = mbname_from_intname(di->sval); - - if (!userid) { - userid = xstrdupnull(mbname_userid(mbname)); - } - else if (strcmpnull(userid, mbname_userid(mbname)) != 0) { - mbname_free(&mbname); - free(userid); - return 0; - } - - mbname_free(&mbname); - } - - free(userid); - /* also returns 1 if all mailboxes belong to no user (shared)*/ - return 1; -} - -static void cmd_get(struct dlist *dl) -{ - int r = IMAP_PROTOCOL_ERROR; - mbname_t *mbname = NULL; - - ucase(dl->name); - - if (strcmp(dl->name, "USER") == 0) { - struct open_backup *open = NULL; - - if (dl->sval) { - mbname = mbname_from_userid(dl->sval); - r = backupd_open_backup(&open, mbname); - if (r) goto done; - - r = backup_mailbox_foreach(open->backup, 0, - BACKUP_MAILBOX_NO_RECORDS, - backupd_print_mailbox, NULL); - if (r) goto done; - - r = backup_seen_foreach(open->backup, 0, - backupd_print_seen, NULL); - if (r) goto done; - - r = backupd_print_subscriptions(open->backup); - if (r) goto done; - - r = backup_sieve_foreach(open->backup, 0, - backupd_print_sieve, NULL); - } - else { - r = IMAP_PROTOCOL_BAD_PARAMETERS; - } - } - else if (strcmp(dl->name, "MAILBOXES") == 0) { - struct dlist *di; - if (is_mailboxes_single_user(dl)) { - for (di = dl->head; di; di = di->next) { - r = cmd_get_mailbox(di, 0); - /* it's not an error for a mailbox to not exist here */ - if (r == IMAP_MAILBOX_NONEXISTENT) r = 0; - if (r) break; - } - } - else { - /* reject MAILBOXES requests that span multiple users. - * sync_client will promote these to USER requests, which - * it always sends one at a time with a restart in between - */ - r = IMAP_PROTOCOL_BAD_PARAMETERS; - } - } - else if (strcmp(dl->name, "FULLMAILBOX") == 0) { - r = cmd_get_mailbox(dl, 1); - } - else if (strcmp(dl->name, "META") == 0) { - r = cmd_get_meta(dl); - } - else if (strcmp(dl->name, "UNIQUEIDS") == 0) { - r = 0; // we don't send anything back other than OK - } - else { - r = IMAP_PROTOCOL_ERROR; - } - -done: - - if (mbname) mbname_free(&mbname); - - syslog(LOG_DEBUG, "sending response to %s: %i (%s)", - dl->name, r, error_message(r)); - prot_printf(backupd_out, "%s\r\n", backupd_response(r)); -} - -static void cmd_restart(void) -{ - open_backups_list_close(&backupd_open_backups, 0); -} diff --git a/backup/ctl_backups.c b/backup/ctl_backups.c deleted file mode 100644 index 21e2cf3ddba..00000000000 --- a/backup/ctl_backups.c +++ /dev/null @@ -1,1119 +0,0 @@ -/* ctl_backups.c -- tool for managing replication-based backup files - * - * Copyright (c) 1994-2015 Carnegie Mellon University. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * 3. The name "Carnegie Mellon University" must not be used to - * endorse or promote products derived from this software without - * prior written permission. For permission or any legal - * details, please contact - * Carnegie Mellon University - * Center for Technology Transfer and Enterprise Creation - * 4615 Forbes Avenue - * Suite 302 - * Pittsburgh, PA 15213 - * (412) 268-7393, fax: (412) 268-7395 - * innovation@andrew.cmu.edu - * - * 4. Redistributions of any form whatsoever must retain the following - * acknowledgment: - * "This product includes software developed by Computing Services - * at Carnegie Mellon University (http://www.cmu.edu/computing/)." - * - * CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO - * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY - * AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE - * FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN - * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING - * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - * - */ - -#include - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "lib/cyrusdb.h" -#include "lib/util.h" -#include "lib/xmalloc.h" - -#include "imap/global.h" -#include "imap/imap_err.h" - -#include "backup/backup.h" - -static struct namespace ctl_backups_namespace; - -EXPORTED void fatal(const char *error, int code) -{ - fprintf(stderr, "fatal error: %s\n", error); - cyrus_done(); - exit(code); -} - -static const char *argv0 = NULL; -static void usage(void) -{ - fprintf(stderr, "Usage:\n"); - fprintf(stderr, " %s [options] compact [mode] backup...\n", argv0); - fprintf(stderr, " %s [options] list [list_opts] [[mode] backup...]\n", argv0); - fprintf(stderr, " %s [options] lock [lock_opts] [mode] backup\n", argv0); - fprintf(stderr, " %s [options] reindex [mode] backup...\n", argv0); - fprintf(stderr, " %s [options] stat [mode] backup...\n", argv0); - fprintf(stderr, " %s [options] verify [mode] backup...\n", argv0); - - fprintf(stderr, "\n%s\n", - "Commands:\n" - " compact # compact specified backups\n" - " list [list_opts] # list backups (all if none specified)\n" - " lock [lock_opts] # lock specified backup\n" - " reindex # reindex specified backups\n" - " stat # show stats of specified backups\n" - " verify # verify specified backups\n" - ); - - fprintf(stderr, "%s\n", - "Options:\n" - " -C alt_config # alternate config file\n" - " -F # force (run command even if not needed)\n" - " -S # stop on error\n" - " -V # don't verify checksums (faster read-only ops)\n" - " -j # output in JSON format\n" - " -v # verbose (repeat for more verbosity)\n" - " -w # wait for locks (don't skip locked backups)\n" - ); - - fprintf(stderr, "%s\n", - "List options:\n" - " -t [hours] # stale (no update in hours) backups only (default: 24)\n" - ); - - fprintf(stderr, "%s\n", - "Lock options:\n" - " -c # exclusively create backup\n" - " -p # lock backup and wait for eof on stdin (default)\n" - " -s # lock backup and open index in sqlite3\n" - " -x command # lock backup and execute command\n" - ); - - fprintf(stderr, "%s\n", - "Modes:\n" - " -A # all known backups\n" - " -D # specified backups interpreted as domains\n" - " -P # specified backups interpreted as userid prefixes\n" - " -f # specified backups interpreted as filenames\n" - " -m # specified backups interpreted as mboxnames\n" - " -u # specified backups interpreted as userids (default)\n" - "\n" - " Modes -A, -D, -P not available for all commands\n" /* FIXME which */ - ); - - exit(EX_USAGE); -} - -enum ctlbu_mode { - CTLBU_MODE_UNSPECIFIED = 0, - CTLBU_MODE_FILENAME, - CTLBU_MODE_MBOXNAME, - CTLBU_MODE_USERNAME, - CTLBU_MODE_DOMAIN, - CTLBU_MODE_PREFIX, - CTLBU_MODE_ALL, -}; - -enum ctlbu_lock_mode { - CTLBU_LOCK_MODE_UNSPECIFIED = 0, - CTLBU_LOCK_MODE_PIPE, - CTLBU_LOCK_MODE_SQL, - CTLBU_LOCK_MODE_EXEC, -}; - -struct ctlbu_cmd_options { - enum ctlbu_mode mode; - enum ctlbu_lock_mode lock_mode; - enum backup_open_nonblock wait; - enum backup_open_create create; - int verbose; - int stop_on_error; - int list_stale; - int force; - int noverify; - int jsonout; - const char *lock_exec_cmd; - const char *domain; -}; - -enum ctlbu_cmd { - CTLBU_CMD_UNSPECIFIED = 0, - CTLBU_CMD_COMPACT, - CTLBU_CMD_DELETE, - CTLBU_CMD_LIST, - CTLBU_CMD_LOCK, - CTLBU_CMD_MOVE, - CTLBU_CMD_RECONSTRUCT, - CTLBU_CMD_REINDEX, - CTLBU_CMD_STAT, - CTLBU_CMD_VERIFY, -}; - -static int ctlbu_skips_fails = 0; - -/* same signature as foreach_cb */ -static int cmd_compact_one(void *rock, - const char *userid, size_t userid_len, - const char *fname, size_t fname_len); -static int cmd_delete_one(void *rock, - const char *userid, size_t userid_len, - const char *fname, size_t fname_len); -static int cmd_list_one(void *rock, - const char *userid, size_t userid_len, - const char *fname, size_t fname_len); -static int cmd_lock_one(void *rock, - const char *userid, size_t userid_len, - const char *fname, size_t fname_len); -static int cmd_move_one(void *rock, - const char *userid, size_t userid_len, - const char *fname, size_t fname_len); -static int cmd_reindex_one(void *rock, - const char *userid, size_t userid_len, - const char *fname, size_t fname_len); -static int cmd_stat_one(void *rock, - const char *userid, size_t userid_len, - const char *fname, size_t fname_len); -static int cmd_verify_one(void *rock, - const char *userid, size_t userid_len, - const char *fname, size_t fname_len); - -static foreach_cb *const cmd_func[] = { - NULL, - cmd_compact_one, - cmd_delete_one, - cmd_list_one, - cmd_lock_one, - cmd_move_one, - NULL, /* reconstruct one doesn't make sense */ - cmd_reindex_one, - cmd_stat_one, - cmd_verify_one, -}; - -static int lock_run_pipe(const char *userid, const char *fname, - enum backup_open_nonblock nonblock, - enum backup_open_create create); -static int lock_run_sqlite(const char *userid, const char *fname, - enum backup_open_nonblock nonblock, - enum backup_open_create create); -static int lock_run_exec(const char *userid, const char *fname, - const char *cmd, - enum backup_open_nonblock nonblock, - enum backup_open_create create); - -static enum ctlbu_cmd parse_cmd_string(const char *cmd) -{ - assert(cmd != NULL); - - switch(cmd[0]) { - case 'c': - if (strcmp(cmd, "compact") == 0) return CTLBU_CMD_COMPACT; - break; - case 'd': - if (strcmp(cmd, "delete") == 0) return CTLBU_CMD_DELETE; - break; - case 'l': - if (strcmp(cmd, "list") == 0) return CTLBU_CMD_LIST; - if (strcmp(cmd, "lock") == 0) return CTLBU_CMD_LOCK; - break; - case 'm': - if (strcmp(cmd, "move") == 0) return CTLBU_CMD_MOVE; - break; - case 'r': - if (strcmp(cmd, "reconstruct") == 0) return CTLBU_CMD_RECONSTRUCT; - if (strcmp(cmd, "reindex") == 0) return CTLBU_CMD_REINDEX; - break; - case 's': - if (strcmp(cmd, "stat") == 0) return CTLBU_CMD_STAT; - break; - case 'v': - if (strcmp(cmd, "verify") == 0) return CTLBU_CMD_VERIFY; - break; - }; - - return CTLBU_CMD_UNSPECIFIED; -} - -static void print_status(const char *cmd, - const char *userid, const char *fname, - const struct ctlbu_cmd_options *options, - int r) -{ - char *status = NULL; - switch (r) { - case 1: - status = xstrdup("skipped"); - break; - case 0: - status = xstrdup("ok"); - break; - case IMAP_MAILBOX_LOCKED: - status = xstrdup("locked"); - break; - default: - status = strconcat("failed (", error_message(r), ")", NULL); - break; - } - - if (options->jsonout) { - json_t *out = json_object(); - json_object_set_new(out, "command", json_string(cmd)); - json_object_set_new(out, "userid", json_string(userid)); - json_object_set_new(out, "fname", json_string(fname)); - json_object_set_new(out, "status", json_string(status)); - - const size_t flags = JSON_INDENT(2) | JSON_PRESERVE_ORDER; - json_dumpf(out, stdout, flags); - puts(""); - json_decref(out); - } - else { - printf("%s %s: %s\n", cmd, userid ? userid : fname, status); - } - - free(status); -} - -static int domain_filter(void *rock, - const char *key, size_t key_len, - const char *data __attribute__((unused)), - size_t data_len __attribute__((unused))) -{ - struct ctlbu_cmd_options *options = (struct ctlbu_cmd_options *) rock; - char *userid = NULL; - mbname_t *mbname = NULL; - const char *domain = NULL; - int doit = 0; - - /* input args might not be 0-terminated, so make a safe copy */ - if (!key_len) return 0; - userid = xstrndup(key, key_len); - - mbname = mbname_from_userid(userid); - domain = mbname_domain(mbname); - if (!domain) - domain = config_defdomain; - - if (domain) - doit = !strcmp(domain, options->domain); - - mbname_free(&mbname); - free(userid); - - return doit; -} - -static void save_argv0(const char *s) -{ - const char *slash = strrchr(s, '/'); - if (slash) - argv0 = slash + 1; - else - argv0 = s; -} - -int main(int argc, char **argv) -{ - save_argv0(argv[0]); - - int opt, r = 0; - const char *alt_config = NULL; - enum ctlbu_cmd cmd = CTLBU_CMD_UNSPECIFIED; - struct ctlbu_cmd_options options = {0}; - options.wait = BACKUP_OPEN_NONBLOCK; - - /* keep in alphabetical order */ - static const char short_options[] = ":AC:DFPSVcfjmpst:uvwx:"; - - static const struct option long_options[] = { - { "all", no_argument, NULL, 'A' }, - /* n.b. no long-option for -C */ - { "domains", no_argument, NULL, 'D' }, - { "force", no_argument, NULL, 'F' }, - { "prefixes", no_argument, NULL, 'P' }, - { "stop-on-error", no_argument, NULL, 'S' }, - { "no-verify", no_argument, NULL, 'V' }, - { "create", no_argument, NULL, 'c' }, - { "filenames", no_argument, NULL, 'f' }, - { "json", no_argument, NULL, 'j' }, - { "mailboxes", no_argument, NULL, 'm' }, - { "pause", no_argument, NULL, 'p' }, - { "sqlite3", no_argument, NULL, 's' }, - { "stale", optional_argument, NULL, 't' }, - { "userids", no_argument, NULL, 'u' }, - { "verbose", no_argument, NULL, 'v' }, - { "wait-for-locks", no_argument, NULL, 'w' }, - { "execute", required_argument, NULL, 'x' }, - - { 0, 0, 0, 0 }, - }; - - while (-1 != (opt = getopt_long(argc, argv, - short_options, long_options, NULL))) - { - switch (opt) { - case 'A': - if (options.mode != CTLBU_MODE_UNSPECIFIED) usage(); - options.mode = CTLBU_MODE_ALL; - break; - case 'C': - alt_config = optarg; - break; - case 'D': - if (options.mode != CTLBU_MODE_UNSPECIFIED) usage(); - options.mode = CTLBU_MODE_DOMAIN; - break; - case 'F': - options.force = 1; - break; - case 'P': - if (options.mode != CTLBU_MODE_UNSPECIFIED) usage(); - options.mode = CTLBU_MODE_PREFIX; - break; - case 'S': - options.stop_on_error = 1; - break; - case 'V': - options.noverify = 1; - break; - case 'c': - options.create = BACKUP_OPEN_CREATE_EXCL; - break; - case 'j': - if (options.verbose) usage(); - options.jsonout = 1; - break; - case 'f': - if (options.mode != CTLBU_MODE_UNSPECIFIED) usage(); - options.mode = CTLBU_MODE_FILENAME; - break; - case 'm': - if (options.mode != CTLBU_MODE_UNSPECIFIED) usage(); - options.mode = CTLBU_MODE_MBOXNAME; - break; - case 'p': - if (options.lock_mode != CTLBU_LOCK_MODE_UNSPECIFIED) usage(); - options.lock_mode = CTLBU_LOCK_MODE_PIPE; - break; - case 's': - if (options.lock_mode != CTLBU_LOCK_MODE_UNSPECIFIED) usage(); - options.lock_mode = CTLBU_LOCK_MODE_SQL; - break; - case 't': - options.list_stale = atoi(optarg); - if (!options.list_stale) usage(); - break; - case 'u': - if (options.mode != CTLBU_MODE_UNSPECIFIED) usage(); - options.mode = CTLBU_MODE_USERNAME; - break; - case 'v': - if (options.jsonout) usage(); - options.verbose ++; - break; - case 'x': - if (options.lock_mode != CTLBU_LOCK_MODE_UNSPECIFIED) usage(); - options.lock_mode = CTLBU_LOCK_MODE_EXEC; - options.lock_exec_cmd = optarg; - break; - case 'w': - options.wait = BACKUP_OPEN_BLOCK; - break; - case ':': - if (optopt == 't') options.list_stale = 24; - else usage(); - break; - default: - usage(); - break; - } - } - - /* get the command */ - if (optind == argc) usage(); - cmd = parse_cmd_string(argv[optind++]); - if (cmd == CTLBU_CMD_UNSPECIFIED) usage(); - - if (options.lock_mode != CTLBU_LOCK_MODE_UNSPECIFIED - && cmd != CTLBU_CMD_LOCK) - usage(); - - switch (cmd) { - /* list defaults to all */ - case CTLBU_CMD_LIST: - if (options.mode == CTLBU_MODE_UNSPECIFIED && argc - optind == 0) - options.mode = CTLBU_MODE_ALL; - break; - - /* some commands only accept one backup at a time */ - case CTLBU_CMD_LOCK: - case CTLBU_CMD_MOVE: - case CTLBU_CMD_DELETE: - if (options.mode == CTLBU_MODE_ALL) usage(); - if (options.mode == CTLBU_MODE_DOMAIN) usage(); - if (options.mode == CTLBU_MODE_PREFIX) usage(); - if (argc - optind > 1) usage(); - break; - - /* reconstruct doesn't accept named backups */ - case CTLBU_CMD_RECONSTRUCT: - if (options.mode != CTLBU_MODE_UNSPECIFIED) usage(); - if (optind != argc) usage(); - break; - - default: - break; - } - - /* default mode is username */ - if (options.mode == CTLBU_MODE_UNSPECIFIED) - options.mode = CTLBU_MODE_USERNAME; - - /* mode all doesn't want any named backups */ - if (options.mode == CTLBU_MODE_ALL && optind != argc) usage(); - - cyrus_init(alt_config, "ctl_backups", 0, 0); - - if ((r = mboxname_init_namespace(&ctl_backups_namespace, NAMESPACE_OPTION_ADMIN))) { - fatal(error_message(r), EX_CONFIG); - } - mboxevent_setnamespace(&ctl_backups_namespace); - - if (cmd == CTLBU_CMD_RECONSTRUCT) { - /* special handling for reconstruct */ - // FIXME - } - else if (options.mode == CTLBU_MODE_ALL) { - /* loop over entire backups.db */ - struct db *backups_db = NULL; - - r = backupdb_open(&backups_db, NULL); - - if (!r) - r = cyrusdb_foreach(backups_db, NULL, 0, NULL, - cmd_func[cmd], &options, - NULL); - - if (backups_db) - cyrusdb_close(backups_db); - } - else if (options.mode == CTLBU_MODE_DOMAIN) { - /* loop over domains named on command line */ - struct db *backups_db = NULL; - int i; - - r = backupdb_open(&backups_db, NULL); - - for (i = optind; i < argc && !r; i++) { - options.domain = argv[i]; - - r = cyrusdb_foreach(backups_db, NULL, 0, - domain_filter, - cmd_func[cmd], &options, - NULL); - } - - if (backups_db) - cyrusdb_close(backups_db); - } - else if (options.mode == CTLBU_MODE_PREFIX) { - /* loop over prefixes named on command line */ - struct db *backups_db = NULL; - int i; - - r = backupdb_open(&backups_db, NULL); - - for (i = optind; i < argc && !r; i++) { - r = cyrusdb_foreach(backups_db, - argv[i], strlen(argv[i]), - NULL, - cmd_func[cmd], &options, - NULL); - } - - if (backups_db) - cyrusdb_close(backups_db); - } - else { - /* loop over backups named on command line */ - struct buf userid = BUF_INITIALIZER; - struct buf fname = BUF_INITIALIZER; - int i; - - for (i = optind; i < argc; i++) { - buf_reset(&userid); - buf_reset(&fname); - mbname_t *mbname = NULL; - - if (options.mode == CTLBU_MODE_USERNAME) - mbname = mbname_from_userid(argv[i]); - else if (options.mode == CTLBU_MODE_MBOXNAME) - mbname = mbname_from_extname(argv[i], &ctl_backups_namespace, NULL); - - if (mbname) { - r = backup_get_paths(mbname, &fname, NULL, BACKUP_OPEN_NOCREATE); - if (r) { - /* XXX this should print cyrusdb_strerror(r), except that - * just returns "cyrusdb error", unhelpfully */ - fprintf(stderr, "%s: not found in backups.db\n", argv[i]); - syslog(LOG_DEBUG, "ctl_backups '%s': %s (%d)", argv[i], cyrusdb_strerror(r), r); - mbname_free(&mbname); - continue; - } - if (mbname_userid(mbname)) { - buf_setcstr(&userid, mbname_userid(mbname)); - } - } - else - buf_setcstr(&fname, argv[i]); - - if (!r && cmd_func[cmd]) - r = cmd_func[cmd](&options, - buf_cstring(&userid), - buf_len(&userid), - buf_cstring(&fname), - buf_len(&fname)); - - if (mbname) mbname_free(&mbname); - - if (r) break; - } - - buf_free(&userid); - buf_free(&fname); - } - - backup_cleanup_staging_path(); - cyrus_done(); - exit(r || ctlbu_skips_fails ? EX_TEMPFAIL : EX_OK); -} - -static int cmd_compact_one(void *rock, - const char *key, size_t key_len, - const char *data, size_t data_len) -{ - struct ctlbu_cmd_options *options = (struct ctlbu_cmd_options *) rock; - char *userid = NULL; - char *fname = NULL; - int r = 0; - - /* input args might not be 0-terminated, so make a safe copy */ - if (key_len) - userid = xstrndup(key, key_len); - if (data_len) - fname = xstrndup(data, data_len); - - r = backup_compact(fname, options->wait, options->force, - options->verbose, stdout); - - print_status("compact", userid, fname, options, r); - - if (userid) free(userid); - if (fname) free(fname); - - if (r) ++ctlbu_skips_fails; - if (r == IMAP_MAILBOX_LOCKED) r = 0; - return options->stop_on_error ? r : 0; -} - -static int cmd_delete_one(void *rock, - const char *userid, size_t userid_len, - const char *fname, size_t fname_len) -{ - struct ctlbu_cmd_options *options = (struct ctlbu_cmd_options *) rock; - (void) options; - fprintf(stderr, "unimplemented: %s %s[%zu] %s[%zu]\n", __func__, - userid, userid_len, fname, fname_len); - return -1; -} - -static int cmd_list_one(void *rock, - const char *key, size_t key_len, - const char *data, size_t data_len) -{ - struct ctlbu_cmd_options *options = (struct ctlbu_cmd_options *) rock; - struct backup *backup = NULL; - struct backup_chunk *latest_chunk = NULL; - struct stat data_stat_buf = {0}; - char *userid = NULL; - char *fname = NULL; - char timestamp[32] = "[unknown]"; - int r = 0; - - /* input args might not be 0-terminated, so make a safe copy */ - if (key_len) - userid = xstrndup(key, key_len); - if (data_len) - fname = xstrndup(data, data_len); - - r = backup_open_paths(&backup, fname, NULL, - options->wait, BACKUP_OPEN_NOCREATE); - if (!r && !options->noverify) - r = backup_verify(backup, BACKUP_VERIFY_QUICK, 0, NULL); - - if (r) { - fprintf(stderr, "%s: %s\n", userid ? userid : fname, error_message(r)); - goto done; - } - - latest_chunk = backup_get_latest_chunk(backup); - if (latest_chunk) { - if (options->list_stale) { - /* skip out early if it's not stale */ - if (time(NULL) - latest_chunk->ts_end < 3600 * options->list_stale) - goto done; - } - - strftime(timestamp, sizeof(timestamp), "%F %T", - localtime(&latest_chunk->ts_end)); - } - - r = backup_stat(backup, &data_stat_buf, NULL); - if (r) { - fprintf(stderr, "fstat %s: %s\n", userid ? userid : fname, strerror(errno)); - data_stat_buf.st_size = -1; - } - - if (options->jsonout) { - json_t *out = json_object(); - json_object_set_new(out, "timestamp", json_string(timestamp)); - json_object_set_new(out, "compressed", json_integer(data_stat_buf.st_size)); - json_object_set_new(out, "userid", json_string(userid)); - json_object_set_new(out, "fname", json_string(fname)); - - const size_t flags = JSON_INDENT(2) | JSON_PRESERVE_ORDER; - json_dumpf(out, stdout, flags); - puts(""); - json_decref(out); - } - else { - printf("%s\t" OFF_T_FMT "\t%s\t%s\n", - timestamp, - data_stat_buf.st_size, - userid ? userid : "[unknown]", - fname); - } - -done: - if (latest_chunk) backup_chunk_free(&latest_chunk); - if (backup) backup_close(&backup); - if (userid) free(userid); - if (fname) free(fname); - - if (r) ++ctlbu_skips_fails; - if (r == IMAP_MAILBOX_LOCKED) r = 0; - return options->stop_on_error ? r : 0; -} - -static int cmd_lock_one(void *rock, - const char *key, size_t key_len, - const char *data, size_t data_len) -{ - struct ctlbu_cmd_options *options = (struct ctlbu_cmd_options *) rock; - char *userid = NULL; - char *fname = NULL; - int r = 0; - - assert(data != NULL && data_len > 0); - - /* input args might not be 0-terminated, so make a safe copy */ - if (key_len) - userid = xstrndup(key, key_len); - if (data_len) - fname = xstrndup(data, data_len); - - switch (options->lock_mode) { - case CTLBU_LOCK_MODE_UNSPECIFIED: - case CTLBU_LOCK_MODE_PIPE: - r = lock_run_pipe(userid, fname, options->wait, options->create); - break; - case CTLBU_LOCK_MODE_SQL: - r = lock_run_sqlite(userid, fname, options->wait, options->create); - break; - case CTLBU_LOCK_MODE_EXEC: - r = lock_run_exec(userid, fname, options->lock_exec_cmd, - options->wait, options->create); - break; - } - - if (userid) free(userid); - if (fname) free(fname); - - /* don't care about stop_on_error: lock command only accepts one backup */ - if (r) ++ctlbu_skips_fails; - return r; -} - -static int cmd_move_one(void *rock, - const char *userid, size_t userid_len, - const char *fname, size_t fname_len) -{ - struct ctlbu_cmd_options *options = (struct ctlbu_cmd_options *) rock; - (void) options; - fprintf(stderr, "unimplemented: %s %s[%zu] %s[%zu]\n", __func__, - userid, userid_len, fname, fname_len); - return -1; -} - -static int cmd_reindex_one(void *rock, - const char *key, size_t key_len, - const char *data, size_t data_len) -{ - struct ctlbu_cmd_options *options = (struct ctlbu_cmd_options *) rock; - char *userid = NULL; - char *fname = NULL; - int r; - - /* input args might not be 0-terminated, so make a safe copy */ - if (key_len) - userid = xstrndup(key, key_len); - if (data_len) - fname = xstrndup(data, data_len); - - r = backup_reindex(fname, options->wait, options->verbose, stdout); - - print_status("reindex", userid, fname, options, r); - - if (userid) free(userid); - if (fname) free(fname); - - if (r) ++ctlbu_skips_fails; - if (r == IMAP_MAILBOX_LOCKED) r = 0; - return options->stop_on_error ? r : 0; -} - -static int _sum_message_lengths_cb(const struct backup_message *message, - void *rock) -{ - size_t *sum = (size_t *) rock; - - *sum += message->length; - - return 0; -} - -static int cmd_stat_one(void *rock, - const char *key, size_t key_len, - const char *data, size_t data_len) -{ - struct ctlbu_cmd_options *options = (struct ctlbu_cmd_options *) rock; - struct backup *backup = NULL; - char *userid = NULL; - char *fname = NULL; - struct backup_chunk_list *all_chunks = NULL, *keep_chunks = NULL; - struct backup_chunk *latest_chunk = NULL; - struct backup_chunk *chunk; - time_t since; - size_t compactable = 0, uncompressed = 0; - struct stat data_stat; - double cmp_ratio, utl_ratio; - char start_time[32] = "[unknown]"; - char end_time[32] = "[unknown]"; - int r; - - /* input args might not be 0-terminated, so make a safe copy */ - if (key_len) - userid = xstrndup(key, key_len); - if (data_len) - fname = xstrndup(data, data_len); - - r = backup_open_paths(&backup, fname, NULL, - options->wait, BACKUP_OPEN_NOCREATE); - if (r) goto done; - - if (!options->noverify) { - r = backup_verify(backup, BACKUP_VERIFY_QUICK, 0, NULL); - if (r) goto done; - } - - const int retention = config_getduration(IMAPOPT_BACKUP_RETENTION, 'd'); - if (retention > 0) { - since = time(0) - retention; - } - else { - /* zero or negative retention means "keep forever" */ - since = -1; - } - - all_chunks = backup_get_chunks(backup); - keep_chunks = backup_get_live_chunks(backup, since); - - if (!all_chunks || !keep_chunks) { - r = IMAP_INTERNAL; - goto done; - } - - /* uncompressed length is sum of lengths of all chunks */ - for (chunk = all_chunks->head; chunk; chunk = chunk->next) { - uncompressed += chunk->length; - } - - /* compactable length is sum of live message lengths in keep chunks */ - for (chunk = keep_chunks->head; chunk; chunk = chunk->next) { - backup_message_foreach(backup, chunk->id, &since, - _sum_message_lengths_cb, &compactable); - } - - /* compression ratio is compressed length / uncompressed length */ - r = backup_stat(backup, &data_stat, NULL); - if (r) goto done; - cmp_ratio = 100.0 * data_stat.st_size / uncompressed; - - /* utilisation ratio is compactable length / uncompressed length */ - utl_ratio = 100.0 * compactable / uncompressed; - - /* start/end time are from latest chunk */ - latest_chunk = backup_get_latest_chunk(backup); - if (latest_chunk) { - strftime(start_time, sizeof(start_time), "%F %T", - localtime(&latest_chunk->ts_start)); - strftime(end_time, sizeof(end_time), "%F %T", - localtime(&latest_chunk->ts_end)); - } - - if (options->jsonout) { - json_t *out = json_object(); - json_object_set_new(out, "userid", json_string(userid)); - json_object_set_new(out, "fname", json_string(fname)); - json_object_set_new(out, "compressed", json_integer(data_stat.st_size)); - json_object_set_new(out, "uncompressed", json_integer(uncompressed)); - json_object_set_new(out, "compactable", json_integer(compactable)); - json_object_set_new(out, "compression_ratio", json_real(cmp_ratio)); - json_object_set_new(out, "utilisation_ratio", json_real(utl_ratio)); - json_object_set_new(out, "last_start_time", json_string(start_time)); - json_object_set_new(out, "last_end_time", json_string(end_time)); - - const size_t flags = JSON_INDENT(2) | JSON_PRESERVE_ORDER; - json_dumpf(out, stdout, flags); - puts(""); - json_decref(out); - } - else { - printf("%s\t" OFF_T_FMT "\t" SIZE_T_FMT "\t" SIZE_T_FMT "\t%6.1f%%\t%6.1f%%\t%21s\t%s\n", - userid ? userid : fname, - data_stat.st_size, - uncompressed, - compactable, - cmp_ratio, - utl_ratio, - start_time, - end_time); - } - -done: - if (r) { - fprintf(stderr, "%s: %s\n", userid ? userid : fname, error_message(r)); - } - - if (latest_chunk) backup_chunk_free(&latest_chunk); - if (all_chunks) backup_chunk_list_free(&all_chunks); - if (keep_chunks) backup_chunk_list_free(&keep_chunks); - if (backup) backup_close(&backup); - if (fname) free(fname); - if (userid) free(userid); - - return options->stop_on_error ? r : 0; -} - -static int cmd_verify_one(void *rock, - const char *key, size_t key_len, - const char *data, size_t data_len) -{ - struct ctlbu_cmd_options *options = (struct ctlbu_cmd_options *) rock; - struct backup *backup = NULL; - char *userid = NULL; - char *fname = NULL; - int r; - - /* input args might not be 0-terminated, so make a safe copy */ - if (key_len) - userid = xstrndup(key, key_len); - if (data_len) - fname = xstrndup(data, data_len); - - r = backup_open_paths(&backup, fname, NULL, - options->wait, BACKUP_OPEN_NOCREATE); - - /* n.b. deliberately ignoring nonsensical noverify option here */ - if (!r) r = backup_verify(backup, BACKUP_VERIFY_FULL, options->verbose, stdout); - - print_status("verify", userid, fname, options, r); - - if (backup) backup_close(&backup); - if (userid) free(userid); - if (fname) free(fname); - - if (r) ++ctlbu_skips_fails; - if (r == IMAP_MAILBOX_LOCKED) r = 0; - return options->stop_on_error ? r : 0; -} - -static int lock_run_pipe(const char *userid, const char *fname, - enum backup_open_nonblock nonblock, - enum backup_open_create create) -{ - printf("* Trying to obtain lock on %s...\n", userid ? userid : fname); - - struct backup *backup = NULL; - int r; - - r = backup_open_paths(&backup, fname, NULL, nonblock, create); - - if (!r) - r = backup_verify(backup, BACKUP_VERIFY_QUICK, 0, NULL); - - if (r) { - printf("NO failed (%s)\n", error_message(r)); - r = backup_close(&backup); - return EX_SOFTWARE; // FIXME would something else be more appropriate? - } - - printf("OK locked\n"); - - /* wait until stdin closes */ - char buf[PROT_BUFSIZE] = {0}; - while (!feof(stdin)) { - if (!fgets(buf, sizeof(buf), stdin)) - break; - } - - r = backup_close(&backup); - if (r) fprintf(stderr, "warning: backup_close() returned %i\n", r); - - return 0; -} - -static int lock_run_sqlite(const char *userid, const char *fname, - enum backup_open_nonblock nonblock, - enum backup_open_create create) -{ - fprintf(stderr, "trying to obtain lock on %s...\n", userid ? userid : fname); - - struct backup *backup = NULL; - const char *index_fname = NULL; - int r, status; - pid_t pid; - - r = backup_open_paths(&backup, fname, NULL, nonblock, create); - - if (!r) - r = backup_verify(backup, BACKUP_VERIFY_QUICK, 0, NULL); - - if (r) { - fprintf(stderr, "unable to lock %s: %s\n", - userid ? userid : fname, - error_message(r)); - r = backup_close(&backup); - return EX_SOFTWARE; - } - - index_fname = backup_get_index_fname(backup); - - /* FIXME probably need to do something with signals here */ - - pid = fork(); - - switch (pid) { - case -1: - perror("fork"); - r = EX_SOFTWARE; - break; - - case 0: - /* child */ - fprintf(stderr, "execlp: %s %s\n", "sqlite3", index_fname); - execlp("sqlite3", "sqlite3", index_fname, NULL); - /* execlp never returns */ - perror("execlp sqlite3"); - _exit(EX_SOFTWARE); - break; - - default: - /* parent */ - waitpid(pid, &status, 0); - if (WIFEXITED(status)) - r = WEXITSTATUS(status); - else - r = EX_SOFTWARE; - break; - } - - backup_close(&backup); - return r; -} - -static const char data_fname_env[] = "ctl_backups_lock_data_fname"; -static const char index_fname_env[] = "ctl_backups_lock_index_fname"; - -static int lock_run_exec(const char *userid, const char *fname, - const char *cmd, - enum backup_open_nonblock nonblock, - enum backup_open_create create) -{ - fprintf(stderr, "trying to obtain lock on %s...\n", userid ? userid : fname); - - struct backup *backup = NULL; - int r; - - r = backup_open_paths(&backup, fname, NULL, nonblock, create); - - if (!r) - r = backup_verify(backup, BACKUP_VERIFY_QUICK, 0, NULL); - - if (r) { - fprintf(stderr, "unable to lock %s: %s\n", - userid ? userid : fname, - error_message(r)); - r = backup_close(&backup); - return EX_SOFTWARE; - } - - setenv(data_fname_env, fname, 1); - setenv(index_fname_env, backup_get_index_fname(backup), 1); - - r = system(cmd); - - unsetenv(data_fname_env); - unsetenv(index_fname_env); - - if (r == -1) - r = EX_SOFTWARE; - else if (WIFEXITED(r)) - r = WEXITSTATUS(r); - else - r = EX_SOFTWARE; - - backup_close(&backup); - return r; -} diff --git a/backup/cyr_backup.c b/backup/cyr_backup.c deleted file mode 100644 index 7139bcad4b7..00000000000 --- a/backup/cyr_backup.c +++ /dev/null @@ -1,1009 +0,0 @@ -/* cyr_backup.c -- tool for examining replication-based backup files - * - * Copyright (c) 1994-2016 Carnegie Mellon University. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * 3. The name "Carnegie Mellon University" must not be used to - * endorse or promote products derived from this software without - * prior written permission. For permission or any legal - * details, please contact - * Carnegie Mellon University - * Center for Technology Transfer and Enterprise Creation - * 4615 Forbes Avenue - * Suite 302 - * Pittsburgh, PA 15213 - * (412) 268-7393, fax: (412) 268-7395 - * innovation@andrew.cmu.edu - * - * 4. Redistributions of any form whatsoever must retain the following - * acknowledgment: - * "This product includes software developed by Computing Services - * at Carnegie Mellon University (http://www.cmu.edu/computing/)." - * - * CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO - * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY - * AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE - * FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN - * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING - * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - * - */ - -#include - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "lib/cyrusdb.h" -#include "lib/gzuncat.h" -#include "lib/map.h" -#include "lib/strarray.h" -#include "lib/util.h" - -#include "imap/global.h" -#include "imap/imap_err.h" -#include "imap/json_support.h" - -#include "backup/backup.h" - -static struct namespace cyr_backup_namespace; - -EXPORTED void fatal(const char *error, int code) -{ - fprintf(stderr, "fatal error: %s\n", error); - cyrus_done(); - exit(code); -} - -static const char *argv0 = NULL; -static void usage(void) -{ - fprintf(stderr, "Usage:\n"); - fprintf(stderr, " %s [options] [mode] backup list chunks\n", argv0); - fprintf(stderr, " %s [options] [mode] backup list mailboxes\n", argv0); - fprintf(stderr, " %s [options] [mode] backup list messages\n", argv0); - fprintf(stderr, " %s [options] [mode] backup list all\n", argv0); - fprintf(stderr, " %s [options] [mode] backup show chunks id...\n", argv0); - fprintf(stderr, " %s [options] [mode] backup show mailboxes [mboxname | uniqueid]...\n", argv0); - fprintf(stderr, " %s [options] [mode] backup show messages guid...\n", argv0); - fprintf(stderr, " %s [options] [mode] backup dump chunk id\n", argv0); - fprintf(stderr, " %s [options] [mode] backup dump message guid\n", argv0); - fprintf(stderr, " %s [options] [mode] backup json chunks\n", argv0); - fprintf(stderr, " %s [options] [mode] backup json mailboxes\n", argv0); - fprintf(stderr, " %s [options] [mode] backup json messages\n", argv0); - fprintf(stderr, " %s [options] [mode] backup json headers guid...\n", argv0); - - fprintf(stderr, "\n%s\n", - "Commands:\n" - " list # list specified items\n" - " show # show detailed information for specified items\n" - " dump # show entire item\n" - " json # show detailed information in json format\n" - ); - - fprintf(stderr, "%s\n", - "Options:\n" - " -C alt_config # alternate config file\n" - " -v # verbose\n" - ); - - fprintf(stderr, "%s\n", - "Modes:\n" - " -f # specified backup interpreted as filename\n" - " -m # specified backup interpreted as mboxname\n" - " -u # specified backup interpreted as userid (default)\n" - ); - - exit(EX_USAGE); -} - -static void save_argv0(const char *s) -{ - const char *slash = strrchr(s, '/'); - if (slash) - argv0 = slash + 1; - else - argv0 = s; -} - -enum cyrbu_mode { - CYRBU_MODE_UNSPECIFIED = 0, - CYRBU_MODE_FILENAME, - CYRBU_MODE_MBOXNAME, - CYRBU_MODE_USERNAME, -}; - -struct cyrbu_cmd_options { - strarray_t *argv; - int verbose; -}; - -typedef int cyrbu_cmd_func(struct backup *, - const struct cyrbu_cmd_options *); - -static int cmd_list_all(struct backup *backup, - const struct cyrbu_cmd_options *options); -static int cmd_list_chunks(struct backup *backup, - const struct cyrbu_cmd_options *options); -static int cmd_list_mailboxes(struct backup *backup, - const struct cyrbu_cmd_options *options); -static int cmd_list_messages(struct backup *backup, - const struct cyrbu_cmd_options *options); -static int cmd_show_chunks(struct backup *backup, - const struct cyrbu_cmd_options *options); -static int cmd_show_mailboxes(struct backup *backup, - const struct cyrbu_cmd_options *options); -static int cmd_show_messages(struct backup *backup, - const struct cyrbu_cmd_options *options); -static int cmd_dump_chunk(struct backup *backup, - const struct cyrbu_cmd_options *options); -static int cmd_dump_mailbox(struct backup *backup, - const struct cyrbu_cmd_options *options); -static int cmd_dump_message(struct backup *backup, - const struct cyrbu_cmd_options *options); -static int cmd_json_chunks(struct backup *backup, - const struct cyrbu_cmd_options *options); -static int cmd_json_mailboxes(struct backup *backup, - const struct cyrbu_cmd_options *options); -static int cmd_json_messages(struct backup *backup, - const struct cyrbu_cmd_options *options); -static int cmd_json_headers(struct backup *backup, - const struct cyrbu_cmd_options *options); - -enum cyrbu_cmd { - CYRBU_CMD_UNSPECIFIED = 0, - CYRBU_CMD_LIST_ALL, - CYRBU_CMD_LIST_CHUNKS, - CYRBU_CMD_LIST_MAILBOXES, - CYRBU_CMD_LIST_MESSAGES, - CYRBU_CMD_SHOW_CHUNKS, - CYRBU_CMD_SHOW_MAILBOXES, - CYRBU_CMD_SHOW_MESSAGES, - CYRBU_CMD_DUMP_CHUNK, - CYRBU_CMD_DUMP_MAILBOX, - CYRBU_CMD_DUMP_MESSAGE, - CYRBU_CMD_JSON_CHUNKS, - CYRBU_CMD_JSON_MAILBOXES, - CYRBU_CMD_JSON_MESSAGES, - CYRBU_CMD_JSON_HEADERS, -}; - -static cyrbu_cmd_func *const cmd_func[] = { - NULL, - cmd_list_all, - cmd_list_chunks, - cmd_list_mailboxes, - cmd_list_messages, - cmd_show_chunks, - cmd_show_mailboxes, - cmd_show_messages, - cmd_dump_chunk, - cmd_dump_mailbox, - cmd_dump_message, - cmd_json_chunks, - cmd_json_mailboxes, - cmd_json_messages, - cmd_json_headers, -}; - -static enum cyrbu_cmd parse_cmd_string(const char *command, const char *sub) -{ - switch (command[0]) { - case 'd': - if (strcmp(command, "dump") == 0) { - if (strcmp(sub, "chunk") == 0) return CYRBU_CMD_DUMP_CHUNK; - else if (strcmp(sub, "mailbox") == 0) return CYRBU_CMD_DUMP_MAILBOX; - else if (strcmp(sub, "message") == 0) return CYRBU_CMD_DUMP_MESSAGE; - } - break; - case 'j': - if (strcmp(command, "json") == 0) { - if (strcmp(sub, "chunks") == 0) return CYRBU_CMD_JSON_CHUNKS; - else if (strcmp(sub, "mailboxes") == 0) return CYRBU_CMD_JSON_MAILBOXES; - else if (strcmp(sub, "messages") == 0) return CYRBU_CMD_JSON_MESSAGES; - else if (strcmp(sub, "headers") == 0) return CYRBU_CMD_JSON_HEADERS; - } - break; - case 'l': - if (strcmp(command, "list") == 0) { - if (strcmp(sub, "all") == 0) return CYRBU_CMD_LIST_ALL; - else if (strcmp(sub, "chunks") == 0) return CYRBU_CMD_LIST_CHUNKS; - else if (strcmp(sub, "mailboxes") == 0) return CYRBU_CMD_LIST_MAILBOXES; - else if (strcmp(sub, "messages") == 0) return CYRBU_CMD_LIST_MESSAGES; - } - break; - case 's': - if (strcmp(command, "show") == 0) { - if (strcmp(sub, "chunks") == 0) return CYRBU_CMD_SHOW_CHUNKS; - else if (strcmp(sub, "mailboxes") == 0) return CYRBU_CMD_SHOW_MAILBOXES; - else if (strcmp(sub, "messages") == 0) return CYRBU_CMD_SHOW_MESSAGES; - } - break; - default: - break; - } - - return CYRBU_CMD_UNSPECIFIED; -} - -int main(int argc, char **argv) -{ - save_argv0(argv[0]); - - struct cyrbu_cmd_options options = {0}; - enum cyrbu_mode mode = CYRBU_MODE_UNSPECIFIED; - enum cyrbu_cmd cmd = CYRBU_CMD_UNSPECIFIED; - const char *alt_config = NULL; - const char *backup_name = NULL; - const char *command = NULL; - const char *subcommand = NULL; - struct backup *backup = NULL; - mbname_t *mbname = NULL; - int i, opt, r = 0; - - /* keep this in alphabetical order */ - static const char short_options[] = "C:fmuv"; - - static const struct option long_options[] = { - /* n.b. no long option for -C */ - { "filename", no_argument, NULL, 'f' }, - { "mailbox", no_argument, NULL, 'm' }, - { "userid", no_argument, NULL, 'u' }, - { "verbose", no_argument, NULL, 'v' }, - - { 0, 0, 0, 0 }, - }; - - while (-1 != (opt = getopt_long(argc, argv, - short_options, long_options, NULL))) - { - switch (opt) { - case 'C': - alt_config = optarg; - break; - case 'f': - if (mode != CYRBU_MODE_UNSPECIFIED) usage(); - mode = CYRBU_MODE_FILENAME; - break; - case 'm': - if (mode != CYRBU_MODE_UNSPECIFIED) usage(); - mode = CYRBU_MODE_MBOXNAME; - break; - case 'u': - if (mode != CYRBU_MODE_UNSPECIFIED) usage(); - mode = CYRBU_MODE_USERNAME; - break; - case 'v': - options.verbose++; - break; - default: - usage(); - break; - } - } - - /* default mode is username */ - if (mode == CYRBU_MODE_UNSPECIFIED) - mode = CYRBU_MODE_USERNAME; - - /* get the backup name */ - if (optind == argc) usage(); - backup_name = argv[optind++]; - - /* get the command */ - if (optind == argc) usage(); - command = argv[optind++]; - - /* get the subcommand */ - if (optind == argc) usage(); - subcommand = argv[optind++]; - - /* parse the command and subcommand */ - cmd = parse_cmd_string(command, subcommand); - - /* check remaining arguments based on command */ - switch (cmd) { - case CYRBU_CMD_LIST_ALL: - case CYRBU_CMD_LIST_CHUNKS: - case CYRBU_CMD_LIST_MAILBOXES: - case CYRBU_CMD_LIST_MESSAGES: - case CYRBU_CMD_JSON_CHUNKS: - case CYRBU_CMD_JSON_MAILBOXES: - case CYRBU_CMD_JSON_MESSAGES: - /* these want no more arguments */ - if (optind != argc) usage(); - break; - case CYRBU_CMD_SHOW_CHUNKS: - case CYRBU_CMD_SHOW_MAILBOXES: - case CYRBU_CMD_SHOW_MESSAGES: - case CYRBU_CMD_JSON_HEADERS: - /* these need at least one more argument */ - if (optind == argc) usage(); - break; - case CYRBU_CMD_DUMP_CHUNK: - case CYRBU_CMD_DUMP_MAILBOX: - case CYRBU_CMD_DUMP_MESSAGE: - /* these need exactly one more argument */ - if (argc - optind != 1) usage(); - break; - default: - usage(); - break; - } - - /* build a nice args list */ - options.argv = strarray_new(); - for (i = optind; i < argc; i++) { - strarray_add(options.argv, argv[i]); - } - - // FIXME finish parsing options - - cyrus_init(alt_config, "cyr_backup", 0, 0); - - if ((r = mboxname_init_namespace(&cyr_backup_namespace, NAMESPACE_OPTION_ADMIN))) { - fatal(error_message(r), EX_CONFIG); - } - mboxevent_setnamespace(&cyr_backup_namespace); - - /* use xmalloc rather than malloc for json internals */ - json_set_alloc_funcs(xmalloc, free); - - /* open backup */ - switch (mode) { - case CYRBU_MODE_FILENAME: - r = backup_open_paths(&backup, backup_name, NULL, - BACKUP_OPEN_BLOCK, BACKUP_OPEN_NOCREATE); - break; - case CYRBU_MODE_MBOXNAME: - mbname = mbname_from_extname(backup_name, &cyr_backup_namespace, NULL); - if (!mbname) usage(); - r = backup_open(&backup, mbname, - BACKUP_OPEN_BLOCK, BACKUP_OPEN_NOCREATE); - break; - case CYRBU_MODE_USERNAME: - mbname = mbname_from_userid(backup_name); - if (!mbname) usage(); - r = backup_open(&backup, mbname, - BACKUP_OPEN_BLOCK, BACKUP_OPEN_NOCREATE); - break; - default: - usage(); - break; - } - - /* verify the backup */ - if (!r) - r = backup_verify(backup, BACKUP_VERIFY_QUICK, options.verbose, stdout); - - /* run command */ - if (!r && cmd_func[cmd]) - r = cmd_func[cmd](backup, &options); - - if (r) { - fprintf(stderr, "%s: %s\n", backup_name, error_message(r)); - syslog(LOG_ERR, "backup error %s: %s", backup_name, error_message(r)); - } - - /* close backup */ - if (backup) - backup_close(&backup); - - /* clean up and exit */ - mbname_free(&mbname); - backup_cleanup_staging_path(); - cyrus_done(); - - strarray_free(options.argv); - exit(r ? EX_TEMPFAIL : EX_OK); -} - -static int cmd_list_all(struct backup *backup, - const struct cyrbu_cmd_options *options) -{ - fprintf(stderr, "listing chunks:\n"); - cmd_list_chunks(backup, options); - - fprintf(stderr, "\nlisting mailboxes:\n"); - cmd_list_mailboxes(backup, options); - - fprintf(stderr, "\nlisting messages:\n"); - cmd_list_messages(backup, options); - - return 0; -} - -static int cmd_list_chunks(struct backup *backup, - const struct cyrbu_cmd_options *options) -{ - struct backup_chunk_list *chunk_list = NULL; - struct backup_chunk *chunk; - struct stat data_stat_buf; - int r; - - (void) options; - - r = backup_stat(backup, &data_stat_buf, NULL); - if (r) return r; - - chunk_list = backup_get_chunks(backup); - if (!chunk_list) return -1; - - fprintf(stdout, " id offset\tlength\tratio%%\tstart time end time\n"); - for (chunk = chunk_list->head; chunk; chunk = chunk->next) { - char ts_start[32] = "[unknown]"; - char ts_end[32] = "[unknown]"; - double ratio; - - strftime(ts_start, sizeof(ts_start), "%F %T", - localtime(&chunk->ts_start)); - strftime(ts_end, sizeof(ts_end), "%F %T", - localtime(&chunk->ts_end)); - - if (chunk->next) { - ratio = 100.0 * (chunk->next->offset - chunk->offset) / chunk->length; - } - else { - ratio = 100.0 * (data_stat_buf.st_size - chunk->offset) / chunk->length; - } - - fprintf(stdout, "%7d " OFF_T_FMT "\t" SIZE_T_FMT "\t%6.1f\t%s %s\n", - chunk->id, - chunk->offset, - chunk->length, - ratio, - ts_start, - ts_end); - } - - backup_chunk_list_free(&chunk_list); - return 0; -} - -static int list_mailbox_cb(const struct backup_mailbox *mailbox, - void *rock) -{ - const struct cyrbu_cmd_options *options = - (const struct cyrbu_cmd_options *) rock; - char ts_last_appenddate[32] = "[unknown]"; - - (void) options; - - strftime(ts_last_appenddate, sizeof(ts_last_appenddate), "%F %T", - localtime(&mailbox->last_appenddate)); - - fprintf(stdout, "%s %s %s\n", - mailbox->uniqueid, - ts_last_appenddate, - mailbox->mboxname); - - return 0; -} - -static int cmd_list_mailboxes(struct backup *backup, - const struct cyrbu_cmd_options *options) -{ - fprintf(stdout, "%-36s %-19s %s\n", - "uniqueid", - "last append date", - "mboxname"); - - return backup_mailbox_foreach(backup, 0, - BACKUP_MAILBOX_NO_RECORDS, - list_mailbox_cb, (void *) options); -} - -static int list_message_cb(const struct backup_message *message, void *rock) -{ - const struct cyrbu_cmd_options *options = - (const struct cyrbu_cmd_options *) rock; - - (void) options; - - fprintf(stdout, "%s\n", message_guid_encode(message->guid)); - - return 0; -} - -static int cmd_list_messages(struct backup *backup, - const struct cyrbu_cmd_options *options) -{ - return backup_message_foreach(backup, 0, NULL, - list_message_cb, (void *) options); -} - -static int cmd_show_chunks(struct backup *backup, - const struct cyrbu_cmd_options *options) -{ - // FIXME - (void) backup; - (void) options; - fprintf(stderr, "%s: unimplemented\n", __func__); - return -1; -} - -static int cmd_show_mailboxes(struct backup *backup, - const struct cyrbu_cmd_options *options) -{ - struct backup_mailbox *mailbox = NULL; - struct backup_mailbox_message *record = NULL; - int i; - - for (i = 0; i < strarray_size(options->argv); i++) { - char ts_deleted[32] = ""; - const char *arg = strarray_nth(options->argv, i); - - /* argument could be a uniqueid */ - mailbox = backup_get_mailbox_by_uniqueid(backup, arg, - BACKUP_MAILBOX_ALL_RECORDS); - - /* or it could be an mboxname */ - if (!mailbox) { - mbname_t *mbname = mbname_from_extname(arg, - &cyr_backup_namespace, - NULL); - if (!mbname) continue; - mailbox = backup_get_mailbox_by_name(backup, mbname, - BACKUP_MAILBOX_ALL_RECORDS); - mbname_free(&mbname); - } - - /* or it could be junk */ - if (!mailbox) continue; - - fprintf(stdout, "mboxname: %s\n", mailbox->mboxname); - fprintf(stdout, "uniqueid: %s\n", mailbox->uniqueid); - - if (mailbox->deleted) { - strftime(ts_deleted, sizeof(ts_deleted), "%F %T", - localtime(&mailbox->deleted)); - - fprintf(stdout, "deleted: %s\n", ts_deleted); - } - - fprintf(stdout, "messages:\n"); - fprintf(stdout, " uid expunged time guid\n"); - - for (record = mailbox->records->head; record; record = record->next) { - char ts_expunged[32] = " "; - - if (record->expunged) - strftime(ts_expunged, sizeof(ts_expunged), "%F %T", - localtime(&record->expunged)); - - fprintf(stdout, "%10d %s %s\n", - record->uid, - ts_expunged, - message_guid_encode(&record->guid)); - } - - fprintf(stdout, "\n"); - - backup_mailbox_free(&mailbox); - } - - return 0; -} - -static int show_message_headers(const struct buf *buf, void *rock) -{ - FILE *out = (FILE *) rock; - - const char *start = buf_cstring(buf); - const char *end = strstr(start, "\r\n\r\n"); - - if (!end) return -1; - - fwrite(start, 1, end - start, out); - fputs("\r\n", out); - - return 0; -} - -static int cmd_show_messages(struct backup *backup, - const struct cyrbu_cmd_options *options) -{ - struct backup_message *message = NULL; - struct backup_mailbox_list *mailboxes = NULL; - struct backup_mailbox *mailbox; - struct message_guid want_guid; - int i; - - for (i = 0; i < strarray_size(options->argv); i++) { - if (!message_guid_decode(&want_guid, strarray_nth(options->argv, i))) - continue; - - message = backup_get_message(backup, &want_guid); - if (!message) - continue; - - fprintf(stdout, "guid:\t%s\n", message_guid_encode(message->guid)); - - mailboxes = backup_get_mailboxes_by_message(backup, message, - BACKUP_MAILBOX_NO_RECORDS); - if (mailboxes) { - fprintf(stdout, "mailboxes:\n"); - for (mailbox = mailboxes->head; mailbox; mailbox = mailbox->next) { - fprintf(stdout, "\t%s\t%s\n", - mailbox->uniqueid, - mailbox->mboxname); - } - - backup_mailbox_list_empty(mailboxes); - free(mailboxes); - } - - fprintf(stdout, "headers:\n"); - backup_read_message_data(backup, message, - show_message_headers, stdout); - - fprintf(stdout, "\n"); - backup_message_free(&message); - } - - return 0; -} - -static int dump_buf(const struct buf *buf, void *rock) -{ - FILE *out = (FILE *) rock; - int r = fputs(buf_cstring(buf), out); - return r < 0 ? r : 0; -} - -static int cmd_dump_chunk(struct backup *backup, - const struct cyrbu_cmd_options *options) -{ - struct backup_chunk *chunk = NULL; - int chunk_id; - int r; - - chunk_id = atoi(strarray_nth(options->argv, 0)); - if (chunk_id <= 0) return -1; - - chunk = backup_get_chunk(backup, chunk_id); - if (!chunk) return IMAP_NOTFOUND; - - r = backup_read_chunk_data(backup, chunk, dump_buf, stdout); - - backup_chunk_free(&chunk); - return r; -} - -static int cmd_dump_mailbox(struct backup *backup, - const struct cyrbu_cmd_options *options) -{ - // FIXME - (void) backup; - (void) options; - fprintf(stderr, "%s: unimplemented\n", __func__); - return -1; -} - -static int cmd_dump_message(struct backup *backup, - const struct cyrbu_cmd_options *options) -{ - struct backup_message *message = NULL; - struct message_guid want_guid; - int r; - - if (!message_guid_decode(&want_guid, strarray_nth(options->argv, 0))) - return IMAP_NOTFOUND; - - message = backup_get_message(backup, &want_guid); - if (!message) - return IMAP_NOTFOUND; - - r = backup_read_message_data(backup, message, dump_buf, stdout); - - backup_message_free(&message); - - return r; -} - -static int cmd_json_chunks(struct backup *backup, - const struct cyrbu_cmd_options *options) -{ - struct backup_chunk_list *chunks = NULL; - struct backup_chunk *chunk = NULL; - json_t *jchunks = NULL; - struct stat data_stat_buf; - int r; - - (void) options; - - r = backup_stat(backup, &data_stat_buf, NULL); - if (r) return r; - - jchunks = json_array(); - chunks = backup_get_chunks(backup); - - for (chunk = chunks->head; chunk; chunk = chunk->next) { - char ts_start[32] = "[unknown]"; - char ts_end[32] = "[unknown]"; - json_t *jchunk = json_object(); - double ratio; - - strftime(ts_start, sizeof(ts_start), "%F %T", - localtime(&chunk->ts_start)); - strftime(ts_end, sizeof(ts_end), "%F %T", - localtime(&chunk->ts_end)); - - if (chunk->next) { - ratio = 100.0 * (chunk->next->offset - chunk->offset) / chunk->length; - } - else { - ratio = 100.0 * (data_stat_buf.st_size - chunk->offset) / chunk->length; - } - - /* XXX which fields do we want? */ - json_object_set_new(jchunk, "id", json_integer(chunk->id)); - json_object_set_new(jchunk, "offset", json_integer(chunk->offset)); - json_object_set_new(jchunk, "length", json_integer(chunk->length)); - json_object_set_new(jchunk, "ratio", json_real(ratio)); - json_object_set_new(jchunk, "start time", json_string(ts_start)); - json_object_set_new(jchunk, "end time", json_string(ts_end)); - - json_array_append_new(jchunks, jchunk); - } - - backup_chunk_list_free(&chunks); - - if (!r) { - const int flags = JSON_PRESERVE_ORDER | JSON_INDENT(2); - char *dump; - - dump = json_dumps(jchunks, flags); - printf("%s\n", dump); - free(dump); - } - - json_decref(jchunks); - return r; -} - -static int json_mailbox_cb(const struct backup_mailbox *mailbox, void *rock) -{ - json_t *jmailboxes = (json_t *) rock; - json_t *jmailbox = json_object(); - char ts_last_appenddate[32] = "[unknown]"; - - strftime(ts_last_appenddate, sizeof(ts_last_appenddate), "%F %T", - localtime(&mailbox->last_appenddate)); - - /* XXX which fields are we interested in? */ - json_object_set_new(jmailbox, "uniqueid", json_string(mailbox->uniqueid)); - json_object_set_new(jmailbox, "mboxname", json_string(mailbox->mboxname)); - json_object_set_new(jmailbox, "last_appenddate", json_string(ts_last_appenddate)); - - if (mailbox->records && mailbox->records->count) { - struct backup_mailbox_message *iter; - json_t *jmessages = json_array(); - - for (iter = mailbox->records->head; iter; iter = iter->next) { - json_t *jrecord = json_object(); - - json_object_set_new(jrecord, "uid", json_integer(iter->uid)); - json_object_set_new(jrecord, "guid", - json_string(message_guid_encode(&iter->guid))); - - if (iter->expunged) { - char ts_expunged[32] = " "; - strftime(ts_expunged, sizeof(ts_expunged), "%F %T", - localtime(&iter->expunged)); - json_object_set_new(jrecord, "expunged", - json_string(ts_expunged)); - } - - json_array_append_new(jmessages, jrecord); - } - - json_object_set_new(jmailbox, "messages", jmessages); - } - - json_array_append_new(jmailboxes, jmailbox); - return 0; -} - -static int cmd_json_mailboxes(struct backup *backup, - const struct cyrbu_cmd_options *options) -{ - json_t *mailboxes = json_array(); - int r; - - (void) options; - - r = backup_mailbox_foreach(backup, 0, BACKUP_MAILBOX_ALL_RECORDS, - json_mailbox_cb, mailboxes); - - if (!r) { - const int flags = JSON_PRESERVE_ORDER | JSON_INDENT(2); - char *dump; - - dump = json_dumps(mailboxes, flags); - printf("%s\n", dump); - free(dump); - } - - json_decref(mailboxes); - return r; -} - -struct json_message_rock { - struct backup *backup; - json_t *jmessages; -}; - -static int json_message_cb(const struct backup_message *message, void *rock) -{ - struct json_message_rock *jmrock = (struct json_message_rock *) rock; - struct backup_mailbox_list *mailboxes; - struct backup_mailbox *mailbox; - json_t *jmessage = json_object(); - - /* XXX what fields do we want? */ - json_object_set_new(jmessage, "guid", - json_string(message_guid_encode(message->guid))); - - mailboxes = backup_get_mailboxes_by_message(jmrock->backup, message, - BACKUP_MAILBOX_NO_RECORDS); - if (mailboxes && mailboxes->count) { - json_t *jmailboxes = json_array(); - - for (mailbox = mailboxes->head; mailbox; mailbox = mailbox->next) { - json_t *jmailbox = json_object(); - - json_object_set_new(jmailbox, "uniqueid", - json_string(mailbox->uniqueid)); - json_object_set_new(jmailbox, "mboxname", - json_string(mailbox->mboxname)); - - json_array_append_new(jmailboxes, jmailbox); - } - - json_object_set_new(jmessage, "mailboxes", jmailboxes); - } - backup_mailbox_list_empty(mailboxes); - free(mailboxes); - - json_array_append_new(jmrock->jmessages, jmessage); - - return 0; -} - -static int cmd_json_messages(struct backup *backup, - const struct cyrbu_cmd_options *options) -{ - json_t *messages = json_array(); - struct json_message_rock rock = { backup, messages }; - int r; - - (void) options; - - r = backup_message_foreach(backup, 0, NULL, json_message_cb, &rock); - - if (!r) { - const int flags = JSON_PRESERVE_ORDER | JSON_INDENT(2); - char *dump; - - dump = json_dumps(messages, flags); - printf("%s\n", dump); - free(dump); - } - - json_decref(messages); - return r; -} - -static int json_headers_cb(const struct buf *buf, void *rock) -{ - const char *header = NULL, *next = buf_cstring(buf); - char *name = NULL; - char *value = NULL; - size_t len; - json_t *jmessage = (json_t *) rock; - json_t *jheader; - - while (next && *next) { - header = next; - - /* advance next pointer before we carry on */ - do { - next = strchr(next + 1, '\n'); - } while (next && (next[1] == ' ' || next[1] == '\t')); - if (next) { - if (next[1] == '\0' /* end of file */ - || next[1] == '\n' /* two line breaks in a row marks end of headers */ - || (next[1] == '\r' && next[2] == '\n')) { - next = NULL; - } - else { - next++; - } - } - - /* now process the current header */ - len = strcspn(header, ":\r\n"); - if (header[len] != ':') - continue; - - name = xstrndup(header, len); - message_parse_string(header + len + 1, &value); - - jheader = json_object_get(jmessage, name); - if (!jheader) { - jheader = json_array(); - json_object_set_new(jmessage, name, jheader); - } - - json_array_append_new(jheader, json_string(value)); - - free(name); - free(value); - name = value = NULL; - } - - return 0; -} - -static int cmd_json_headers(struct backup *backup, - const struct cyrbu_cmd_options *options) -{ - json_t *jmessages = json_object(); - json_t *jheaders = NULL; - struct backup_message *message = NULL; - struct message_guid want_guid; - int i, r = 0; - - for (i = 0; i < strarray_size(options->argv); i++) { - if (!message_guid_decode(&want_guid, strarray_nth(options->argv, i))) - continue; - - message = backup_get_message(backup, &want_guid); - if (!message) - continue; - - jheaders = json_object(); - backup_read_message_data(backup, message, json_headers_cb, jheaders); - - json_object_set_new(jmessages, message_guid_encode(&want_guid), jheaders); - jheaders = NULL; - - backup_message_free(&message); - } - - if (!r) { - const int flags = JSON_PRESERVE_ORDER | JSON_INDENT(2); - char *dump; - - dump = json_dumps(jmessages, flags); - printf("%s\n", dump); - free(dump); - } - - json_decref(jmessages); - return r; -} diff --git a/backup/lcb.c b/backup/lcb.c deleted file mode 100644 index d9eef265cc2..00000000000 --- a/backup/lcb.c +++ /dev/null @@ -1,844 +0,0 @@ -/* lcb.c -- replication-based backup api - * - * Copyright (c) 1994-2015 Carnegie Mellon University. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * 3. The name "Carnegie Mellon University" must not be used to - * endorse or promote products derived from this software without - * prior written permission. For permission or any legal - * details, please contact - * Carnegie Mellon University - * Center for Technology Transfer and Enterprise Creation - * 4615 Forbes Avenue - * Suite 302 - * Pittsburgh, PA 15213 - * (412) 268-7393, fax: (412) 268-7395 - * innovation@andrew.cmu.edu - * - * 4. Redistributions of any form whatsoever must retain the following - * acknowledgment: - * "This product includes software developed by Computing Services - * at Carnegie Mellon University (http://www.cmu.edu/computing/)." - * - * CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO - * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY - * AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE - * FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN - * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING - * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - * - */ - -#include - -#include -#include -#include -#include -#include -#include - -#include "lib/cyrusdb.h" -#include "lib/cyr_lock.h" -#include "lib/gzuncat.h" -#include "lib/map.h" -#include "lib/sqldb.h" -#include "lib/util.h" -#include "lib/xmalloc.h" -#include "lib/xsha1.h" -#include "lib/xstrlcat.h" -#include "lib/xstrlcpy.h" -#include "lib/xunlink.h" - -#include "imap/dlist.h" -#include "imap/global.h" -#include "imap/imap_err.h" - -#include "backup/backup.h" - -#define LIBCYRUS_BACKUP_SOURCE /* this file is part of libcyrus_backup */ -#include "backup/lcb_internal.h" -#include "backup/lcb_sqlconsts.h" - -static const char *NOUSERID = "\%SHARED"; - -/* remove this process's staging directory. - * will warn about and clean up files that are hanging around - these should - * be removed by dlist_unlink_files but may be missed if we're shutdown by a - * signal. - */ -EXPORTED void backup_cleanup_staging_path(void) -{ - char name[MAX_MAILBOX_PATH]; - const char *base = config_backupstagingpath(); - DIR *dirp; - int r; - - r = snprintf(name, MAX_MAILBOX_PATH, "%s/sync./%lu", - base, (unsigned long) getpid()); - if (r >= MAX_MAILBOX_PATH) { - /* path was truncated, don't try to delete it */ - return; - } - - /* make sure it's empty */ - if ((dirp = opendir(name))) { - struct dirent *d; - while ((d = readdir(dirp))) { - if (!strcmp(d->d_name, ".") || !strcmp(d->d_name, "..")) - continue; - - char *tmp = strconcat(name, "/", d->d_name, NULL); - syslog(LOG_INFO, "%s: unlinking leftover stage file: %s", __func__, tmp); - xunlink(tmp); - free(tmp); - } - closedir(dirp); - } - - r = rmdir(name); - if (r && errno != ENOENT) - syslog(LOG_WARNING, "%s rmdir %s: %m", __func__, name); -} - -/* - * use cases: - * - backupd needs to be able to append to data stream and update index (exclusive) - * - backupd maybe needs to create a new backup from scratch (exclusive) - * - reindex needs to gzuc data stream and rewrite index (exclusive) - * - compact needs to rewrite data stream and index (exclusive) - * - restore needs to read data stream and index (shared) - * - * with only one shared case, might as well always lock exclusively... - */ -HIDDEN int backup_real_open(struct backup **backupp, - const char *data_fname, const char *index_fname, - enum backup_open_reindex reindex, - enum backup_open_nonblock nonblock, - enum backup_open_create create) -{ - struct backup *backup = xzmalloc(sizeof *backup); - int r; - - backup->fd = -1; - - backup->data_fname = xstrdup(data_fname); - backup->index_fname = xstrdup(index_fname); - - int open_flags = O_RDWR | O_APPEND; - - switch (create) { - case BACKUP_OPEN_CREATE_EXCL: open_flags |= O_EXCL; /* fall thru */ - case BACKUP_OPEN_CREATE: open_flags |= O_CREAT; /* */ - case BACKUP_OPEN_NOCREATE: break; - } - - while (backup->fd == -1) { - struct stat sbuf1, sbuf2; - - int fd = open(backup->data_fname, - open_flags, - S_IRUSR | S_IWUSR); - if (fd < 0) { - switch (errno) { - case EEXIST: - r = IMAP_MAILBOX_EXISTS; - break; - case ENOENT: - r = IMAP_MAILBOX_NONEXISTENT; - break; - default: - xsyslog(LOG_ERR, "IOERROR: open failed", - "filename=<%s>", backup->data_fname); - r = IMAP_IOERROR; - break; - } - - goto error; - } - - r = lock_setlock(fd, /*excl*/ 1, nonblock, backup->data_fname); - if (r) { - if (errno == EWOULDBLOCK || errno == EAGAIN) { - r = IMAP_MAILBOX_LOCKED; - } - else { - xsyslog(LOG_ERR, "IOERROR: lock_setlock failed", - "filename=<%s>", backup->data_fname); - r = IMAP_IOERROR; - } - goto error; - } - - r = fstat(fd, &sbuf1); - if (!r) r = stat(backup->data_fname, &sbuf2); - if (r) { - xsyslog(LOG_ERR, "IOERROR: stat failed", - "filename=<%s>", backup->data_fname); - r = IMAP_IOERROR; - close(fd); - goto error; - } - - if (sbuf1.st_ino == sbuf2.st_ino) { - backup->fd = fd; - break; - } - - close(fd); - } - - if (reindex) { - // when reindexing, we want to move the old index out of the way - // and create a new, empty one -- while holding the lock - char oldindex_fname[PATH_MAX]; - snprintf(oldindex_fname, sizeof(oldindex_fname), "%s.%" PRId64, - backup->index_fname, (int64_t) time(NULL)); - - r = rename(backup->index_fname, oldindex_fname); - if (r && errno != ENOENT) { - xsyslog(LOG_ERR, "IOERROR: rename failed", - "source=<%s> dest=<%s>", - backup->index_fname, oldindex_fname); - r = IMAP_IOERROR; - goto error; - } - - backup->oldindex_fname = xstrdup(oldindex_fname); - } - else { - // if there's data in the data file but the index file is empty - // or doesn't exist, insist on a reindex before opening - struct stat data_statbuf; - r = fstat(backup->fd, &data_statbuf); - if (r) { - xsyslog(LOG_ERR, "IOERROR: fstat failed", - "filename=<%s>", backup->data_fname); - r = IMAP_IOERROR; - goto error; - } - if (data_statbuf.st_size > 0) { - struct stat index_statbuf; - r = stat(backup->index_fname, &index_statbuf); - if (r && errno != ENOENT) { - xsyslog(LOG_ERR, "IOERROR: stat failed", - "filename=<%s>", backup->index_fname); - r = IMAP_IOERROR; - goto error; - } - - if ((r && errno == ENOENT) || index_statbuf.st_size == 0) { - xsyslog(LOG_ERR, "IOERROR: reindex needed", - "filename=<%s>", backup->index_fname); - r = IMAP_MAILBOX_BADFORMAT; - goto error; - } - } - } - - backup->db = sqldb_open(backup->index_fname, backup_index_initsql, - backup_index_version, backup_index_upgrade, - SQLDB_DEFAULT_TIMEOUT); - if (!backup->db) { - r = IMAP_INTERNAL; // FIXME what does it mean to error here? - goto error; - } - - *backupp = backup; - return 0; - -error: - backup_close(&backup); - if (!r) r = IMAP_INTERNAL; - return r; -} - -EXPORTED int backup_open(struct backup **backupp, - const mbname_t *mbname, - enum backup_open_nonblock nonblock, - enum backup_open_create create) -{ - struct buf data_fname = BUF_INITIALIZER; - struct buf index_fname = BUF_INITIALIZER; - - int r = backup_get_paths(mbname, &data_fname, &index_fname, create); - /* XXX convert CYRUSDB return code to IMAP */ - if (r) goto done; - - r = backup_real_open(backupp, - buf_cstring(&data_fname), buf_cstring(&index_fname), - BACKUP_OPEN_NOREINDEX, nonblock, create); - if (r) goto done; - -done: - buf_free(&data_fname); - buf_free(&index_fname); - - return r; -} - -/* Uses mkstemp() to create a new, unique, backup path for the given user. - * - * On success, the file is not unlinked, presuming that it will shortly be - * used for storing backup data. This also ensures its uniqueness remains: - * this function won't generate the same value again as long as the previous - * file is intact, so there's no user-rename race. - * - * If out_fd is non-NULL, on successful return it will contain an open, locked - * file descriptor for the new file. In this case the caller must unlock - * and close the fd. - * - * On error, returns NULL and logs to syslog, without touching out_fd. - */ -static const char *_make_path(const mbname_t *mbname, int *out_fd) -{ - static char pathresult[PATH_MAX]; - - const char *userid = mbname_userid(mbname); - const char *partition = partlist_backup_select(); - const char *ret = NULL; - - if (!userid) userid = NOUSERID; - - if (!partition) { - syslog(LOG_ERR, - "unable to make backup path for %s: " - "couldn't select partition", - userid); - return NULL; - } - - char hash_buf[2]; - char *template = strconcat(partition, - "/", dir_hash_b(userid, config_fulldirhash, hash_buf), - "/", userid, "_XXXXXX", - NULL); - - /* make sure the destination directory exists */ - cyrus_mkdir(template, 0755); - - int fd = mkstemp(template); - if (fd < 0) { - syslog(LOG_ERR, "unable to make backup path for %s: %m", userid); - goto error; - } - - /* lock it -- even if we're just going to immediately unlock it */ - int r = lock_setlock(fd, /*excl*/ 1, /*nb*/ 0, template); - if (r) { - syslog(LOG_ERR, - "unable to obtain exclusive lock on just-created file %s: %m", - template); - /* don't unlink it, we don't know what's in it */ - goto error; - } - - /* save the path */ - if (strlcpy(pathresult, template, sizeof(pathresult)) >= sizeof(pathresult)) { - syslog(LOG_ERR, - "unable to make backup path for %s: path too long", - userid); - xunlink(template); - goto error; - } - ret = pathresult; - - /* save or close the fd */ - if (out_fd) - *out_fd = fd; - else - close(fd); - - free(template); - return ret; - -error: - if (fd >= 0) close(fd); - free(template); - return NULL; -} - -EXPORTED int backup_get_paths(const mbname_t *mbname, - struct buf *data_fname, struct buf *index_fname, - enum backup_open_create create) -{ - struct db *backups_db = NULL; - struct txn *tid = NULL; - - int r = backupdb_open(&backups_db, &tid); - if (r) return r; - - const char *userid = mbname_userid(mbname); - const char *backup_path = NULL; - size_t path_len = 0; - - if (!userid) userid = NOUSERID; - - r = cyrusdb_fetch(backups_db, - userid, strlen(userid), - &backup_path, &path_len, - &tid); - - if (r == CYRUSDB_NOTFOUND && create) { - syslog(LOG_DEBUG, "%s not found in backups.db, creating new record", userid); - backup_path = _make_path(mbname, NULL); - if (!backup_path) { - r = CYRUSDB_INTERNAL; - goto done; - } - path_len = strlen(backup_path); - - r = cyrusdb_create(backups_db, - userid, strlen(userid), - backup_path, path_len, - &tid); - if (r) cyrusdb_abort(backups_db, tid); - else r = cyrusdb_commit(backups_db, tid); - - tid = NULL; - - /* if we didn't store it in the database successfully, trash the file, - * it won't be used */ - if (r) xunlink(backup_path); - } - - if (r) goto done; - - if (path_len == 0) { - syslog(LOG_DEBUG, - "unexpectedly got zero length backup path for user %s", - userid); - r = CYRUSDB_INTERNAL; - goto done; - } - - if (data_fname) - buf_setmap(data_fname, backup_path, path_len); - - if (index_fname) { - buf_setmap(index_fname, backup_path, path_len); - buf_appendcstr(index_fname, ".index"); - } - -done: - if (backups_db) { - if (tid) cyrusdb_abort(backups_db, tid); - cyrusdb_close(backups_db); - } - return r; -} - -/* - * If index_fname is NULL, it will be automatically derived from data_fname - */ -EXPORTED int backup_open_paths(struct backup **backupp, - const char *data_fname, - const char *index_fname, - enum backup_open_nonblock nonblock, - enum backup_open_create create) -{ - if (index_fname) - return backup_real_open(backupp, data_fname, index_fname, - BACKUP_OPEN_NOREINDEX, nonblock, create); - - char *tmp = strconcat(data_fname, ".index", NULL); - int r = backup_real_open(backupp, data_fname, tmp, - BACKUP_OPEN_NOREINDEX, nonblock, create); - free(tmp); - - return r; -} - -EXPORTED int backup_close(struct backup **backupp) -{ - struct backup *backup = *backupp; - *backupp = NULL; - - gzFile gzfile = NULL; - int r1 = 0, r2 = 0; - - if (!backup) return 0; - - if (backup->append_state) { - if (backup->append_state->mode != BACKUP_APPEND_INACTIVE) - r1 = backup_append_end(backup, NULL); - - gzfile = backup->append_state->gzfile; - - free(backup->append_state); - backup->append_state = NULL; - } - - if (backup->db) r2 = sqldb_close(&backup->db); - - if (backup->oldindex_fname) { - if (r2) { - /* something went wrong closing the new index, put the old one back */ - rename(backup->oldindex_fname, backup->index_fname); - } - else { - if (!config_getswitch(IMAPOPT_BACKUP_KEEP_PREVIOUS)) { - xunlink(backup->oldindex_fname); - } - } - } - - if (backup->fd >= 0) { - /* closing the file will also release the lock on the fd */ - if (gzfile) - gzclose_w(gzfile); - else - close(backup->fd); - } - - if (backup->index_fname) free(backup->index_fname); - if (backup->data_fname) free(backup->data_fname); - if (backup->oldindex_fname) free(backup->oldindex_fname); - - free(backup); - return r1 ? r1 : r2; -} - -EXPORTED int backup_unlink(struct backup **backupp) -{ - struct backup *backup = *backupp; - - xunlink(backup->index_fname); - xunlink(backup->data_fname); - - return backup_close(backupp); -} - -EXPORTED const char *backup_get_data_fname(const struct backup *backup) -{ - return backup->data_fname; -} - -EXPORTED const char *backup_get_index_fname(const struct backup *backup) -{ - return backup->index_fname; -} - -EXPORTED int backup_stat(const struct backup *backup, - struct stat *data_statp, - struct stat *index_statp) -{ - struct stat data_statbuf, index_statbuf; - int r; - - r = fstat(backup->fd, &data_statbuf); - if (r) { - xsyslog(LOG_ERR, "IOERROR: fstat failed", - "filename=<%s>", backup->data_fname); - return IMAP_IOERROR; - } - - r = stat(backup->index_fname, &index_statbuf); - if (r) { - xsyslog(LOG_ERR, "IOERROR: stat failed", - "filename=<%s>", backup->index_fname); - return IMAP_IOERROR; - } - - if (data_statp) - memcpy(data_statp, &data_statbuf, sizeof data_statbuf); - if (index_statp) - memcpy(index_statp, &index_statbuf, sizeof index_statbuf); - - return 0; -} - -static ssize_t _prot_fill_cb(unsigned char *buf, size_t len, void *rock) -{ - struct gzuncat *gzuc = (struct gzuncat *) rock; - int r = gzuc_read(gzuc, buf, len); - - if (r < 0) - xsyslog(LOG_ERR, "IOERROR: gzuc_read failed", - "return=<%d>", r); - if (r < -1) - errno = EIO; - - return r; -} - -EXPORTED int backup_reindex(const char *name, - enum backup_open_nonblock nonblock, - int verbose, FILE *out) -{ - struct buf data_fname = BUF_INITIALIZER; - struct buf index_fname = BUF_INITIALIZER; - struct backup *backup = NULL; - int r; - - buf_printf(&data_fname, "%s", name); - buf_printf(&index_fname, "%s.index", name); - - r = backup_real_open(&backup, - buf_cstring(&data_fname), buf_cstring(&index_fname), - BACKUP_OPEN_REINDEX, nonblock, - BACKUP_OPEN_NOCREATE); - buf_free(&index_fname); - buf_free(&data_fname); - if (r) return r; - - struct gzuncat *gzuc = gzuc_new(backup->fd); - - time_t prev_member_ts = -1; - - struct buf cmd = BUF_INITIALIZER; - while (gzuc && !gzuc_eof(gzuc)) { - gzuc_member_start(gzuc); - off_t member_offset = gzuc_member_offset(gzuc); - - if (verbose) - fprintf(out, "\nfound chunk at offset " OFF_T_FMT "\n\n", member_offset); - - struct protstream *member = prot_readcb(_prot_fill_cb, gzuc); - prot_setisclient(member, 1); /* don't sync literals */ - - // FIXME stricter timestamp sequence checks - time_t member_start_ts = -1; - time_t member_end_ts = -1; - time_t ts = -1; - - while (1) { - struct dlist *dl = NULL; - - int c = parse_backup_line(member, &ts, &cmd, &dl); - if (c == EOF) { - const char *error = prot_error(member); - if (error && 0 != strcmp(error, PROT_EOF_STRING)) { - syslog(LOG_ERR, - "IOERROR: %s: error reading chunk at offset " OFF_T_FMT ", byte %i: %s", - name, member_offset, prot_bytes_in(member), error); - - if (out) - fprintf(out, "error reading chunk at offset " OFF_T_FMT ", byte %i: %s\n", - member_offset, prot_bytes_in(member), error); - - r = IMAP_IOERROR; - } - member_end_ts = ts; - break; - } - - if (member_start_ts == -1) { - if (prev_member_ts != -1 && prev_member_ts > ts) { - fatal("member timestamp older than previous", EX_DATAERR); - } - member_start_ts = ts; - char file_sha1[2 * SHA1_DIGEST_LENGTH + 1]; - sha1_file(backup->fd, backup->data_fname, member_offset, file_sha1); - backup_real_append_start(backup, member_start_ts, - member_offset, file_sha1, 1, 0); - } - else if (member_start_ts > ts) - fatal("line timestamp older than previous", EX_DATAERR); - - if (strcmp(buf_cstring(&cmd), "APPLY") != 0) { - dlist_unlink_files(dl); - dlist_free(&dl); - continue; - } - - ucase(dl->name); - - r = backup_append(backup, dl, &ts, BACKUP_APPEND_NOFLUSH); - if (r) { - // FIXME do something - syslog(LOG_ERR, "backup_append returned %d", r); - fprintf(out, "backup_append returned %d\n", r); - } - - dlist_unlink_files(dl); - dlist_free(&dl); - } - - if (backup->append_state && backup->append_state->mode) - backup_real_append_end(backup, member_end_ts); - prot_free(member); - gzuc_member_end(gzuc, NULL); - - prev_member_ts = member_start_ts; - } - buf_free(&cmd); - - if (verbose) - fprintf(out, "reached end of file\n"); - - gzuc_free(&gzuc); - backup_close(&backup); - - return r; -} - -struct _rename_meta { - const char *userid; - char *fname; - char *ext_ptr; - int fd; -}; -#define RENAME_META_INITIALIZER { NULL, NULL, NULL, -1 } - -static void _rename_meta_set_fname(struct _rename_meta *meta, const char *data_fname) -{ - size_t len = strlen(data_fname) + strlen(".index") + 1; - meta->fname = xmalloc(len); - snprintf(meta->fname, len, "%s.index", data_fname); - meta->ext_ptr = strrchr(meta->fname, '.'); - *meta->ext_ptr = '\0'; -} - -static void _rename_meta_fini(struct _rename_meta *meta) -{ - if (meta->fname) free(meta->fname); - memset(meta, 0, sizeof(*meta)); - meta->fd = -1; -} - -EXPORTED int backup_rename(const mbname_t *old_mbname, const mbname_t *new_mbname) -{ - struct db *backups_db = NULL; - struct txn *tid = NULL; - struct _rename_meta old = RENAME_META_INITIALIZER; - struct _rename_meta new = RENAME_META_INITIALIZER; - old.userid = mbname_userid(old_mbname); - new.userid = mbname_userid(new_mbname); - const char *path; - size_t path_len; - int r; - - if (!old.userid) old.userid = NOUSERID; - if (!new.userid) new.userid = NOUSERID; - - /* bail out if the names are the same */ - if (strcmp(old.userid, new.userid) == 0) - return 0; - - /* exclusively open backups database */ - r = backupdb_open(&backups_db, &tid); - if (r) goto error; // FIXME log - - /* make sure new_mbname isn't already in use */ - r = cyrusdb_fetch(backups_db, - new.userid, strlen(new.userid), - &path, &path_len, - &tid); - if (!r) r = CYRUSDB_EXISTS; - if (r) goto error; // FIXME log - - /* locate (but not create) backup for old_mbname, open and lock it */ - r = cyrusdb_fetch(backups_db, - old.userid, strlen(old.userid), - &path, &path_len, - &tid); - if (r) goto error; // FIXME log - - _rename_meta_set_fname(&old, path); - - old.fd = open(old.fname, - O_RDWR | O_APPEND, /* no O_CREAT */ - S_IRUSR | S_IWUSR); - if (old.fd < 0) { - xsyslog(LOG_ERR, "IOERROR: open failed", - "filename=<%s>", old.fname); - r = -1; - goto error; - } - - /* non-blocking, to avoid deadlock */ - r = lock_setlock(old.fd, /*excl*/ 1, /*nb*/ 1, old.fname); - if (r) { - xsyslog(LOG_ERR, "IOERROR: lock_setlock failed", - "filename=<%s>", old.fname); - goto error; - } - - /* make a path for new_mbname, open and lock it */ - path = _make_path(new_mbname, &new.fd); - if (!path) goto error; // FIXME log - _rename_meta_set_fname(&new, path); - - /* copy old data and index files to new paths */ - r = cyrus_copyfile(old.fname, new.fname, 0); - if (r) goto error; // FIXME log - *old.ext_ptr = *new.ext_ptr = '.'; - r = cyrus_copyfile(old.fname, new.fname, 0); - *old.ext_ptr = *new.ext_ptr = '\0'; - if (r) goto error; // FIXME log - - /* files exist under both names now. try to update the database */ - r = cyrusdb_create(backups_db, - new.userid, strlen(new.userid), - new.fname, strlen(new.fname), - &tid); - if (r) goto error; // FIXME log - - r = cyrusdb_delete(backups_db, - old.userid, strlen(old.userid), - &tid, 0); - if (r) goto error; // FIXME log - - r = cyrusdb_commit(backups_db, tid); - tid = NULL; - if (r) goto error; // FIXME log - - /* database update succeeded. unlink old names */ - xunlink(old.fname); - *old.ext_ptr = '.'; - xunlink(old.fname); - *old.ext_ptr = '\0'; - - /* unlock and close backup files */ - lock_unlock(new.fd, new.fname); - close(new.fd); - lock_unlock(old.fd, old.fname); - close(old.fd); - - /* close backups database */ - cyrusdb_close(backups_db); - - /* clean up and exit */ - _rename_meta_fini(&old); - _rename_meta_fini(&new); - return 0; - -error: - /* we didn't finish, so unlink the new filenames if we got that far */ - if (new.fname) { - xunlink(new.fname); - *new.ext_ptr = '.'; - xunlink(new.fname); - *new.ext_ptr = '\0'; - } - - /* close the files if we got that far (also unlocks) */ - if (new.fd != -1) - close(new.fd); - if (old.fd != -1) - close(old.fd); - - /* abort any transaction and close the database */ - if (backups_db) { - if (tid) cyrusdb_abort(backups_db, tid); - cyrusdb_close(backups_db); - } - - /* clean up and exit */ - _rename_meta_fini(&old); - _rename_meta_fini(&new); - return r; -} diff --git a/backup/lcb_append.c b/backup/lcb_append.c deleted file mode 100644 index 7df99eb33c7..00000000000 --- a/backup/lcb_append.c +++ /dev/null @@ -1,326 +0,0 @@ -/* lcb_append.c -- replication-based backup api - append functions - * - * Copyright (c) 1994-2016 Carnegie Mellon University. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * 3. The name "Carnegie Mellon University" must not be used to - * endorse or promote products derived from this software without - * prior written permission. For permission or any legal - * details, please contact - * Carnegie Mellon University - * Center for Technology Transfer and Enterprise Creation - * 4615 Forbes Avenue - * Suite 302 - * Pittsburgh, PA 15213 - * (412) 268-7393, fax: (412) 268-7395 - * innovation@andrew.cmu.edu - * - * 4. Redistributions of any form whatsoever must retain the following - * acknowledgment: - * "This product includes software developed by Computing Services - * at Carnegie Mellon University (http://www.cmu.edu/computing/)." - * - * CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO - * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY - * AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE - * FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN - * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING - * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - * - */ -#include -#include -#include -#include - -#include "lib/sqldb.h" -#include "lib/xmalloc.h" -#include "lib/xsha1.h" - -#include "imap/imap_err.h" - -#include "backup/backup.h" - -#define LIBCYRUS_BACKUP_SOURCE /* this file is part of libcyrus_backup */ -#include "backup/lcb_internal.h" -#include "backup/lcb_sqlconsts.h" - -static int retry_gzwrite(gzFile gzfile, const char *str, size_t len, const char *fname) -{ - /* gzprintf's internal buffer is limited to about 8K, which a dlist will - * exceed if there's a message in it, so use gzwrite rather than gzprintf - * for writing dlist contents. - */ - const char *p = str; - size_t left = len; - - while (left) { - int n = MIN(left, INT32_MAX); - int wrote = gzwrite(gzfile, p, n); - if (wrote > 0) { - left -= wrote; - p += wrote; - } - else { - int r; - const char *err = gzerror(gzfile, &r); - xsyslog(LOG_ERR, "IOERROR: gzwrite failed", - "filename=<%s> error=<%s>", - fname, err); - - if (r == Z_STREAM_ERROR) - fatal("gzwrite: invalid stream", EX_IOERR); - else if (r == Z_MEM_ERROR) - fatal("gzwrite: out of memory", EX_TEMPFAIL); - - return r; - } - } - - return 0; -} - -HIDDEN int backup_real_append_start(struct backup *backup, - time_t ts, off_t offset, - const char *file_sha1, - int index_only, - enum backup_append_flush flush) -{ - int r; - - if (backup->append_state != NULL - && backup->append_state->mode != BACKUP_APPEND_INACTIVE) { - fatal("backup append already started", EX_SOFTWARE); - } - - if (!backup->append_state) - backup->append_state = xzmalloc(sizeof(*backup->append_state)); - - if (index_only) backup->append_state->mode |= BACKUP_APPEND_INDEXONLY; - - backup->append_state->wrote = 0; - SHA1Init(&backup->append_state->sha_ctx); - - char header[80]; - snprintf(header, sizeof(header), "# cyrus backup: chunk start\r\n"); - - if (!index_only) { - if (!backup->append_state->gzfile) { - backup->append_state->gzfile = gzdopen(backup->fd, "ab"); - if (!backup->append_state->gzfile) { - fprintf(stderr, "%s: gzdopen fd %i failed: %s\n", - __func__, backup->fd, strerror(errno)); - goto error; - } - } - - r = retry_gzwrite(backup->append_state->gzfile, - header, strlen(header), backup->data_fname); - if (!r && flush) - r = gzflush(backup->append_state->gzfile, Z_FULL_FLUSH); - - if (r) goto error; - } - - SHA1Update(&backup->append_state->sha_ctx, header, strlen(header)); - backup->append_state->wrote += strlen(header); - - struct sqldb_bindval bval[] = { - { ":ts_start", SQLITE_INTEGER, { .i = ts } }, - { ":offset", SQLITE_INTEGER, { .i = offset } }, - { ":file_sha1", SQLITE_TEXT, { .s = file_sha1 } }, - { NULL, SQLITE_NULL, { .s = NULL } }, - }; - - r = sqldb_begin(backup->db, "backup_append"); - if (r) goto error; - - r = sqldb_exec(backup->db, backup_index_start_sql, bval, NULL, NULL); - if (r) { - syslog(LOG_ERR, "%s: something went wrong: %i", __func__, r); - sqldb_rollback(backup->db, "backup_append"); - goto error; - } - - backup->append_state->chunk_id = sqldb_lastid(backup->db); - - backup->append_state->mode |= BACKUP_APPEND_ACTIVE; - return 0; - -error: - backup->append_state->mode = BACKUP_APPEND_INACTIVE; - return -1; -} - -EXPORTED int backup_append_start(struct backup *backup, - const time_t *tsp, - enum backup_append_flush flush) -{ - char file_sha1[2 * SHA1_DIGEST_LENGTH + 1]; - off_t offset = lseek(backup->fd, 0, SEEK_END); - time_t ts = tsp ? *tsp : time(NULL); - - sha1_file(backup->fd, backup->data_fname, SHA1_LIMIT_WHOLE_FILE, file_sha1); - - return backup_real_append_start(backup, ts, offset, file_sha1, 0, flush); -} - -EXPORTED int backup_append(struct backup *backup, - struct dlist *dlist, - const time_t *tsp, - enum backup_append_flush flush) -{ - if (!backup->append_state || backup->append_state->mode == BACKUP_APPEND_INACTIVE) - fatal("backup append not started", EX_SOFTWARE); - - off_t start = backup->append_state->wrote; - size_t len = 0; - time_t ts = tsp ? *tsp : time(NULL); - struct buf buf = BUF_INITIALIZER; - struct dlist_print_iter *iter = NULL; - const int index_only = backup->append_state->mode & BACKUP_APPEND_INDEXONLY; - int r; - - /* preload buffer with timestamp preamble */ - buf_printf(&buf, "%" PRId64 " APPLY ", (int64_t) ts); - - /* iterate over the dlist */ - iter = dlist_print_iter_new(dlist, 1); - do { - /* track the sha1sum */ - SHA1Update(&backup->append_state->sha_ctx, buf_cstring(&buf), buf_len(&buf)); - - /* if we're not in index-only mode, write the data out */ - if (!index_only) { - r = retry_gzwrite(backup->append_state->gzfile, - buf_cstring(&buf), buf_len(&buf), - backup->data_fname); - if (r) goto error; - } - - /* count the written bytes */ - len += buf_len(&buf); - backup->append_state->wrote += buf_len(&buf); - } while (dlist_print_iter_step(iter, &buf)); - dlist_print_iter_free(&iter); - - /* finally, end with "\r\n" */ - buf_setcstr(&buf, "\r\n"); - SHA1Update(&backup->append_state->sha_ctx, buf_cstring(&buf), buf_len(&buf)); - if (!index_only) { - r = retry_gzwrite(backup->append_state->gzfile, - buf_cstring(&buf), buf_len(&buf), - backup->data_fname); - if (r) goto error; - } - len += buf_len(&buf); - backup->append_state->wrote += buf_len(&buf); - - /* flush if necessary */ - if (flush && !index_only) { - r = gzflush(backup->append_state->gzfile, Z_FULL_FLUSH); - if (r != Z_OK) { - syslog(LOG_ERR, "IOERROR: %s gzflush %s: %i %i", __func__, backup->data_fname, r, errno); - goto error; - } - } - - buf_free(&buf); - - /* update the index */ - return backup_index(backup, dlist, ts, start, len); - -error: - buf_free(&buf); - if (iter) dlist_print_iter_free(&iter); - return IMAP_INTERNAL; -} - -HIDDEN int backup_real_append_end(struct backup *backup, time_t ts) -{ - int r; - - if (!backup->append_state) - fatal("backup append not started", EX_SOFTWARE); - if (backup->append_state->mode == BACKUP_APPEND_INACTIVE) - fatal("backup append not started", EX_SOFTWARE); - - if (!(backup->append_state->mode & BACKUP_APPEND_INDEXONLY)) { - r = gzflush(backup->append_state->gzfile, Z_FINISH); - if (r != Z_OK) { - syslog(LOG_ERR, "IOERROR: gzflush %s failed: %i", - backup->data_fname, r); - sqldb_rollback(backup->db, "backup_append"); - goto done; - } - } - - unsigned char sha1_raw[SHA1_DIGEST_LENGTH]; - char data_sha1[2 * SHA1_DIGEST_LENGTH + 1]; - SHA1Final(sha1_raw, &backup->append_state->sha_ctx); - r = bin_to_hex(sha1_raw, SHA1_DIGEST_LENGTH, data_sha1, BH_LOWER); - assert(r == 2 * SHA1_DIGEST_LENGTH); - - struct sqldb_bindval bval[] = { - { ":id", SQLITE_INTEGER, { .i = backup->append_state->chunk_id } }, - { ":ts_end", SQLITE_INTEGER, { .i = ts } }, - { ":length", SQLITE_INTEGER, { .i = backup->append_state->wrote } }, - { ":data_sha1", SQLITE_TEXT, { .s = data_sha1 } }, - { NULL, SQLITE_NULL, { .s = NULL } }, - }; - - r = sqldb_exec(backup->db, backup_index_end_sql, bval, NULL, NULL); - if (r) { - syslog(LOG_ERR, "%s: something went wrong: %i", __func__, r); - sqldb_rollback(backup->db, "backup_append"); - } - else { - sqldb_commit(backup->db, "backup_append"); - } - -done: - backup->append_state->mode = BACKUP_APPEND_INACTIVE; - backup->append_state->wrote = 0; - - return r; -} - -EXPORTED int backup_append_end(struct backup *backup, const time_t *tsp) -{ - time_t ts = tsp ? *tsp : time(NULL); - return backup_real_append_end(backup, ts); -} - -EXPORTED int backup_append_abort(struct backup *backup) -{ - if (!backup->append_state) - fatal("backup append not started", EX_SOFTWARE); - if (backup->append_state == BACKUP_APPEND_INACTIVE) - fatal("backup append not started", EX_SOFTWARE); - - sqldb_rollback(backup->db, "backup_append"); - - // FIXME - // can we truncate back to the length we started this append at? - // ftruncate(2) says nothing about behaviour on descriptors - // opened with O_APPEND... - // seems like it might work, but test it first. - - // FIXME at least z_finish the damn file... - - backup->append_state->mode = BACKUP_APPEND_INACTIVE; - return 0; -} - diff --git a/backup/lcb_backupdb.c b/backup/lcb_backupdb.c deleted file mode 100644 index 2cc18464983..00000000000 --- a/backup/lcb_backupdb.c +++ /dev/null @@ -1,67 +0,0 @@ -/* lcb_backupdb.c -- replication-based backup api database functions - * - * Copyright (c) 1994-2016 Carnegie Mellon University. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * 3. The name "Carnegie Mellon University" must not be used to - * endorse or promote products derived from this software without - * prior written permission. For permission or any legal - * details, please contact - * Carnegie Mellon University - * Center for Technology Transfer and Enterprise Creation - * 4615 Forbes Avenue - * Suite 302 - * Pittsburgh, PA 15213 - * (412) 268-7393, fax: (412) 268-7395 - * innovation@andrew.cmu.edu - * - * 4. Redistributions of any form whatsoever must retain the following - * acknowledgment: - * "This product includes software developed by Computing Services - * at Carnegie Mellon University (http://www.cmu.edu/computing/)." - * - * CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO - * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY - * AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE - * FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN - * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING - * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - * - */ - -#include - -#include "lib/cyrusdb.h" -#include "lib/libconfig.h" - -#include "imap/global.h" - -#include "backup/backup.h" - -#define LIBCYRUS_BACKUP_SOURCE /* this file is part of libcyrus_backup */ - -EXPORTED int backupdb_open(struct db **backup_dbp, struct txn **tidp) -{ - char *fname = xstrdupnull(config_getstring(IMAPOPT_BACKUP_DB_PATH)); - int flags = CYRUSDB_CREATE; - - if (!fname) - fname = strconcat(config_dir, FNAME_BACKUPDB, NULL); - - int r = cyrusdb_lockopen(config_backup_db, fname, flags, backup_dbp, tidp); - - free(fname); - return r; -} diff --git a/backup/lcb_compact.c b/backup/lcb_compact.c deleted file mode 100644 index e3020df7a3e..00000000000 --- a/backup/lcb_compact.c +++ /dev/null @@ -1,622 +0,0 @@ -/* lcb_compact.c -- replication-based backup api - backup compaction - * - * Copyright (c) 1994-2016 Carnegie Mellon University. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * 3. The name "Carnegie Mellon University" must not be used to - * endorse or promote products derived from this software without - * prior written permission. For permission or any legal - * details, please contact - * Carnegie Mellon University - * Center for Technology Transfer and Enterprise Creation - * 4615 Forbes Avenue - * Suite 302 - * Pittsburgh, PA 15213 - * (412) 268-7393, fax: (412) 268-7395 - * innovation@andrew.cmu.edu - * - * 4. Redistributions of any form whatsoever must retain the following - * acknowledgment: - * "This product includes software developed by Computing Services - * at Carnegie Mellon University (http://www.cmu.edu/computing/)." - * - * CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO - * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY - * AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE - * FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN - * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING - * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - * - */ -#include - -#include -#include -#include - -#include "lib/gzuncat.h" -#include "lib/libconfig.h" -#include "lib/xunlink.h" - -#include "imap/imap_err.h" -#include "imap/sync_support.h" - -#include "backup/backup.h" - -#define LIBCYRUS_BACKUP_SOURCE /* this file is part of libcyrus_backup */ -#include "backup/lcb_internal.h" -#include "backup/lcb_sqlconsts.h" - -static size_t compact_minsize = 0; -static size_t compact_maxsize = 0; -static int compact_work_threshold = 0; - -static void compact_readconfig(void) -{ - /* read and normalise config values */ - if (compact_minsize == 0) { - compact_minsize = (size_t) - MAX(0, config_getbytesize(IMAPOPT_BACKUP_COMPACT_MINSIZE, 'K')); - } - - if (compact_maxsize == 0) { - compact_maxsize = (size_t) - MAX(0, config_getbytesize(IMAPOPT_BACKUP_COMPACT_MAXSIZE, 'K')); - } - - if (compact_work_threshold == 0) { - compact_work_threshold = - MAX(1, config_getint(IMAPOPT_BACKUP_COMPACT_WORK_THRESHOLD)); - } -} - -static int compact_open(const char *name, - struct backup **originalp, - struct backup **compactp, - enum backup_open_nonblock nonblock) -{ - struct backup *original = NULL; - struct backup *compact = NULL; - - struct buf original_data_fname = BUF_INITIALIZER; - struct buf original_index_fname = BUF_INITIALIZER; - struct buf compact_data_fname = BUF_INITIALIZER; - struct buf compact_index_fname = BUF_INITIALIZER; - - int r; - - buf_printf(&original_data_fname, "%s", name); - buf_printf(&original_index_fname, "%s.index", name); - buf_printf(&compact_data_fname, "%s.new", name); - buf_printf(&compact_index_fname, "%s.index.new", name); - - r = backup_real_open(&original, - buf_cstring(&original_data_fname), - buf_cstring(&original_index_fname), - BACKUP_OPEN_NOREINDEX, - nonblock, - BACKUP_OPEN_NOCREATE); - if (r) goto done; - - r = backup_real_open(&compact, - buf_cstring(&compact_data_fname), - buf_cstring(&compact_index_fname), - BACKUP_OPEN_NOREINDEX, - BACKUP_OPEN_NONBLOCK, // FIXME think about this - BACKUP_OPEN_CREATE_EXCL); - if (r) { - backup_close(&original); - goto done; - } - - *originalp = original; - *compactp = compact; - -done: - buf_free(&original_data_fname); - buf_free(&original_index_fname); - buf_free(&compact_data_fname); - buf_free(&compact_index_fname); - - return r; -} - -static int compact_closerename(struct backup **originalp, - struct backup **compactp, - time_t now) -{ - struct backup *original = *originalp; - struct backup *compact = *compactp; - struct buf ts_data_fname = BUF_INITIALIZER; - struct buf ts_index_fname = BUF_INITIALIZER; - int r; - - buf_printf(&ts_data_fname, "%s.%ld", original->data_fname, now); - buf_printf(&ts_index_fname, "%s.%ld", original->index_fname, now); - - /* link original files into timestamped names */ - r = link(original->data_fname, buf_cstring(&ts_data_fname)); - if (!r) r = link(original->index_fname, buf_cstring(&ts_index_fname)); - - if (r) { - /* on error, trash the new links and bail out */ - xunlink(buf_cstring(&ts_data_fname)); - xunlink(buf_cstring(&ts_index_fname)); - goto done; - } - - /* replace original files with compacted files */ - r = rename(compact->data_fname, original->data_fname); - if (!r) r = rename(compact->index_fname, original->index_fname); - - if (r) { - /* on error, put original files back */ - xunlink(original->data_fname); - xunlink(original->index_fname); - if (link(buf_cstring(&ts_data_fname), original->data_fname)) - xsyslog(LOG_ERR, "IOERROR: failed to link file back!", - "source=<%s> dest=<%s>", - buf_cstring(&ts_data_fname), - original->data_fname); - if (link(buf_cstring(&ts_index_fname), original->index_fname)) - xsyslog(LOG_ERR, "IOERROR: failed to link file back!", - "source=<%s> dest=<%s>", - buf_cstring(&ts_index_fname), - original->index_fname); - goto done; - } - - /* finally, clean up the timestamped ones */ - if (!config_getswitch(IMAPOPT_BACKUP_KEEP_PREVIOUS)) { - xunlink(buf_cstring(&ts_data_fname)); - xunlink(buf_cstring(&ts_index_fname)); - } - - /* release our locks */ - backup_close(originalp); - backup_close(compactp); - -done: - buf_free(&ts_data_fname); - buf_free(&ts_index_fname); - return r; -} - -/* a small chunk is candidate for combining with the next - * if the sum of their lengths is smaller than max_chunksize - */ -static int want_combine(size_t length, const struct backup_chunk *next_chunk) -{ - /* can't combine if there's no subsequent chunk */ - if (!next_chunk) - return 0; - - /* don't combine if the chunks are both big enough already */ - if (length >= compact_minsize && next_chunk->length >= compact_minsize) - return 0; - - /* no upper size limit, so combine them */ - if (!compact_maxsize) - return 1; - - /* don't combine if upper size limit may be exceeded */ - if (length + next_chunk->length > compact_maxsize) - return 0; - - /* combine */ - return 1; -} - -/* a large chunk is candidate for splitting - * if it won't create a new too-small chunk - */ -static int want_split(const struct backup_chunk *chunk, const size_t *wrotep) -{ - /* don't split if there's no maximum size */ - if (!compact_maxsize) - return 0; - - /* don't split if we're writing and haven't written enough */ - if (wrotep && *wrotep < compact_maxsize) - return 0; - - /* don't split if the chunk isn't long enough */ - if (chunk->length < compact_maxsize + compact_minsize) - return 0; - - /* if we're not writing, we're done */ - if (!wrotep) - return 1; - - /* we might have written past the desirable split boundary due to a big - * dlist, so check whether the remainder is worth splitting for */ - size_t new_chunk_size = chunk->length - *wrotep; - - /* split if what's left is big enough to be its own chunk */ - if (new_chunk_size > compact_minsize) - return 1; - - /* don't split it */ - return 0; -} - -static int compact_required(struct backup_chunk_list *all_chunks, - struct backup_chunk_list *keep_chunks) -{ - struct backup_chunk *chunk; - int to_be_compacted = 0; - - compact_readconfig(); - - /* count chunks to be discarded */ - if (all_chunks->count > keep_chunks->count) - to_be_compacted += all_chunks->count - keep_chunks->count; - - if (to_be_compacted >= compact_work_threshold) - return 1; - - /* nothing more to do if there are no boundaries defined */ - if (!compact_minsize && !compact_maxsize) - return 0; - - /* nothing more to do if the boundaries are contradictory */ - if (compact_minsize && compact_maxsize - && compact_minsize >= compact_maxsize) - return 0; - - /* count chunks to be combined/split */ - for (chunk = keep_chunks->head; chunk; chunk = chunk->next) { - if (want_combine(chunk->length, chunk->next)) - to_be_compacted++; - - if (want_split(chunk, NULL)) - to_be_compacted++; - - if (to_be_compacted >= compact_work_threshold) - return 1; - } - - return 0; -} - -static int want_append_message(struct dlist *dlist, - struct sync_msgid_list *keep_message_guids) -{ - struct dlist *di, *next; - - for (di = dlist->head; di; di = next) { - struct message_guid *guid = NULL; - - /* save next pointer now in case we need to unstitch */ - next = di->next; - - if (!dlist_tofile(di, NULL, &guid, NULL, NULL)) - continue; - - if (!sync_msgid_lookup(keep_message_guids, guid)) { - syslog(LOG_DEBUG, "%s: MESSAGE no longer needed: %s", - __func__, message_guid_encode(guid)); - dlist_unstitch(dlist, di); - dlist_unlink_files(di); - dlist_free(&di); - } - } - - if (dlist->head) { - syslog(LOG_DEBUG, "%s: keeping MESSAGE line", __func__); - return 1; - } - - syslog(LOG_DEBUG, "%s: MESSAGE line has no more messages", __func__); - return 0; -} - -static int want_append_mailbox(struct backup *orig_backup, - int orig_chunk_id, - struct dlist *dlist) -{ - struct dlist *record = NULL; - const char *uniqueid = NULL; - struct backup_mailbox *mailbox = NULL; - int mailbox_last_chunk_id = 0; - - if (!dlist_getatom(dlist, "UNIQUEID", &uniqueid)) { - syslog(LOG_DEBUG, "%s: MAILBOX line with no UNIQUEID", __func__); - return 1; /* better keep it for now */ - } - - dlist_getlist(dlist, "RECORD", &record); - if (record && record->head) { - struct dlist *ki = NULL, *next = NULL; - int keep = 0; - - /* keep MAILBOX lines that contain the last RECORD for any message, */ - /* pruning out stale RECORDs */ - for (ki = record->head; ki; ki = next) { - const char *guid = NULL; - struct backup_mailbox_message *mailbox_message = NULL; - - /* save next pointer now in case we need to unstitch */ - next = ki->next; - - if (!dlist_getatom(ki, "GUID", &guid)) { - syslog(LOG_DEBUG, "%s: MAILBOX RECORD with no GUID", __func__); - keep = 1; /* better keep it for now */ - continue; - } - - mailbox_message = backup_get_mailbox_message(orig_backup, uniqueid, guid); - if (mailbox_message) { - int mailbox_message_last_chunk_id = mailbox_message->last_chunk_id; - backup_mailbox_message_free(&mailbox_message); - - if (mailbox_message_last_chunk_id == orig_chunk_id) { - syslog(LOG_DEBUG, "%s: keeping MAILBOX line containing last RECORD for guid %s", - __func__, guid); - keep = 1; - continue; - } - } - - /* don't need this record */ - syslog(LOG_DEBUG, "%s: pruning stale MAILBOX RECORD for guid %s", - __func__, guid); - dlist_unstitch(record, ki); - dlist_unlink_files(ki); - dlist_free(&ki); - } - - if (keep) return 1; - } - - mailbox = backup_get_mailbox_by_uniqueid(orig_backup, uniqueid, - BACKUP_MAILBOX_NO_RECORDS); - if (!mailbox) { - /* what? */ - syslog(LOG_DEBUG, "%s: couldn't find mailbox entry for uniqueid %s", __func__, uniqueid); - return 1; /* better keep it for now */ - } - - mailbox_last_chunk_id = mailbox->last_chunk_id; - backup_mailbox_free(&mailbox); - - if (mailbox_last_chunk_id == orig_chunk_id) { - /* keep all mailbox lines from the chunk recorded as its last */ - syslog(LOG_DEBUG, "%s: keeping MAILBOX line from its last known chunk", __func__); - return 1; - } - - syslog(LOG_DEBUG, "%s: discarding stale MAILBOX line (chunk %d, last %d, uniqueid %s)", - __func__, orig_chunk_id, mailbox_last_chunk_id, uniqueid); - return 0; -} - -static int want_append(struct backup *orig_backup, - int orig_chunk_id, - struct dlist *dlist, - struct sync_msgid_list *keep_message_guids) -{ - if (strcmp(dlist->name, "MESSAGE") == 0) { - return want_append_message(dlist, keep_message_guids); - } - else if (strcmp(dlist->name, "MAILBOX") == 0) { - return want_append_mailbox(orig_backup, orig_chunk_id, dlist); - } - /* FIXME detect other stale data types */ - else { - return 1; - } -} - -static ssize_t _prot_fill_cb(unsigned char *buf, size_t len, void *rock) -{ - struct gzuncat *gzuc = (struct gzuncat *) rock; - int r = gzuc_read(gzuc, buf, len); - - if (r < 0) - xsyslog(LOG_ERR, "IOERROR: gzuc_read failed", - "return=<%d>", r); - if (r < -1) - errno = EIO; - - return r; -} - -static int _keep_message_guids_cb(const struct backup_message *message, - void *rock) -{ - struct sync_msgid_list *list = (struct sync_msgid_list *) rock; - sync_msgid_insert(list, message->guid); - return 0; -} - -/* returns: - * 0 on success - * 1 if compact was not needed - * negative on error - */ -EXPORTED int backup_compact(const char *name, - enum backup_open_nonblock nonblock, - int force, int verbose, FILE *out) -{ - struct backup *original = NULL; - struct backup *compact = NULL; - struct backup_chunk_list *all_chunks = NULL; - struct backup_chunk_list *keep_chunks = NULL; - struct backup_chunk *chunk = NULL; - struct sync_msgid_list *keep_message_guids = NULL; - struct gzuncat *gzuc = NULL; - struct protstream *in = NULL; - time_t since, chunk_start_time, ts; - int r; - - compact_readconfig(); - - r = compact_open(name, &original, &compact, nonblock); - if (r) return r; - - /* calculate current time after obtaining locks, in case of a wait */ - const time_t now = time(NULL); - - const int retention = config_getduration(IMAPOPT_BACKUP_RETENTION, 'd'); - if (retention > 0) { - since = now - retention; - } - else { - /* zero or negative retention means "keep forever" */ - since = -1; - } - - all_chunks = backup_get_chunks(original); - if (!all_chunks) goto error; - - keep_chunks = backup_get_live_chunks(original, since); - if (!keep_chunks) goto error; - - if (!force && !compact_required(all_chunks, keep_chunks)) { - /* nothing to do */ - backup_chunk_list_free(&all_chunks); - backup_chunk_list_free(&keep_chunks); - backup_unlink(&compact); - backup_close(&original); - return 1; - } - - if (verbose) { - fprintf(out, "keeping " SIZE_T_FMT " chunks:\n", keep_chunks->count); - - for (chunk = keep_chunks->head; chunk; chunk = chunk->next) { - fprintf(out, " %d", chunk->id); - } - - fprintf(out, "\n"); - } - - gzuc = gzuc_new(original->fd); - if (!gzuc) goto error; - - chunk_start_time = -1; - ts = 0; - struct buf cmd = BUF_INITIALIZER; - for (chunk = keep_chunks->head; chunk; chunk = chunk->next) { - keep_message_guids = sync_msgid_list_create(0); - r = backup_message_foreach(original, chunk->id, &since, - _keep_message_guids_cb, keep_message_guids); - if (r) goto error; - - gzuc_member_start_from(gzuc, chunk->offset); - - in = prot_readcb(_prot_fill_cb, gzuc); - - while (1) { - struct dlist *dl = NULL; - - int c = parse_backup_line(in, &ts, &cmd, &dl); - - if (c == EOF) { - const char *error = prot_error(in); - if (error && 0 != strcmp(error, PROT_EOF_STRING)) { - syslog(LOG_ERR, - "IOERROR: %s: error reading chunk at offset " OFF_T_FMT ", byte %i: %s", - name, chunk->offset, prot_bytes_in(in), error); - - if (out) - fprintf(out, "error reading chunk at offset " OFF_T_FMT ", byte %i: %s\n", - chunk->offset, prot_bytes_in(in), error); - - /* chunk is corrupt, discard the rest of it and get on with - * the next. the next replication will fill in anything that - * was lost. - */ - goto next_chunk; - } - - break; - } - - if (chunk_start_time == -1) { - r = backup_append_start(compact, &ts, BACKUP_APPEND_NOFLUSH); - if (r) goto error; - chunk_start_time = ts; - } - - // XXX if this line is worth keeping - if (want_append(original, chunk->id, dl, keep_message_guids)) { - // FIXME if message is removed due to unneeded chunk, - // subsequent mailbox lines for it will fail here - // so we need to be able to tell which lines apply to messages we don't want anymore - r = backup_append(compact, dl, &ts, BACKUP_APPEND_NOFLUSH); - if (r) goto error; - } - - dlist_unlink_files(dl); - dlist_free(&dl); - - // if this line put us over compact_maxsize - if (want_split(chunk, &compact->append_state->wrote)) { - r = backup_append_end(compact, &ts); - chunk_start_time = -1; - - if (verbose) { - fprintf(out, "splitting chunk %d\n", chunk->id); - } - } - } -next_chunk: - - // if we're due to start a new chunk - if (compact->append_state && compact->append_state->mode) { - if (!want_combine(compact->append_state->wrote, chunk->next)) { - r = backup_append_end(compact, &ts); - chunk_start_time = -1; - } - else if (verbose) { - fprintf(out, "combining chunks %d and %d\n", - chunk->id, chunk->next->id); - } - } - - prot_free(in); - in = NULL; - gzuc_member_end(gzuc, NULL); - - sync_msgid_list_free(&keep_message_guids); - } - buf_free(&cmd); - - if (compact->append_state && compact->append_state->mode) - backup_append_end(compact, &ts); - - gzuc_free(&gzuc); - - backup_chunk_list_free(&keep_chunks); - - /* if we get here okay, then the compact succeeded */ - r = compact_closerename(&original, &compact, now); - if (r) goto error; - - return 0; - -error: - if (in) prot_free(in); - if (gzuc) gzuc_free(&gzuc); - if (keep_message_guids) sync_msgid_list_free(&keep_message_guids); - if (all_chunks) backup_chunk_list_free(&all_chunks); - if (keep_chunks) backup_chunk_list_free(&keep_chunks); - if (compact) backup_unlink(&compact); - if (original) backup_close(&original); - - return r ? r : -1; -} diff --git a/backup/lcb_indexr.c b/backup/lcb_indexr.c deleted file mode 100644 index 81a2a6d660f..00000000000 --- a/backup/lcb_indexr.c +++ /dev/null @@ -1,1236 +0,0 @@ -/* lcb_indexr.c -- replication-based backup api - index reading functions - * - * Copyright (c) 1994-2015 Carnegie Mellon University. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * 3. The name "Carnegie Mellon University" must not be used to - * endorse or promote products derived from this software without - * prior written permission. For permission or any legal - * details, please contact - * Carnegie Mellon University - * Center for Technology Transfer and Enterprise Creation - * 4615 Forbes Avenue - * Suite 302 - * Pittsburgh, PA 15213 - * (412) 268-7393, fax: (412) 268-7395 - * innovation@andrew.cmu.edu - * - * 4. Redistributions of any form whatsoever must retain the following - * acknowledgment: - * "This product includes software developed by Computing Services - * at Carnegie Mellon University (http://www.cmu.edu/computing/)." - * - * CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO - * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY - * AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE - * FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN - * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING - * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - * - */ -#include - -#include -#include - -#include "lib/xmalloc.h" - -#include "backup/backup.h" - -#define LIBCYRUS_BACKUP_SOURCE /* this file is part of libcyrus_backup */ -#include "backup/lcb_internal.h" -#include "backup/lcb_sqlconsts.h" - -static int _column_int(sqlite3_stmt *stmt, int column) -{ - assert(sqlite3_column_type(stmt, column) == SQLITE_INTEGER || - sqlite3_column_type(stmt, column) == SQLITE_NULL); - return sqlite3_column_int(stmt, column); -} - -static sqlite3_int64 _column_int64(sqlite3_stmt *stmt, int column) -{ - assert(sqlite3_column_type(stmt, column) == SQLITE_INTEGER || - sqlite3_column_type(stmt, column) == SQLITE_NULL); - return sqlite3_column_int64(stmt, column); -} - -static const char * _column_text(sqlite3_stmt *stmt, int column) -{ - assert(sqlite3_column_type(stmt, column) == SQLITE_TEXT || - sqlite3_column_type(stmt, column) == SQLITE_NULL); - return (const char *) sqlite3_column_text(stmt, column); -} - -static int _get_mailbox_id_cb(sqlite3_stmt *stmt, void *rock) { - int *idp = (int *) rock; - - *idp = _column_int(stmt, 0); - - return 0; -} - -EXPORTED int backup_get_mailbox_id(struct backup *backup, const char *uniqueid) -{ - struct sqldb_bindval bval[] = { - { ":uniqueid", SQLITE_TEXT, { .s = uniqueid } }, - { NULL, SQLITE_NULL, { .s = NULL } }, - }; - - int id = -1; - - int r = sqldb_exec(backup->db, backup_index_mailbox_select_uniqueid_sql, - bval, _get_mailbox_id_cb, &id); - if (r) { - syslog(LOG_ERR, "%s: something went wrong: %i %s", - __func__, r, uniqueid); - } - - return id; -} - -static void backup_mailbox_message_list_add( - struct backup_mailbox_message_list *list, - struct backup_mailbox_message *mailbox_message) -{ - mailbox_message->next = NULL; - - if (!list->head) - list->head = mailbox_message; - - if (list->tail) - list->tail->next = mailbox_message; - - list->tail = mailbox_message; - - list->count++; -} - -EXPORTED struct backup_mailbox_message *backup_mailbox_message_list_remove( - struct backup_mailbox_message_list *list, - struct backup_mailbox_message *mailbox_message) -{ - struct backup_mailbox_message *node, *prev; - - assert(list != NULL); - assert(mailbox_message != NULL); - - prev = NULL; - node = list->head; - while (node && node != mailbox_message) { - prev = node; - node = node->next; - } - - if (!node) return NULL; - assert(node == mailbox_message); - - if (prev) { - prev->next = node->next; - } - else { - assert(node == list->head); - list->head = node->next; - } - - if (!node->next) { - assert(node == list->tail); - list->tail = prev; - } - - node->next = NULL; - list->count--; - return node; -} - -EXPORTED void backup_mailbox_message_list_empty( - struct backup_mailbox_message_list *list) -{ - struct backup_mailbox_message *mailbox_message, *next; - - mailbox_message = list->head; - while (mailbox_message) { - next = mailbox_message->next; - backup_mailbox_message_free(&mailbox_message); - mailbox_message = next; - } - - memset(list, 0, sizeof(*list)); -} - -EXPORTED void backup_mailbox_list_add(struct backup_mailbox_list *list, - struct backup_mailbox *mailbox) -{ - mailbox->next = NULL; - - if (!list->head) - list->head = mailbox; - - if (list->tail) - list->tail->next = mailbox; - - list->tail = mailbox; - - list->count++; -} - -EXPORTED struct backup_mailbox *backup_mailbox_list_remove( - struct backup_mailbox_list *list, - struct backup_mailbox *mailbox) -{ - struct backup_mailbox *node, *prev; - - assert(list != NULL); - assert(mailbox != NULL); - - prev = NULL; - node = list->head; - while (node && node != mailbox) { - prev = node; - node = node->next; - } - - if (!node) return NULL; - assert(node == mailbox); - - if (prev) { - prev->next = node->next; - } - else { - assert(node == list->head); - list->head = node->next; - } - - if (!node->next) { - assert(node == list->tail); - list->tail = prev; - } - - node->next = NULL; - list->count--; - return node; -} - -EXPORTED void backup_mailbox_list_empty(struct backup_mailbox_list *list) -{ - struct backup_mailbox *mailbox, *next; - - mailbox = list->head; - while (mailbox) { - next = mailbox->next; - backup_mailbox_free(&mailbox); - mailbox = next; - } - - memset(list, 0, sizeof(*list)); -} - -struct _mailbox_message_row_rock { - struct message_guid *match_guid; - struct backup_mailbox_message_list *save_list; - struct backup_mailbox_message **save_one; -}; - -static int _mailbox_message_row_cb(sqlite3_stmt *stmt, void *rock) -{ - struct _mailbox_message_row_rock *mbrrock = - (struct _mailbox_message_row_rock *) rock; - struct backup_mailbox_message *mailbox_message; - char *guid_str = NULL; - int r = 0; - - mailbox_message = xzmalloc(sizeof *mailbox_message); - - int column = 0; - mailbox_message->id = _column_int(stmt, column++); - mailbox_message->mailbox_id = _column_int(stmt, column++); - mailbox_message->mailbox_uniqueid = xstrdupnull(_column_text(stmt, column++)); - mailbox_message->message_id = _column_int(stmt, column++); - mailbox_message->last_chunk_id = _column_int(stmt, column++); - mailbox_message->uid = _column_int(stmt, column++); - mailbox_message->modseq = _column_int64(stmt, column++); - mailbox_message->last_updated = _column_int64(stmt, column++); - mailbox_message->flags = xstrdupnull(_column_text(stmt, column++)); - mailbox_message->internaldate = _column_int64(stmt, column++); - guid_str = xstrdupnull(_column_text(stmt, column++)); - mailbox_message->size = _column_int(stmt, column++); - mailbox_message->annotations = xstrdupnull(_column_text(stmt, column++)); - mailbox_message->expunged = _column_int(stmt, column++); - - message_guid_decode(&mailbox_message->guid, guid_str); - free(guid_str); - - if (mbrrock->save_list) { - if (!mbrrock->match_guid - || message_guid_equal(mbrrock->match_guid, &mailbox_message->guid)) { - backup_mailbox_message_list_add(mbrrock->save_list, mailbox_message); - mailbox_message = NULL; - } - } - else if (mbrrock->save_one) { - *mbrrock->save_one = mailbox_message; - mailbox_message = NULL; - } - - if (mailbox_message) - backup_mailbox_message_free(&mailbox_message); - - return r; -} - -EXPORTED struct backup_mailbox_message_list *backup_get_mailbox_messages( - struct backup *backup, - int chunk_id) -{ - struct backup_mailbox_message_list *mailbox_message_list = - xzmalloc(sizeof *mailbox_message_list); - - struct sqldb_bindval bval[] = { - { ":last_chunk_id", SQLITE_INTEGER, { .i = chunk_id } }, - { NULL, SQLITE_NULL, { .s = NULL } }, - }; - - const char *sql = chunk_id ? - backup_index_mailbox_message_select_chunkid_sql : - backup_index_mailbox_message_select_all_sql; - - struct _mailbox_message_row_rock mbrrock = { - NULL, - mailbox_message_list, - NULL, - }; - - int r = sqldb_exec(backup->db, sql, bval, _mailbox_message_row_cb, - &mbrrock); - - if (r) { - backup_mailbox_message_list_empty(mailbox_message_list); - free(mailbox_message_list); - return NULL; - } - - return mailbox_message_list; -} - -EXPORTED struct backup_mailbox_message *backup_get_mailbox_message( - struct backup *backup, - const char *uniqueid, - const char *guid) -{ - struct backup_mailbox_message *mailbox_message = NULL; - - struct sqldb_bindval bval[] = { - { ":uniqueid", SQLITE_TEXT, { .s = uniqueid } }, - { ":guid", SQLITE_TEXT, { .s = guid } }, - { NULL, SQLITE_NULL, { .s = NULL } }, - }; - - struct _mailbox_message_row_rock mbrrock = { - NULL, NULL, &mailbox_message, - }; - - int r = sqldb_exec(backup->db, backup_index_mailbox_message_select_one_sql, - bval, _mailbox_message_row_cb, &mbrrock); - - if (r) { - syslog(LOG_DEBUG, "%s: something went wrong: %i %s %s", - __func__, r, uniqueid, guid); - } - - return mailbox_message; -} - -struct _mailbox_row_rock { - sqldb_t *db; - backup_mailbox_foreach_cb proc; - void *rock; - struct message_guid *match_guid; - struct backup_mailbox_list *save_list; - struct backup_mailbox **save_one; - enum backup_mailbox_want_records want_records; -}; - -static int _mailbox_row_cb(sqlite3_stmt *stmt, void *rock) -{ - struct _mailbox_row_rock *mbrock = (struct _mailbox_row_rock *) rock; - struct backup_mailbox *mailbox = xzmalloc(sizeof *mailbox); - int r = 0; - - int column = 0; - mailbox->id = _column_int(stmt, column++); - mailbox->last_chunk_id = _column_int(stmt, column++); - mailbox->uniqueid = xstrdupnull(_column_text(stmt, column++)); - mailbox->mboxname = xstrdupnull(_column_text(stmt, column++)); - mailbox->mboxtype = xstrdupnull(_column_text(stmt, column++)); - mailbox->last_uid = _column_int(stmt, column++); - mailbox->highestmodseq = _column_int64(stmt, column++); - mailbox->recentuid = _column_int(stmt, column++); - mailbox->recenttime = _column_int64(stmt, column++); - mailbox->last_appenddate = _column_int64(stmt, column++); - mailbox->pop3_last_login = _column_int64(stmt, column++); - mailbox->pop3_show_after = _column_int64(stmt, column++); - mailbox->uidvalidity = _column_int(stmt, column++); - mailbox->partition = xstrdupnull(_column_text(stmt, column++)); - mailbox->acl = xstrdupnull(_column_text(stmt, column++)); - mailbox->options = xstrdupnull(_column_text(stmt, column++)); - mailbox->sync_crc = _column_int(stmt, column++); - mailbox->sync_crc_annot = _column_int(stmt, column++); - mailbox->quotaroot = xstrdupnull(_column_text(stmt, column++)); - mailbox->xconvmodseq = _column_int64(stmt, column++); - mailbox->annotations = xstrdupnull(_column_text(stmt, column++)); - mailbox->deleted = _column_int64(stmt, column++); - - if (mbrock->want_records) { - if (mbrock->want_records == BACKUP_MAILBOX_MATCH_RECORDS && !mbrock->match_guid) { - syslog(LOG_WARNING, "%s: request for guid-matched records without guid", - __func__); - /* will return all records */ - } - - mailbox->records = xzmalloc(sizeof *mailbox->records); - - struct sqldb_bindval bval[] = { - { ":mailbox_id", SQLITE_INTEGER, { .i = mailbox->id } }, - { NULL, SQLITE_NULL, { .s = NULL } }, - }; - - struct _mailbox_message_row_rock mbrrock = { - mbrock->match_guid, - mailbox->records, - NULL, - }; - - r = sqldb_exec(mbrock->db, - backup_index_mailbox_message_select_mailbox_sql, - bval, - _mailbox_message_row_cb, &mbrrock); - - if (r) goto error; - } - - if (mbrock->proc) - r = mbrock->proc(mailbox, mbrock->rock); - - if (mbrock->save_list) - backup_mailbox_list_add(mbrock->save_list, mailbox); - else if (mbrock->save_one) - *mbrock->save_one = mailbox; - else - backup_mailbox_free(&mailbox); - - return r; - -error: - if (mailbox) backup_mailbox_free(&mailbox); - return -1; -} - -EXPORTED int backup_mailbox_foreach(struct backup *backup, - int chunk_id, - enum backup_mailbox_want_records want_records, - backup_mailbox_foreach_cb cb, - void *rock) -{ - struct _mailbox_row_rock mbrock = { backup->db, cb, rock, NULL, - NULL, NULL, want_records}; - - struct sqldb_bindval bval[] = { - { ":last_chunk_id", SQLITE_INTEGER, { .i = chunk_id } }, - { NULL, SQLITE_NULL, { .s = NULL } }, - }; - - const char *sql = chunk_id ? - backup_index_mailbox_select_chunkid_sql : - backup_index_mailbox_select_all_sql; - - int r = sqldb_exec(backup->db, sql, bval, _mailbox_row_cb, &mbrock); - - return r; -} - -EXPORTED struct backup_mailbox_list *backup_get_mailboxes( - struct backup *backup, - int chunk_id, - enum backup_mailbox_want_records want_records) -{ - struct backup_mailbox_list *mailbox_list = xzmalloc(sizeof *mailbox_list); - - struct _mailbox_row_rock mbrock = { backup->db, NULL, NULL, NULL, - mailbox_list, NULL, want_records}; - - struct sqldb_bindval bval[] = { - { ":last_chunk_id", SQLITE_INTEGER, { .i = chunk_id } }, - { NULL, SQLITE_NULL, { .s = NULL } }, - }; - - const char *sql = chunk_id ? - backup_index_mailbox_select_chunkid_sql : - backup_index_mailbox_select_all_sql; - - int r = sqldb_exec(backup->db, sql, bval, _mailbox_row_cb, &mbrock); - - if (r) { - backup_mailbox_list_empty(mailbox_list); - free(mailbox_list); - return NULL; - } - - return mailbox_list; -} - -EXPORTED struct backup_mailbox_list *backup_get_mailboxes_by_message( - struct backup *backup, - const struct backup_message *message, - enum backup_mailbox_want_records want_records) -{ - char *guid = xstrdup(message_guid_encode(message->guid)); - struct backup_mailbox_list *mailbox_list = xzmalloc(sizeof *mailbox_list); - - struct _mailbox_row_rock mbrock = { backup->db, NULL, NULL, NULL, - mailbox_list, NULL, want_records }; - - if (want_records == BACKUP_MAILBOX_MATCH_RECORDS) - mbrock.match_guid = message->guid; - - struct sqldb_bindval bval[] = { - { ":guid", SQLITE_TEXT, { .s = guid } }, - { NULL, SQLITE_NULL, { .s = NULL } }, - }; - - int r = sqldb_exec(backup->db, backup_index_mailbox_select_message_guid_sql, - bval, _mailbox_row_cb, &mbrock); - - free(guid); - - if (r) { - backup_mailbox_list_empty(mailbox_list); - free(mailbox_list); - return NULL; - } - - return mailbox_list; -} - -EXPORTED struct backup_mailbox *backup_get_mailbox_by_uniqueid( - struct backup *backup, - const char *uniqueid, - enum backup_mailbox_want_records want_records) -{ - struct backup_mailbox *mailbox = NULL; - - struct _mailbox_row_rock mbrock = { backup->db, NULL, NULL, NULL, - NULL, &mailbox, want_records }; - - struct sqldb_bindval bval[] = { - { ":uniqueid", SQLITE_TEXT, { .s = uniqueid } }, - { NULL, SQLITE_NULL, { .s = NULL } }, - }; - - int r = sqldb_exec(backup->db, backup_index_mailbox_select_uniqueid_sql, - bval, _mailbox_row_cb, &mbrock); - - if (r) { - if (mailbox) backup_mailbox_free(&mailbox); - return NULL; - } - - return mailbox; -} - -EXPORTED struct backup_mailbox *backup_get_mailbox_by_name( - struct backup *backup, - const mbname_t *mbname, - enum backup_mailbox_want_records want_records) -{ - struct backup_mailbox *mailbox = NULL; - - struct _mailbox_row_rock mbrock = { backup->db, NULL, NULL, NULL, - NULL, &mailbox, want_records }; - - struct sqldb_bindval bval[] = { - { ":mboxname", SQLITE_TEXT, { .s = mbname_intname(mbname) } }, - { NULL, SQLITE_NULL, { .s = NULL } }, - }; - - int r = sqldb_exec(backup->db, backup_index_mailbox_select_mboxname_sql, - bval, _mailbox_row_cb, &mbrock); - - if (r) { - if (mailbox) backup_mailbox_free(&mailbox); - return NULL; - } - - return mailbox; -} - -EXPORTED struct dlist *backup_mailbox_to_dlist( - const struct backup_mailbox *mailbox) -{ - struct dlist *dl = dlist_newkvlist(NULL, "MAILBOX"); - - dlist_setatom(dl, "UNIQUEID", mailbox->uniqueid); - dlist_setatom(dl, "MBOXNAME", mailbox->mboxname); - dlist_setatom(dl, "MBOXTYPE", mailbox->mboxtype); - dlist_setnum32(dl, "LAST_UID", mailbox->last_uid); - dlist_setnum64(dl, "HIGHESTMODSEQ", mailbox->highestmodseq); - dlist_setnum32(dl, "RECENTUID", mailbox->recentuid); - dlist_setdate(dl, "RECENTTIME", mailbox->recenttime); - dlist_setdate(dl, "LAST_APPENDDATE", mailbox->last_appenddate); - dlist_setdate(dl, "POP3_LAST_LOGIN", mailbox->pop3_last_login); - dlist_setdate(dl, "POP3_SHOW_AFTER", mailbox->pop3_show_after); - dlist_setnum32(dl, "UIDVALIDITY", mailbox->uidvalidity); - dlist_setatom(dl, "PARTITION", mailbox->partition); - dlist_setatom(dl, "ACL", mailbox->acl); - dlist_setatom(dl, "OPTIONS", mailbox->options); - dlist_setnum32(dl, "SYNC_CRC", mailbox->sync_crc); - dlist_setnum32(dl, "SYNC_CRC_ANNOT", mailbox->sync_crc_annot); - dlist_setatom(dl, "QUOTAROOT", mailbox->quotaroot); - dlist_setnum64(dl, "XCONVMODSEQ", mailbox->xconvmodseq); - - /* if any flags or annotations from the index can't be parsed into dlist - * format, we just quietly leave them out, and trust sync_client to notice - * the difference and send updates to fix them */ - - if (mailbox->annotations) { - struct dlist *annots = NULL; - dlist_parsemap(&annots, 0, 1, mailbox->annotations, - strlen(mailbox->annotations)); - if (annots) { - annots->name = xstrdup("ANNOTATIONS"); - dlist_stitch(dl, annots); - } - } - - if (mailbox->records) { - struct dlist *records = dlist_newlist(NULL, "RECORD"); - struct backup_mailbox_message *mailbox_message = mailbox->records->head; - - while (mailbox_message) { - struct dlist *record = dlist_newkvlist(records, NULL); - struct dlist *flags = NULL; - - dlist_setnum32(record, "UID", mailbox_message->uid); - dlist_setnum64(record, "MODSEQ", mailbox_message->modseq); - dlist_setdate(record, "LAST_UPDATED", mailbox_message->last_updated); - dlist_setdate(record, "INTERNALDATE", mailbox_message->internaldate); - dlist_setguid(record, "GUID", &mailbox_message->guid); - dlist_setnum32(record, "SIZE", mailbox_message->size); - - /* FLAGS field is mandatory */ - if (mailbox_message->flags) { - dlist_parsemap(&flags, 0, 1, mailbox_message->flags, - strlen(mailbox_message->flags)); - flags->name = xstrdup("FLAGS"); - dlist_stitch(record, flags); - } - else { - flags = dlist_newlist(record, "FLAGS"); - } - - /* convert expunged to flag */ - if (mailbox_message->expunged) - dlist_setflag(flags, "FLAG", "\\Expunged"); - - if (mailbox_message->annotations) { - struct dlist *annots = NULL; - dlist_parsemap(&annots, 0, 1, mailbox_message->annotations, - strlen(mailbox_message->annotations)); - if (annots) { - annots->name = xstrdup("ANNOTATIONS"); - dlist_stitch(record, annots); - } - } - - mailbox_message = mailbox_message->next; - } - - dlist_stitch(dl, records); - } - - return dl; -} - -EXPORTED struct backup_mailbox *backup_mailbox_clone( - const struct backup_mailbox *mailbox) -{ - struct backup_mailbox *clone = xzmalloc(sizeof *clone); - - clone->id = mailbox->id; - clone->last_chunk_id = mailbox->last_chunk_id; - clone->last_uid = mailbox->last_uid; - clone->highestmodseq = mailbox->highestmodseq; - clone->recentuid = mailbox->recentuid; - clone->recenttime = mailbox->recenttime; - clone->last_appenddate = mailbox->last_appenddate; - clone->pop3_last_login = mailbox->pop3_last_login; - clone->pop3_show_after = mailbox->pop3_show_after; - clone->uidvalidity = mailbox->uidvalidity; - clone->sync_crc = mailbox->sync_crc; - clone->sync_crc_annot = mailbox->sync_crc_annot; - clone->xconvmodseq = mailbox->xconvmodseq; - clone->deleted = mailbox->deleted; - - clone->uniqueid = xstrdupnull(mailbox->uniqueid); - clone->mboxname = xstrdupnull(mailbox->mboxname); - clone->mboxtype = xstrdupnull(mailbox->mboxtype); - clone->partition = xstrdupnull(mailbox->partition); - clone->acl = xstrdupnull(mailbox->acl); - clone->options = xstrdupnull(mailbox->options); - clone->quotaroot = xstrdupnull(mailbox->quotaroot); - clone->annotations = xstrdupnull(mailbox->annotations); - - if (mailbox->records) { - struct backup_mailbox_message *iter; - - clone->records = xzmalloc(sizeof *clone->records); - - for (iter = mailbox->records->head; iter; iter = iter->next) { - backup_mailbox_message_list_add(clone->records, - backup_mailbox_message_clone(iter)); - } - } - - return clone; -} - -EXPORTED struct backup_mailbox_message *backup_mailbox_message_clone( - const struct backup_mailbox_message *orig) -{ - struct backup_mailbox_message *clone = xzmalloc(sizeof *clone); - - clone->id = orig->id; - clone->mailbox_id = orig->mailbox_id; - clone->message_id = orig->message_id; - clone->last_chunk_id = orig->last_chunk_id; - clone->uid = orig->uid; - clone->modseq = orig->modseq; - clone->last_updated = orig->last_updated; - clone->internaldate = orig->internaldate; - clone->guid = orig->guid; - clone->size = orig->size; - clone->expunged = orig->expunged; - - clone->mailbox_uniqueid = xstrdupnull(orig->mailbox_uniqueid); - clone->flags = xstrdupnull(orig->flags); - clone->annotations = xstrdupnull(orig->annotations); - - return clone; -} - -EXPORTED void backup_mailbox_message_free( - struct backup_mailbox_message **mailbox_messagep) -{ - struct backup_mailbox_message *mailbox_message = *mailbox_messagep; - *mailbox_messagep = NULL; - - if (mailbox_message->flags) free(mailbox_message->flags); - if (mailbox_message->annotations) free(mailbox_message->annotations); - if (mailbox_message->mailbox_uniqueid) free(mailbox_message->mailbox_uniqueid); - - free(mailbox_message); -} - -EXPORTED void backup_mailbox_free(struct backup_mailbox **mailboxp) -{ - struct backup_mailbox *mailbox = *mailboxp; - *mailboxp = NULL; - - if (mailbox->uniqueid) free(mailbox->uniqueid); - if (mailbox->mboxname) free(mailbox->mboxname); - if (mailbox->mboxtype) free(mailbox->mboxtype); - if (mailbox->partition) free(mailbox->partition); - if (mailbox->acl) free(mailbox->acl); - if (mailbox->options) free(mailbox->options); - if (mailbox->quotaroot) free(mailbox->quotaroot); - if (mailbox->annotations) free(mailbox->annotations); - - if (mailbox->records) { - backup_mailbox_message_list_empty(mailbox->records); - free(mailbox->records); - } - - free(mailbox); -} - -static int _get_message_id_cb(sqlite3_stmt *stmt, void *rock) { - int *idp = (int *) rock; - - *idp = _column_int(stmt, 0); - - return 0; -} - -EXPORTED int backup_get_message_id(struct backup *backup, const char *guid) -{ - struct sqldb_bindval bval[] = { - { ":guid", SQLITE_TEXT, { .s = guid } }, - { NULL, SQLITE_NULL, { .s = NULL } }, - }; - - int id = 0; - - int r = sqldb_exec(backup->db, backup_index_message_select_guid_sql, bval, - _get_message_id_cb, &id); - if (r) { - syslog(LOG_ERR, "%s: something went wrong: %i %s", - __func__, r, guid); - return -1; - } - - return id; -} - -EXPORTED void backup_message_free(struct backup_message **messagep) -{ - struct backup_message *message = *messagep; - *messagep = NULL; - - if (message->guid) free(message->guid); - if (message->partition) free(message->partition); - - free(message); -} - -struct message_row_rock { - backup_message_foreach_cb proc; - void *rock; - struct backup_message **save_one; -}; - -static int _message_row_cb(sqlite3_stmt *stmt, void *rock) -{ - struct message_row_rock *mrock = (struct message_row_rock *) rock; - struct backup_message *message = xzmalloc(sizeof *message); - const char *guid_str = NULL; - int column = 0; - int r = 0; - - message->id = _column_int(stmt, column++); - guid_str = _column_text(stmt, column++); - message->partition = xstrdupnull(_column_text(stmt, column++)); - message->chunk_id = _column_int(stmt, column++); - message->offset = _column_int64(stmt, column++); - message->length = _column_int64(stmt, column++); - - message->guid = xzmalloc(sizeof *message->guid); - if (!message_guid_decode(message->guid, guid_str)) goto error; - - if (mrock->proc) - r = mrock->proc(message, mrock->rock); - - if (mrock->save_one) - *mrock->save_one = message; - else - backup_message_free(&message); - - return r; - -error: - if (message) backup_message_free(&message); - return -1; -} - -EXPORTED struct backup_message *backup_get_message(struct backup *backup, - const struct message_guid *guid) -{ - struct sqldb_bindval bval[] = { - { ":guid", SQLITE_TEXT, { .s = message_guid_encode(guid) } }, - { NULL, SQLITE_NULL, { .s = NULL } }, - }; - - struct backup_message *bm = NULL; - - struct message_row_rock mrock = { NULL, NULL, &bm }; - - int r = sqldb_exec(backup->db, backup_index_message_select_guid_sql, bval, - _message_row_cb, &mrock); - if (r) { - syslog(LOG_ERR, "%s: something went wrong: %i %s", - __func__, r, message_guid_encode(guid)); - if (bm) backup_message_free(&bm); - return NULL; - } - - return bm; -} - -EXPORTED int backup_message_foreach(struct backup *backup, - int chunk_id, const time_t *sincep, - backup_message_foreach_cb cb, void *rock) -{ - const char *sql = NULL; - - struct sqldb_bindval bval[] = { - { ":chunk_id", SQLITE_INTEGER, { .i = chunk_id } }, - { ":since", SQLITE_NULL, { .s = NULL } }, - { NULL, SQLITE_NULL, { .s = NULL } }, - }; - - struct message_row_rock mrock = { cb, rock, NULL }; - - if (chunk_id) { - if (sincep) { - struct sqldb_bindval *since_bval = &bval[1]; - assert(strcmp(since_bval->name, ":since") == 0); - since_bval->type = SQLITE_INTEGER; - since_bval->val.i = *sincep; - sql = backup_index_message_select_live_chunkid_sql; - } - else { - sql = backup_index_message_select_chunkid_sql; - } - } - else { - sql = backup_index_message_select_all_sql; - } - - return sqldb_exec(backup->db, sql, bval, _message_row_cb, &mrock); -} - -EXPORTED void backup_chunk_list_add(struct backup_chunk_list *list, - struct backup_chunk *chunk) -{ - chunk->next = NULL; - - if (list->tail) - list->tail->next = chunk; - - if (!list->head) - list->head = chunk; - - list->tail = chunk; - list->count++; -} - -EXPORTED void backup_chunk_list_empty(struct backup_chunk_list *list) -{ - struct backup_chunk *curr, *next; - curr = list->head; - while (curr) { - next = curr->next; - backup_chunk_free(&curr); - curr = next; - } - - list->head = list->tail = NULL; - list->count = 0; -} - -EXPORTED void backup_chunk_list_free(struct backup_chunk_list **chunk_listp) -{ - struct backup_chunk_list *chunk_list = *chunk_listp; - *chunk_listp = NULL; - - backup_chunk_list_empty(chunk_list); - free(chunk_list); -} - -struct _chunk_row_rock { - struct backup_chunk_list *save_list; - struct backup_chunk **save_one; -}; - -static int _chunk_row_cb(sqlite3_stmt *stmt, void *rock) -{ - struct _chunk_row_rock *crock = (struct _chunk_row_rock *) rock; - - struct backup_chunk *chunk = xzmalloc(sizeof(*chunk)); - - int column = 0; - chunk->id = _column_int(stmt, column++); - chunk->ts_start = _column_int64(stmt, column++); - chunk->ts_end = _column_int64(stmt, column++); - chunk->offset = _column_int64(stmt, column++); - chunk->length = _column_int64(stmt, column++); - chunk->file_sha1 = xstrdupnull(_column_text(stmt, column++)); - chunk->data_sha1 = xstrdupnull(_column_text(stmt, column++)); - - if (crock->save_list) { - backup_chunk_list_add(crock->save_list, chunk); - } - else if (crock->save_one) { - *crock->save_one = chunk; - } - else { - syslog(LOG_DEBUG, "%s: useless invocation with nowhere to save to", __func__); - backup_chunk_free(&chunk); - } - - return 0; -} - -EXPORTED struct backup_chunk_list *backup_get_chunks(struct backup *backup) -{ - struct backup_chunk_list *chunk_list = xzmalloc(sizeof *chunk_list); - - struct _chunk_row_rock crock = { chunk_list, NULL }; - - int r = sqldb_exec(backup->db, backup_index_chunk_select_all_sql, - NULL, _chunk_row_cb, &crock); - - if (r) { - backup_chunk_list_free(&chunk_list); - return NULL; - } - - return chunk_list; -} - -EXPORTED struct backup_chunk_list *backup_get_live_chunks(struct backup *backup, - time_t since) -{ - struct backup_chunk_list *chunk_list = xzmalloc(sizeof *chunk_list); - - struct _chunk_row_rock crock = { chunk_list, NULL }; - - struct sqldb_bindval bval[] = { - { ":since", SQLITE_INTEGER, { .i = since } }, - { NULL, SQLITE_NULL, { .s = NULL } }, - }; - - int r = sqldb_exec(backup->db, backup_index_chunk_select_live_sql, - bval, _chunk_row_cb, &crock); - - if (r) { - backup_chunk_list_free(&chunk_list); - return NULL; - } - - return chunk_list; -} - -EXPORTED struct backup_chunk *backup_get_chunk(struct backup *backup, - int chunk_id) -{ - struct backup_chunk *chunk = NULL; - struct _chunk_row_rock crock = { NULL, &chunk }; - - struct sqldb_bindval bval[] = { - { ":id", SQLITE_INTEGER, { .i = chunk_id } }, - { NULL, SQLITE_NULL, { .s = NULL } }, - }; - - int r = sqldb_exec(backup->db, backup_index_chunk_select_id_sql, - bval, _chunk_row_cb, &crock); - - if (r) { - if (chunk) backup_chunk_free(&chunk); - return NULL; - } - - return chunk; -} - -EXPORTED struct backup_chunk *backup_get_latest_chunk(struct backup *backup) -{ - struct backup_chunk *chunk = NULL; - struct _chunk_row_rock crock = { NULL, &chunk }; - - int r = sqldb_exec(backup->db, backup_index_chunk_select_latest_sql, - NULL, _chunk_row_cb, &crock); - - if (r) { - if (chunk) backup_chunk_free(&chunk); - return NULL; - } - - return chunk; -} - -EXPORTED void backup_chunk_free(struct backup_chunk **chunkp) -{ - struct backup_chunk *chunk = *chunkp; - *chunkp = NULL; - - if (chunk->file_sha1) free(chunk->file_sha1); - if (chunk->data_sha1) free(chunk->data_sha1); - - free(chunk); -} - -struct _seen_row_rock { - backup_seen_foreach_cb proc; - void *rock; -}; - -static int _seen_row_cb(sqlite3_stmt *stmt, void *rock) -{ - struct _seen_row_rock *seenrock = (struct _seen_row_rock *) rock; - struct backup_seen *seen = xzmalloc(sizeof *seen); - int r = 0; - - int column = 0; - seen->id = _column_int(stmt, column++); - seen->last_chunk_id = _column_int(stmt, column++); - seen->uniqueid = xstrdupnull(_column_text(stmt, column++)); - seen->lastread = _column_int64(stmt, column++); - seen->lastuid = _column_int(stmt, column++); - seen->lastchange = _column_int64(stmt, column++); - seen->seenuids = xstrdupnull(_column_text(stmt, column++)); - - if (seenrock->proc) - r = seenrock->proc(seen, seenrock->rock); - - backup_seen_free(&seen); - - return r; -} - -EXPORTED int backup_seen_foreach(struct backup *backup, - int chunk_id, - backup_seen_foreach_cb cb, - void *rock) -{ - struct _seen_row_rock seenrock = { cb, rock }; - - struct sqldb_bindval bval[] = { - { ":last_chunk_id", SQLITE_INTEGER, { .i = chunk_id } }, - { NULL, SQLITE_NULL, { .s = NULL } }, - }; - - const char *sql = chunk_id ? - backup_index_seen_select_chunkid_sql : - backup_index_seen_select_all_sql; - - int r = sqldb_exec(backup->db, sql, bval, _seen_row_cb, &seenrock); - - return r; -} - -EXPORTED void backup_seen_free(struct backup_seen **seenp) -{ - struct backup_seen *seen = *seenp; - *seenp = NULL; - - if (seen->uniqueid) free(seen->uniqueid); - if (seen->seenuids) free(seen->seenuids); - - free(seen); -} - -struct _subscription_row_rock { - backup_subscription_foreach_cb proc; - void *rock; -}; - -static int _subscription_row_cb(sqlite3_stmt *stmt, void *rock) -{ - struct _subscription_row_rock *subrock = (struct _subscription_row_rock *) rock; - struct backup_subscription *sub = xzmalloc(sizeof *sub); - int r = 0; - - int column = 0; - sub->id = _column_int(stmt, column++); - sub->last_chunk_id = _column_int(stmt, column++); - sub->mboxname = xstrdupnull(_column_text(stmt, column++)); - sub->unsubscribed = _column_int64(stmt, column++); - - if (subrock->proc) - r = subrock->proc(sub, subrock->rock); - - backup_subscription_free(&sub); - - return r; -} - -EXPORTED int backup_subscription_foreach(struct backup *backup, - int chunk_id, - backup_subscription_foreach_cb cb, - void *rock) -{ - struct _subscription_row_rock subrock = { cb, rock }; - - struct sqldb_bindval bval[] = { - { ":last_chunk_id", SQLITE_INTEGER, { .i = chunk_id } }, - { NULL, SQLITE_NULL, { .s = NULL } }, - }; - - const char *sql = chunk_id ? - backup_index_subscription_select_chunkid_sql : - backup_index_subscription_select_all_sql; - - int r = sqldb_exec(backup->db, sql, bval, _subscription_row_cb, &subrock); - - return r; -} - -EXPORTED void backup_subscription_free(struct backup_subscription **subp) -{ - struct backup_subscription *sub = *subp; - *subp = NULL; - - if (sub->mboxname) free(sub->mboxname); - - free(sub); -} - -struct _sieve_row_rock { - backup_sieve_foreach_cb proc; - void *rock; -}; - -static int _sieve_row_cb(sqlite3_stmt *stmt, void *rock) -{ - struct _sieve_row_rock *srock = (struct _sieve_row_rock *) rock; - struct backup_sieve *sieve = xzmalloc(sizeof *sieve); - int r = 0; - - int column = 0; - sieve->id = _column_int(stmt, column++); - sieve->chunk_id = _column_int(stmt, column++); - sieve->last_update = _column_int64(stmt, column++); - sieve->filename = xstrdupnull(_column_text(stmt, column++)); - message_guid_decode(&sieve->guid, _column_text(stmt, column++)); - sieve->offset = _column_int64(stmt, column++); - sieve->deleted = _column_int64(stmt, column++); - - if (srock->proc) - r = srock->proc(sieve, srock->rock); - - backup_sieve_free(&sieve); - - return r; -} - -EXPORTED int backup_sieve_foreach(struct backup *backup, - int chunk_id, - backup_sieve_foreach_cb cb, - void *rock) -{ - struct _sieve_row_rock srock = { cb, rock }; - - struct sqldb_bindval bval[] = { - { ":chunk_id", SQLITE_INTEGER, { .i = chunk_id } }, - { NULL, SQLITE_NULL, { .s = NULL } }, - }; - - const char *sql = chunk_id ? - backup_index_sieve_select_chunkid_sql : - backup_index_sieve_select_all_sql; - - int r = sqldb_exec(backup->db, sql, bval, _sieve_row_cb, &srock); - - return r; -} - -EXPORTED void backup_sieve_free(struct backup_sieve **sievep) -{ - struct backup_sieve *sieve = *sievep; - *sievep = NULL; - - if (sieve->filename) free(sieve->filename); - - free(sieve); -} diff --git a/backup/lcb_indexw.c b/backup/lcb_indexw.c deleted file mode 100644 index e676483afda..00000000000 --- a/backup/lcb_indexw.c +++ /dev/null @@ -1,708 +0,0 @@ -/* lcb_indexw.c -- replication-based backup api - index writing functions - * - * Copyright (c) 1994-2015 Carnegie Mellon University. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * 3. The name "Carnegie Mellon University" must not be used to - * endorse or promote products derived from this software without - * prior written permission. For permission or any legal - * details, please contact - * Carnegie Mellon University - * Center for Technology Transfer and Enterprise Creation - * 4615 Forbes Avenue - * Suite 302 - * Pittsburgh, PA 15213 - * (412) 268-7393, fax: (412) 268-7395 - * innovation@andrew.cmu.edu - * - * 4. Redistributions of any form whatsoever must retain the following - * acknowledgment: - * "This product includes software developed by Computing Services - * at Carnegie Mellon University (http://www.cmu.edu/computing/)." - * - * CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO - * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY - * AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE - * FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN - * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING - * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - * - */ -#include -#include - -#include "lib/xmalloc.h" - -#include "imap/dlist.h" -#include "imap/imap_err.h" - -#include "backup/backup.h" - -#define LIBCYRUS_BACKUP_SOURCE /* this file is part of libcyrus_backup */ -#include "backup/lcb_internal.h" -#include "backup/lcb_sqlconsts.h" - -static int _index_expunge(struct backup *backup, struct dlist *dl, - time_t ts, off_t dl_offset); -static int _index_mailbox(struct backup *backup, struct dlist *dl, - time_t ts, off_t dl_offset); -static int _index_unmailbox(struct backup *backup, struct dlist *dl, - time_t ts, off_t dl_offset); -static int _index_message(struct backup *backup, struct dlist *dl, - time_t ts, off_t dl_offset, size_t dl_len); -static int _index_rename(struct backup *backup, struct dlist *dl, - time_t ts, off_t dl_offset); -static int _index_seen(struct backup *backup, struct dlist *dl, - time_t ts, off_t dl_offset); -static int _index_sub(struct backup *backup, struct dlist *dl, - time_t ts, off_t dl_offset); -static int _index_sieve(struct backup *backup, struct dlist *dl, - time_t ts, off_t dl_offset); - -HIDDEN int backup_index(struct backup *backup, struct dlist *dlist, - time_t ts, off_t start, size_t len) -{ - int r = IMAP_PROTOCOL_ERROR; - - if (strcmp(dlist->name, "EXPUNGE") == 0) - r = _index_expunge(backup, dlist, ts, start); - else if (strcmp(dlist->name, "MAILBOX") == 0) - r = _index_mailbox(backup, dlist, ts, start); - else if (strcmp(dlist->name, "UNMAILBOX") == 0) - r = _index_unmailbox(backup, dlist, ts, start); - else if (strcmp(dlist->name, "MESSAGE") == 0) - r = _index_message(backup, dlist, ts, start, len); - else if (strcmp(dlist->name, "RENAME") == 0) - r = _index_rename(backup, dlist, ts, start); - else if (strcmp(dlist->name, "RESERVE") == 0) - r = 0; /* nothing to index for a reserve, just return success */ - else if (strcmp(dlist->name, "SEEN") == 0) - r = _index_seen(backup, dlist, ts, start); - else if (strcmp(dlist->name, "SUB") == 0) - r = _index_sub(backup, dlist, ts, start); - else if (strcmp(dlist->name, "UNSUB") == 0) - r = _index_sub(backup, dlist, ts, start); - else if (strcmp(dlist->name, "SIEVE") == 0) - r = _index_sieve(backup, dlist, ts, start); - else if (strcmp(dlist->name, "UNSIEVE") == 0) - r = _index_sieve(backup, dlist, ts, start); - else if (config_debug) { - struct buf tmp = BUF_INITIALIZER; - dlist_printbuf(dlist, 1, &tmp); - syslog(LOG_DEBUG, "ignoring unrecognised dlist: %s", buf_cstring(&tmp)); - buf_free(&tmp); - } - - return r; -} - -static int _index_expunge(struct backup *backup, struct dlist *dl, - time_t ts, off_t dl_offset) -{ - syslog(LOG_DEBUG, "indexing EXPUNGE at " OFF_T_FMT "...", dl_offset); - - const char *mboxname; - const char *uniqueid; - struct dlist *uidl; - struct dlist *di; - struct backup_mailbox *mailbox = NULL; - int r = 0; - - if (!dlist_getatom(dl, "MBOXNAME", &mboxname)) - return IMAP_PROTOCOL_BAD_PARAMETERS; - if (!dlist_getatom(dl, "UNIQUEID", &uniqueid)) - return IMAP_PROTOCOL_BAD_PARAMETERS; - if (!dlist_getlist(dl, "UID", &uidl)) - return IMAP_PROTOCOL_BAD_PARAMETERS; - - mbname_t *mbname = mbname_from_intname(mboxname); - mailbox = backup_get_mailbox_by_name(backup, mbname, 0); - mbname_free(&mbname); - - if (!mailbox) - return IMAP_MAILBOX_NONEXISTENT; - - /* verify that uniqueid matches */ - if (strcmp(mailbox->uniqueid, uniqueid) != 0) { - syslog(LOG_ERR, "%s: uniqueid mismatch for %s: %s on wire, %s in index", - __func__, mboxname, uniqueid, mailbox->uniqueid); - r = IMAP_PROTOCOL_BAD_PARAMETERS; - } - - for (di = uidl->head; di && !r; di = di->next) { - struct sqldb_bindval bval[] = { - { ":mailbox_id", SQLITE_INTEGER, { .i = mailbox->id } }, - { ":uid", SQLITE_INTEGER, { .i = dlist_num(di) } }, - { ":expunged", SQLITE_INTEGER, { .i = ts } }, - }; - - r = sqldb_exec(backup->db, backup_index_mailbox_message_expunge_sql, - bval, NULL, NULL); - if (r) r = IMAP_INTERNAL; - } - - backup_mailbox_free(&mailbox); - return r; -} - -static int _get_magic_flags(struct dlist *flags, int *is_expunged) -{ - struct dlist *found_expunged = NULL; - struct dlist *di; - int found = 0; - - assert(strcmp(flags->name, "FLAGS") == 0); - - for (di = flags->head; di; di = di->next) { - if (strcmp(di->sval, "\\Expunged") == 0) { - if (is_expunged) *is_expunged = 1; - found_expunged = di; - found++; - } - } - - if (found_expunged) { - dlist_unstitch(flags, found_expunged); - dlist_free(&found_expunged); - } - - return found; -} - -static int _index_mailbox(struct backup *backup, struct dlist *dl, - time_t ts, off_t dl_offset) -{ - syslog(LOG_DEBUG, "indexing MAILBOX at " OFF_T_FMT "...", dl_offset); - - const char *uniqueid = NULL; - const char *mboxname = NULL; - const char *mboxtype = NULL; - uint32_t last_uid = 0; - modseq_t highestmodseq = 0; - uint32_t recentuid = 0; - time_t recenttime = 0; - time_t last_appenddate = 0; - time_t pop3_last_login = 0; - time_t pop3_show_after = 0; - uint32_t uidvalidity = 0; - const char *partition = NULL; - const char *acl = NULL; - const char *options = NULL; - struct synccrcs synccrcs = { 0, 0 }; - const char *quotaroot = NULL; - modseq_t xconvmodseq = 0; - struct dlist *annotations = NULL; - struct buf annotations_buf = BUF_INITIALIZER; - struct dlist *record = NULL; - int r; - - if (!dlist_getatom(dl, "UNIQUEID", &uniqueid)) - return IMAP_PROTOCOL_BAD_PARAMETERS; - if (!dlist_getatom(dl, "MBOXNAME", &mboxname)) - return IMAP_PROTOCOL_BAD_PARAMETERS; - if (!dlist_getnum32(dl, "LAST_UID", &last_uid)) - return IMAP_PROTOCOL_BAD_PARAMETERS; - if (!dlist_getnum64(dl, "HIGHESTMODSEQ", &highestmodseq)) - return IMAP_PROTOCOL_BAD_PARAMETERS; - if (!dlist_getnum32(dl, "RECENTUID", &recentuid)) - return IMAP_PROTOCOL_BAD_PARAMETERS; - if (!dlist_getdate(dl, "RECENTTIME", &recenttime)) - return IMAP_PROTOCOL_BAD_PARAMETERS; - if (!dlist_getdate(dl, "LAST_APPENDDATE", &last_appenddate)) - return IMAP_PROTOCOL_BAD_PARAMETERS; - if (!dlist_getdate(dl, "POP3_LAST_LOGIN", &pop3_last_login)) - return IMAP_PROTOCOL_BAD_PARAMETERS; - if (!dlist_getnum32(dl, "UIDVALIDITY", &uidvalidity)) - return IMAP_PROTOCOL_BAD_PARAMETERS; - if (!dlist_getatom(dl, "PARTITION", &partition)) - return IMAP_PROTOCOL_BAD_PARAMETERS; - if (!dlist_getatom(dl, "ACL", &acl)) - return IMAP_PROTOCOL_BAD_PARAMETERS; - if (!dlist_getatom(dl, "OPTIONS", &options)) - return IMAP_PROTOCOL_BAD_PARAMETERS; - if (!dlist_getlist(dl, "RECORD", &record)) - return IMAP_PROTOCOL_BAD_PARAMETERS; - - /* optional */ - dlist_getlist(dl, "ANNOTATIONS", &annotations); - dlist_getdate(dl, "POP3_SHOW_AFTER", &pop3_show_after); - dlist_getatom(dl, "MBOXTYPE", &mboxtype); - dlist_getnum64(dl, "XCONVMODSEQ", &xconvmodseq); - - /* CRCs */ - dlist_getnum32(dl, "SYNC_CRC", &synccrcs.basic); - dlist_getnum32(dl, "SYNC_CRC_ANNOT", &synccrcs.annot); - - /* if we can't start a transaction, bail out before allocating anything else */ - r = sqldb_begin(backup->db, __func__); - if (r) return IMAP_INTERNAL; - - if (annotations) { - dlist_printbuf(annotations, 0, &annotations_buf); - } - - struct sqldb_bindval mbox_bval[] = { - { ":last_chunk_id", SQLITE_INTEGER, { .i = backup->append_state->chunk_id } }, - { ":uniqueid", SQLITE_TEXT, { .s = uniqueid } }, - { ":mboxname", SQLITE_TEXT, { .s = mboxname } }, - { ":mboxtype", SQLITE_TEXT, { .s = mboxtype } }, - { ":last_uid", SQLITE_INTEGER, { .i = last_uid } }, - { ":highestmodseq", SQLITE_INTEGER, { .i = highestmodseq } }, - { ":recentuid", SQLITE_INTEGER, { .i = recentuid } }, - { ":recenttime", SQLITE_INTEGER, { .i = recenttime } }, - { ":last_appenddate", SQLITE_INTEGER, { .i = last_appenddate } }, - { ":pop3_last_login", SQLITE_INTEGER, { .i = pop3_last_login } }, - { ":pop3_show_after", SQLITE_INTEGER, { .i = pop3_show_after } }, - { ":uidvalidity", SQLITE_INTEGER, { .i = uidvalidity } }, - { ":partition", SQLITE_TEXT, { .s = partition } }, - { ":acl", SQLITE_TEXT, { .s = acl } }, - { ":options", SQLITE_TEXT, { .s = options } }, - { ":sync_crc", SQLITE_INTEGER, { .i = synccrcs.basic } }, - { ":sync_crc_annot", SQLITE_INTEGER, { .i = synccrcs.annot } }, - { ":quotaroot", SQLITE_TEXT, { .s = quotaroot } }, - { ":xconvmodseq", SQLITE_INTEGER, { .i = xconvmodseq } }, - { ":annotations", SQLITE_TEXT, { .s = buf_cstring(&annotations_buf) } }, - { ":deleted", SQLITE_NULL, { .s = NULL } }, - { NULL, SQLITE_NULL, { .s = NULL } }, - }; - - r = sqldb_exec(backup->db, backup_index_mailbox_update_sql, - mbox_bval, NULL, NULL); - - if (!r && sqldb_changes(backup->db) == 0) { - r = sqldb_exec(backup->db, backup_index_mailbox_insert_sql, - mbox_bval, NULL, NULL); - if (r) { - syslog(LOG_DEBUG, "%s: something went wrong: %i insert %s", - __func__, r, mboxname); - } - } - else if (r) { - syslog(LOG_DEBUG, "%s: something went wrong: %i update %s", - __func__, r, mboxname); - } - - buf_free(&annotations_buf); - - if (r) goto error; - - if (record->head) { - int mailbox_id = backup_get_mailbox_id(backup, uniqueid); - struct dlist *ki = NULL; - - for (ki = record->head; ki; ki = ki->next) { - uint32_t uid = 0; - modseq_t modseq = 0; - uint32_t last_updated = 0; - struct dlist *flags = NULL; - struct buf flags_buf = BUF_INITIALIZER; - uint32_t internaldate; - const char *guid; - struct dlist *annotations = NULL; - struct buf annotations_buf = BUF_INITIALIZER; - int message_id = -1; - time_t expunged = 0; - - if (!dlist_getnum32(ki, "UID", &uid)) - goto error; - if (!dlist_getnum64(ki, "MODSEQ", &modseq)) - goto error; - if (!dlist_getnum32(ki, "LAST_UPDATED", &last_updated)) - goto error; - if (!dlist_getnum32(ki, "INTERNALDATE", &internaldate)) - goto error; - if (!dlist_getatom(ki, "GUID", &guid)) - goto error; - - /* XXX should this search for guid+size rather than just guid? */ - message_id = backup_get_message_id(backup, guid); - if (message_id == -1) { - syslog(LOG_DEBUG, "%s: something went wrong: %i %s %s", - __func__, r, mboxname, guid); - goto error; - } - if (message_id == 0) { - /* can't link this record, we don't have a message */ - /* possibly we're in compact, and have deleted it? */ - /* XXX this should probably be an error too, but can't be until compact is smarter */ - syslog(LOG_DEBUG, "%s: skipping %s record for %s: message not found", - __func__, mboxname, guid); - continue; - } - - dlist_getlist(ki, "FLAGS", &flags); - if (flags) { - int is_expunged = 0; - - _get_magic_flags(flags, &is_expunged); - - if (is_expunged) { - syslog(LOG_DEBUG, "%s: found expunge flag for message %s", - __func__, guid); - expunged = ts; - } - - dlist_printbuf(flags, 0, &flags_buf); - syslog(LOG_DEBUG, "%s: found flags for message %s: %s", - __func__, guid, buf_cstring(&flags_buf)); - } - - dlist_getlist(ki, "ANNOTATIONS", &annotations); - if (annotations) { - dlist_printbuf(annotations, 0, &annotations_buf); - } - - struct sqldb_bindval record_bval[] = { - { ":mailbox_id", SQLITE_INTEGER, { .i = mailbox_id } }, - { ":message_id", SQLITE_INTEGER, { .i = message_id } }, - { ":last_chunk_id", SQLITE_INTEGER, { .i = backup->append_state->chunk_id } }, - { ":uid", SQLITE_INTEGER, { .i = uid } }, - { ":modseq", SQLITE_INTEGER, { .i = modseq } }, - { ":last_updated", SQLITE_INTEGER, { .i = last_updated } }, - { ":flags", SQLITE_TEXT, { .s = buf_cstring(&flags_buf) } }, - { ":internaldate", SQLITE_INTEGER, { .i = internaldate } }, - { ":annotations", SQLITE_TEXT, { .s = buf_cstring(&annotations_buf) } }, - { ":expunged", SQLITE_NULL, { .s = NULL } }, - { NULL, SQLITE_NULL, { .s = NULL } }, - }; - - /* provide an expunged value if we have one */ - if (expunged) { - struct sqldb_bindval *expunged_bval = &record_bval[9]; - assert(strcmp(expunged_bval->name, ":expunged") == 0); - expunged_bval->type = SQLITE_INTEGER; - expunged_bval->val.i = expunged; - } - - r = sqldb_exec(backup->db, backup_index_mailbox_message_update_sql, - record_bval, NULL, NULL); - - if (!r && sqldb_changes(backup->db) == 0) { - r = sqldb_exec(backup->db, backup_index_mailbox_message_insert_sql, - record_bval, NULL, NULL); - if (r) { - syslog(LOG_DEBUG, "%s: something went wrong: %i insert %s %s", - __func__, r, mboxname, guid); - } - } - else if (r) { - syslog(LOG_DEBUG, "%s: something went wrong: %i update %s %s", - __func__, r, mboxname, guid); - } - - buf_free(&annotations_buf); - buf_free(&flags_buf); - - if (r) goto error; - } - } - - syslog(LOG_DEBUG, "%s: committing index change: %s", __func__, mboxname); - sqldb_commit(backup->db, __func__); - return 0; - -error: - syslog(LOG_DEBUG, "%s: rolling back index change: %s", __func__, mboxname); - sqldb_rollback(backup->db, __func__); - - return IMAP_INTERNAL; -} - -static int _index_unmailbox(struct backup *backup, struct dlist *dl, - time_t ts, off_t dl_offset) -{ - syslog(LOG_DEBUG, "indexing UNMAILBOX at " OFF_T_FMT "...", dl_offset); - - const char *mboxname = dl->sval; - - struct sqldb_bindval bval[] = { - { ":mboxname", SQLITE_TEXT, { .s = mboxname } }, - { ":deleted", SQLITE_INTEGER, { .i = ts } }, - { NULL, SQLITE_NULL, { .s = NULL } }, - }; - - int r = sqldb_exec(backup->db, backup_index_mailbox_delete_sql, bval, NULL, - NULL); - if (r) { - syslog(LOG_DEBUG, "%s: something went wrong: %i %s", - __func__, r, mboxname); - } - - return r ? IMAP_INTERNAL : 0; -} - -static int _index_message(struct backup *backup, struct dlist *dl, - time_t ts, off_t dl_offset, size_t dl_len) -{ - syslog(LOG_DEBUG, "indexing MESSAGE at " OFF_T_FMT " (" SIZE_T_FMT " bytes)...", dl_offset, dl_len); - (void) ts; - - struct dlist *di; - int r = 0; - - /* n.b. APPLY MESSAGE contains a list of messages, not just one */ - for (di = dl->head; di && !r; di = di->next) { - struct message_guid *guid = NULL; - const char *partition = NULL; - unsigned long size = 0; - - if (!dlist_tofile(di, &partition, &guid, &size, NULL)) - continue; - - struct sqldb_bindval bval[] = { - { ":guid", SQLITE_TEXT, { .s = message_guid_encode(guid) } }, - { ":partition", SQLITE_TEXT, { .s = partition } }, - { ":chunk_id", SQLITE_INTEGER, { .i = backup->append_state->chunk_id } }, - { ":offset", SQLITE_INTEGER, { .i = dl_offset } }, - { ":size", SQLITE_INTEGER, { .i = size } }, - { NULL, SQLITE_NULL, { .s = NULL } }, - }; - - r = sqldb_exec(backup->db, backup_index_message_update_sql, bval, NULL, - NULL); - - if (!r && sqldb_changes(backup->db) == 0) { - r = sqldb_exec(backup->db, backup_index_message_insert_sql, bval, - NULL, NULL); - if (r) { - syslog(LOG_DEBUG, "%s: something went wrong: %i insert message %s", - __func__, r, message_guid_encode(guid)); - } - } - else if (r) { - syslog(LOG_DEBUG, "%s: something went wrong: %i update message %s", - __func__, r, message_guid_encode(guid)); - } - } - - return r ? IMAP_INTERNAL : 0; -} - -static int _index_rename(struct backup *backup, struct dlist *dl, - time_t ts, off_t dl_offset) -{ - syslog(LOG_DEBUG, "indexing RENAME at " OFF_T_FMT, dl_offset); - (void) ts; - - const char *uniqueid = NULL; - const char *oldmboxname = NULL; - const char *newmboxname = NULL; - const char *partition = NULL; - uint32_t uidvalidity = 0; - int r = 0; - - /* XXX use uniqueid once sync proto includes it (D73) */ - dlist_getatom(dl, "UNIQUEID", &uniqueid); - (void) uniqueid; /* silence unused warning */ - - if (!dlist_getatom(dl, "OLDMBOXNAME", &oldmboxname)) - return IMAP_PROTOCOL_BAD_PARAMETERS; - if (!dlist_getatom(dl, "NEWMBOXNAME", &newmboxname)) - return IMAP_PROTOCOL_BAD_PARAMETERS; - if (!dlist_getatom(dl, "PARTITION", &partition)) - return IMAP_PROTOCOL_BAD_PARAMETERS; - if (!dlist_getnum32(dl, "UIDVALIDITY", &uidvalidity)) - return IMAP_PROTOCOL_BAD_PARAMETERS; - - struct sqldb_bindval mbox_bval[] = { -// { ":uniqueid", SQLITE_TEXT, { .s = uniqueid } }, - { ":oldmboxname", SQLITE_TEXT, { .s = oldmboxname } }, - { ":newmboxname", SQLITE_TEXT, { .s = newmboxname } }, - { ":partition", SQLITE_TEXT, { .s = partition } }, - { ":uidvalidity", SQLITE_INTEGER, { .i = uidvalidity } }, - { NULL, SQLITE_NULL, { .s = NULL } }, - }; - - r = sqldb_exec(backup->db, backup_index_mailbox_rename_sql, - mbox_bval, NULL, NULL); - - if (r) { - syslog(LOG_DEBUG, "%s: something went wrong: %i rename %s => %s", - __func__, r, oldmboxname, newmboxname); - } - - return r ? IMAP_INTERNAL : 0; -} - -static int _index_seen(struct backup *backup, struct dlist *dl, - time_t ts, off_t dl_offset) -{ - syslog(LOG_DEBUG, "indexing %s at " OFF_T_FMT, dl->name, dl_offset); - (void) ts; - - const char *uniqueid; - time_t lastread; - uint32_t lastuid; - time_t lastchange; - const char *seenuids; - int r; - - if (!dlist_getatom(dl, "UNIQUEID", &uniqueid)) - return IMAP_PROTOCOL_BAD_PARAMETERS; - if (!dlist_getdate(dl, "LASTREAD", &lastread)) - return IMAP_PROTOCOL_BAD_PARAMETERS; - if (!dlist_getnum32(dl, "LASTUID", &lastuid)) - return IMAP_PROTOCOL_BAD_PARAMETERS; - if (!dlist_getdate(dl, "LASTCHANGE", &lastchange)) - return IMAP_PROTOCOL_BAD_PARAMETERS; - if (!dlist_getatom(dl, "SEENUIDS", &seenuids)) - return IMAP_PROTOCOL_BAD_PARAMETERS; - - struct sqldb_bindval bval[] = { - { ":last_chunk_id", SQLITE_INTEGER, { .i = backup->append_state->chunk_id } }, - { ":uniqueid", SQLITE_TEXT, { .s = uniqueid } }, - { ":lastread", SQLITE_INTEGER, { .i = lastread } }, - { ":lastuid", SQLITE_INTEGER, { .i = lastuid } }, - { ":lastchange", SQLITE_INTEGER, { .i = lastchange } }, - { ":seenuids", SQLITE_TEXT, { .s = seenuids } }, - { NULL, SQLITE_NULL, { .s = NULL } }, - }; - - r = sqldb_exec(backup->db, backup_index_seen_update_sql, bval, - NULL, NULL); - - if (!r && sqldb_changes(backup->db) == 0) { - r = sqldb_exec(backup->db, backup_index_seen_insert_sql, bval, - NULL, NULL); - if (r) { - syslog(LOG_DEBUG, "%s: something went wrong: %i insert seen %s", - __func__, r, uniqueid); - } - } - else if (r) { - syslog(LOG_DEBUG, "%s: something went wrong: %i update seen %s", - __func__, r, uniqueid); - } - - return r ? IMAP_INTERNAL : 0; -} - -static int _index_sub(struct backup *backup, struct dlist *dl, - time_t ts, off_t dl_offset) -{ - syslog(LOG_DEBUG, "indexing %s at " OFF_T_FMT, dl->name, dl_offset); - - const char *mboxname = NULL; - int r; - - if (!dlist_getatom(dl, "MBOXNAME", &mboxname)) - return IMAP_PROTOCOL_BAD_PARAMETERS; - - struct sqldb_bindval bval[] = { - { ":last_chunk_id", SQLITE_INTEGER, { .i = backup->append_state->chunk_id } }, - { ":mboxname", SQLITE_TEXT, { .s = mboxname } }, - { ":unsubscribed", SQLITE_NULL, { .s = NULL } }, - { NULL, SQLITE_NULL, { .s = NULL } }, - }; - - /* set the unsubscribed time if this is an UNSUB */ - if (!strcmp(dl->name, "UNSUB")) { - syslog(LOG_DEBUG, "setting unsubscribed to %ld for %s", ts, mboxname); - struct sqldb_bindval *unsubscribed_bval = &bval[2]; - assert(strcmp(unsubscribed_bval->name, ":unsubscribed") == 0); - unsubscribed_bval->type = SQLITE_INTEGER; - unsubscribed_bval->val.i = ts; - } - - r = sqldb_exec(backup->db, backup_index_subscription_update_sql, bval, - NULL, NULL); - - if (!r && sqldb_changes(backup->db) == 0) { - r = sqldb_exec(backup->db, backup_index_subscription_insert_sql, bval, - NULL, NULL); - if (r) { - syslog(LOG_DEBUG, "%s: something went wrong: %i insert subscription %s", - __func__, r, mboxname); - } - } - else if (r) { - syslog(LOG_DEBUG, "%s: something went wrong: %i update subscription %s", - __func__, r, mboxname); - } - - return r ? IMAP_INTERNAL : 0; -} - -static int _index_sieve(struct backup *backup, struct dlist *dl, - time_t ts, off_t dl_offset) -{ - syslog(LOG_DEBUG, "indexing %s at " OFF_T_FMT, dl->name, dl_offset); - - const char *filename; - int r; - - if (!dlist_getatom(dl, "FILENAME", &filename)) - return IMAP_PROTOCOL_BAD_PARAMETERS; - - struct sqldb_bindval bval[] = { - { ":chunk_id", SQLITE_INTEGER, { .i = backup->append_state->chunk_id } }, - { ":last_update", SQLITE_NULL, { .s = NULL } }, - { ":filename", SQLITE_TEXT, { .s = filename } }, - { ":guid", SQLITE_NULL, { .s = NULL } }, - { ":offset", SQLITE_INTEGER, { .i = dl_offset } }, - { ":deleted", SQLITE_INTEGER, { .i = ts } }, - { NULL, SQLITE_NULL, { .s = NULL } }, - }; - - /* mark previous record(s) for this filename as deleted */ - r = sqldb_exec(backup->db, backup_index_sieve_delete_sql, bval, - NULL, NULL); - - if (r) { - syslog(LOG_DEBUG, "%s: something went wrong: %i delete sieve %s", - __func__, r, filename); - } - - if (!r && strcmp(dl->name, "SIEVE") == 0) { - time_t last_update; - const char *content; - size_t content_len; - struct message_guid guid; - - if (!dlist_getdate(dl, "LAST_UPDATE", &last_update)) - return IMAP_PROTOCOL_BAD_PARAMETERS; - if (!dlist_getmap(dl, "CONTENT", &content, &content_len)) - return IMAP_PROTOCOL_BAD_PARAMETERS; - - message_guid_generate(&guid, content, content_len); - - struct sqldb_bindval *last_update_bval = &bval[1]; - assert(strcmp(last_update_bval->name, ":last_update") == 0); - last_update_bval->type = SQLITE_INTEGER; - last_update_bval->val.i = last_update; - - struct sqldb_bindval *guid_bval = &bval[3]; - assert(strcmp(guid_bval->name, ":guid") == 0); - guid_bval->type = SQLITE_TEXT; - guid_bval->val.s = message_guid_encode(&guid); - - /* insert doesn't use :deleted, but clear it still just in case */ - struct sqldb_bindval *deleted_bval = &bval[5]; - assert(strcmp(deleted_bval->name, ":deleted") == 0); - deleted_bval->type = SQLITE_NULL; - deleted_bval->val.s = NULL; - - /* insert the new record for this filename */ - r = sqldb_exec(backup->db, backup_index_sieve_insert_sql, bval, - NULL, NULL); - - if (r) { - syslog(LOG_DEBUG, "%s: something went wrong: %i insert sieve %s", - __func__, r, filename); - } - } - - return r ? IMAP_INTERNAL : 0; -} diff --git a/backup/lcb_internal.c b/backup/lcb_internal.c deleted file mode 100644 index 9505ce9b68e..00000000000 --- a/backup/lcb_internal.c +++ /dev/null @@ -1,122 +0,0 @@ -/* lcb_internal.c -- replication-based backup api - internal utility functions - * - * Copyright (c) 1994-2016 Carnegie Mellon University. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * 3. The name "Carnegie Mellon University" must not be used to - * endorse or promote products derived from this software without - * prior written permission. For permission or any legal - * details, please contact - * Carnegie Mellon University - * Center for Technology Transfer and Enterprise Creation - * 4615 Forbes Avenue - * Suite 302 - * Pittsburgh, PA 15213 - * (412) 268-7393, fax: (412) 268-7395 - * innovation@andrew.cmu.edu - * - * 4. Redistributions of any form whatsoever must retain the following - * acknowledgment: - * "This product includes software developed by Computing Services - * at Carnegie Mellon University (http://www.cmu.edu/computing/)." - * - * CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO - * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY - * AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE - * FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN - * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING - * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - * - */ -#include - -#include -#include - -#include "lib/map.h" - -#include "imap/dlist.h" -#include "imap/imapparse.h" - -#include "backup/backup.h" - -#define LIBCYRUS_BACKUP_SOURCE /* this file is part of libcyrus_backup */ -#include "backup/lcb_internal.h" - -HIDDEN int parse_backup_line(struct protstream *in, time_t *ts, - struct buf *cmd, struct dlist **kin) -{ - struct dlist *dl = NULL; - struct buf buf = BUF_INITIALIZER; - int64_t t; - int c; - - c = prot_getc(in); - if (c == '#') - eatline(in, c); - else - prot_ungetc(c, in); - - c = getint64(in, &t); - if (c == EOF) - goto fail; - - c = getword(in, &buf); - if (c == EOF) - goto fail; - - c = dlist_parse(&dl, /*parsekeys*/ 1, /*isarchive*/ 0, 1, in); - - if (!dl) { - fprintf(stderr, "\ndidn't parse dlist, error %i\n", c); - goto fail; - } - - if (c == '\r') c = prot_getc(in); - if (c != '\n') { - fprintf(stderr, "expected newline, got '%c'\n", c); - eatline(in, c); - goto fail; - } - - if (kin) *kin = dl; - if (cmd) buf_copy(cmd, &buf); - if (ts) *ts = (time_t) t; - buf_free(&buf); - return c; - -fail: - if (dl) dlist_free(&dl); - buf_free(&buf); - return c; -} - -HIDDEN const char *sha1_file(int fd, const char *fname, size_t limit, - char buf[2 * SHA1_DIGEST_LENGTH + 1]) -{ - const char *map = NULL; - size_t len = 0, calc_len; - unsigned char sha1_raw[SHA1_DIGEST_LENGTH]; - int r; - - map_refresh(fd, /*onceonly*/ 1, &map, &len, MAP_UNKNOWN_LEN, fname, NULL); - calc_len = limit == SHA1_LIMIT_WHOLE_FILE ? len : MIN(limit, len); - xsha1((const unsigned char *) map, calc_len, sha1_raw); - map_free(&map, &len); - r = bin_to_hex(sha1_raw, SHA1_DIGEST_LENGTH, buf, BH_LOWER); - assert(r == 2 * SHA1_DIGEST_LENGTH); - - return buf; -} diff --git a/backup/lcb_internal.h b/backup/lcb_internal.h deleted file mode 100644 index 1581daf0cfa..00000000000 --- a/backup/lcb_internal.h +++ /dev/null @@ -1,123 +0,0 @@ -/* lcb_internal.h -- replication-based backup internals - * - * Copyright (c) 1994-2015 Carnegie Mellon University. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * 3. The name "Carnegie Mellon University" must not be used to - * endorse or promote products derived from this software without - * prior written permission. For permission or any legal - * details, please contact - * Carnegie Mellon University - * Center for Technology Transfer and Enterprise Creation - * 4615 Forbes Avenue - * Suite 302 - * Pittsburgh, PA 15213 - * (412) 268-7393, fax: (412) 268-7395 - * innovation@andrew.cmu.edu - * - * 4. Redistributions of any form whatsoever must retain the following - * acknowledgment: - * "This product includes software developed by Computing Services - * at Carnegie Mellon University (http://www.cmu.edu/computing/)." - * - * CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO - * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY - * AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE - * FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN - * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING - * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - * - */ - -#include "lib/sqldb.h" -#include "lib/xsha1.h" - -#include "imap/partlist.h" - -#ifndef LIBCYRUS_BACKUP_SOURCE -#error "backup/lcb_internal.h is for internal use by libcyrus_backup ONLY" -#else -#ifndef BACKUP_LCB_INTERNAL_H -#define BACKUP_LCB_INTERNAL_H - -enum { - BACKUP_APPEND_INACTIVE = 0, - BACKUP_APPEND_ACTIVE = 0x0001, - BACKUP_APPEND_INDEXONLY = 0x0002, -}; - -struct backup_append_state { - unsigned mode; - gzFile gzfile; - int chunk_id; - size_t wrote; - SHA1_CTX sha_ctx; -}; - -struct backup { - int fd; - char *data_fname; - char *index_fname; - char *oldindex_fname; - sqldb_t *db; - struct backup_append_state *append_state; -}; - -enum backup_open_reindex { - BACKUP_OPEN_NOREINDEX = 0, - BACKUP_OPEN_REINDEX = 1, -}; - -HIDDEN int backup_real_open(struct backup **backupp, - const char *data_fname, const char *index_fname, - enum backup_open_reindex reindex, - enum backup_open_nonblock nonblock, - enum backup_open_create create); - -int backup_real_append_start(struct backup *backup, - time_t ts, off_t offset, - const char *file_sha1, - int index_only, - enum backup_append_flush flush); - -int backup_real_append_end(struct backup *backup, time_t ts); - - -HIDDEN int backup_index(struct backup *backup, struct dlist *dlist, - time_t ts, off_t start, size_t len); - -/* parsing data from backup data stream files */ -int parse_backup_line(struct protstream *in, time_t *ts, - struct buf *cmd, struct dlist **kin); - -/* limit is how much of the file to calculate the sha1 of (in bytes), - * or SHA1_LIMIT_WHOLE_FILE for the whole file */ -#define SHA1_LIMIT_WHOLE_FILE ((size_t) -1) -const char *sha1_file(int fd, const char *fname, size_t limit, - char buf[2 * SHA1_DIGEST_LENGTH + 1]); - -struct backup_mailbox *backup_mailbox_list_remove( - struct backup_mailbox_list *list, - struct backup_mailbox *node); - -struct backup_mailbox_message *backup_mailbox_message_list_remove( - struct backup_mailbox_message_list *list, - struct backup_mailbox_message *mailbox_message); - -const char *partlist_backup_select(void); -int partlist_backup_foreach(partlist_foreach_cb proc, void *rock); -void partlist_backup_done(void); -#endif -#endif diff --git a/backup/lcb_partlist.c b/backup/lcb_partlist.c deleted file mode 100644 index 854d9013a85..00000000000 --- a/backup/lcb_partlist.c +++ /dev/null @@ -1,103 +0,0 @@ -/* lcb_partlist.c -- replication-based backup api - partlist functions - * - * Copyright (c) 1994-2016 Carnegie Mellon University. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * 3. The name "Carnegie Mellon University" must not be used to - * endorse or promote products derived from this software without - * prior written permission. For permission or any legal - * details, please contact - * Carnegie Mellon University - * Center for Technology Transfer and Enterprise Creation - * 4615 Forbes Avenue - * Suite 302 - * Pittsburgh, PA 15213 - * (412) 268-7393, fax: (412) 268-7395 - * innovation@andrew.cmu.edu - * - * 4. Redistributions of any form whatsoever must retain the following - * acknowledgment: - * "This product includes software developed by Computing Services - * at Carnegie Mellon University (http://www.cmu.edu/computing/)." - * - * CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO - * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY - * AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE - * FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN - * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING - * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - * - */ -#include - -#include "lib/libconfig.h" -#include "lib/xmalloc.h" - -#include "imap/partlist.h" - -static partlist_t *partlist_backup = NULL; - - -static void partlist_backup_init(void) -{ - if (partlist_backup) { - /* already done */ - return; - } - - partlist_backup = xzmalloc(sizeof(partlist_t)); - partlist_initialize( - partlist_backup, - NULL, - "backuppartition-", - NULL, - config_getstring(IMAPOPT_PARTITION_SELECT_EXCLUDE), - partlist_getmode(config_getstring(IMAPOPT_PARTITION_SELECT_MODE)), - config_getint(IMAPOPT_PARTITION_SELECT_SOFT_USAGE_LIMIT), - config_getint(IMAPOPT_PARTITION_SELECT_USAGE_REINIT) - ); -} - - -HIDDEN const char *partlist_backup_select(void) -{ - /* lazy loading */ - if (!partlist_backup) { - partlist_backup_init(); - } - - return (char *)partlist_select_value(partlist_backup); -} - - -HIDDEN int partlist_backup_foreach(partlist_foreach_cb proc, void *rock) -{ - /* lazy loading */ - if (!partlist_backup) { - partlist_backup_init(); - } - - return partlist_foreach(partlist_backup, proc, rock); -} - - -HIDDEN void partlist_backup_done(void) -{ - if (partlist_backup) { - partlist_free(partlist_backup); - free(partlist_backup); - partlist_backup = NULL; - } -} diff --git a/backup/lcb_read.c b/backup/lcb_read.c deleted file mode 100644 index f597c97c5bb..00000000000 --- a/backup/lcb_read.c +++ /dev/null @@ -1,267 +0,0 @@ -/* lcb_read.c -- replication-based backup api - read functions - * - * Copyright (c) 1994-2016 Carnegie Mellon University. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * 3. The name "Carnegie Mellon University" must not be used to - * endorse or promote products derived from this software without - * prior written permission. For permission or any legal - * details, please contact - * Carnegie Mellon University - * Center for Technology Transfer and Enterprise Creation - * 4615 Forbes Avenue - * Suite 302 - * Pittsburgh, PA 15213 - * (412) 268-7393, fax: (412) 268-7395 - * innovation@andrew.cmu.edu - * - * 4. Redistributions of any form whatsoever must retain the following - * acknowledgment: - * "This product includes software developed by Computing Services - * at Carnegie Mellon University (http://www.cmu.edu/computing/)." - * - * CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO - * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY - * AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE - * FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN - * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING - * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - * - */ -#include -#include - -#include "lib/gzuncat.h" -#include "lib/map.h" -#include "lib/prot.h" -#include "lib/util.h" - -#include "imap/imap_err.h" - -#include "backup/backup.h" - -#define LIBCYRUS_BACKUP_SOURCE /* this file is part of libcyrus_backup */ -#include "backup/lcb_internal.h" -#include "backup/lcb_sqlconsts.h" - -static ssize_t _prot_fill_cb(unsigned char *buf, size_t len, void *rock) -{ - struct gzuncat *gzuc = (struct gzuncat *) rock; - return gzuc_read(gzuc, buf, len); -} - -EXPORTED int backup_read_chunk_data(struct backup *backup, - const struct backup_chunk *chunk, - backup_read_data_cb proc, void *rock) -{ - struct gzuncat *gzuc = NULL; - struct buf buf = BUF_INITIALIZER; - int r = 0; - - gzuc = gzuc_new(backup->fd); - - gzuc_member_start_from(gzuc, chunk->offset); - - while (!gzuc_member_eof(gzuc)) { - char tmp[8192]; /* FIXME whatever */ - ssize_t n = gzuc_read(gzuc, tmp, sizeof(tmp)); - if (n <= 0) - break; - - buf_setmap(&buf, tmp, n); - - r = proc(&buf, rock); - - buf_reset(&buf); - } - gzuc_member_end(gzuc, NULL); - - gzuc_free(&gzuc); - buf_free(&buf); - return r; -} - -EXPORTED int backup_read_message_data(struct backup *backup, - const struct backup_message *message, - backup_read_data_cb proc, void *rock) -{ - struct backup_chunk *chunk = NULL; - struct gzuncat *gzuc = NULL; - struct dlist *dl = NULL; - struct dlist *di; - int r; - - chunk = backup_get_chunk(backup, message->chunk_id); - if (!chunk) return -1; - - gzuc = gzuc_new(backup->fd); - - gzuc_member_start_from(gzuc, chunk->offset); - r = gzuc_seekto(gzuc, message->offset); - if (r) return r; - - struct protstream *ps = prot_readcb(_prot_fill_cb, gzuc); - prot_setisclient(ps, 1); /* don't sync literals */ - r = parse_backup_line(ps, NULL, NULL, &dl); - prot_free(ps); - - gzuc_member_end(gzuc, NULL); - gzuc_free(&gzuc); - - for (di = dl->head; di; di = di->next) { - struct message_guid *guid = NULL; - const char *fname = NULL; - int fd; - - if (!dlist_tofile(di, NULL, &guid, NULL, &fname)) - continue; - - if (!message_guid_equal(message->guid, guid)) - continue; - - fd = open(fname, O_RDWR); - if (fd != -1) { - struct buf buf = BUF_INITIALIZER; - - buf_refresh_mmap(&buf, 1, fd, fname, MAP_UNKNOWN_LEN, NULL); - close(fd); - - r = proc(&buf, rock); - - buf_free(&buf); - } - - break; - } - - dlist_unlink_files(dl); - dlist_free(&dl); - - backup_chunk_free(&chunk); - - return r; -} - -/* we can't link against imap/sync_support.c within the backup library, - * so we need this nasty workaround where the caller provides a - * pointer to sync_msgid_lookup for us to use. - */ -EXPORTED int backup_prepare_message_upload(struct backup *backup, - const char *partition, - struct sync_msgid_list *msgid_list, - sync_msgid_lookup_func msgid_lookup, - struct dlist **uploadp) -{ - struct dlist *upload = NULL; - struct sync_msgid *msgid = NULL; - struct gzuncat *gzuc = NULL; - int r; - - /* nothing to do */ - if (!uploadp) return 0; - - upload = dlist_newlist(NULL, "MESSAGE"); - - gzuc = gzuc_new(backup->fd); - - for (msgid = msgid_list->head; msgid; msgid = msgid->next) { - struct backup_message *message = NULL; - struct backup_chunk *chunk = NULL; - struct dlist *dl = NULL; - struct dlist *di, *next; - - /* already uploaded */ - if (!msgid->need_upload) continue; - - message = backup_get_message(backup, &msgid->guid); - if (!message) { - syslog(LOG_ERR, "%s: couldn't find message %s in backup %s", - __func__, - message_guid_encode(&msgid->guid), - backup->data_fname); - goto next_msgid; - } - - chunk = backup_get_chunk(backup, message->chunk_id); - if (!chunk) goto next_msgid; - - /* read message contents from backup */ - gzuc_member_start_from(gzuc, chunk->offset); - r = gzuc_seekto(gzuc, message->offset); - if (!r) { - struct protstream *ps = prot_readcb(_prot_fill_cb, gzuc); - int c; - prot_setisclient(ps, 1); /* don't sync literals */ - c = parse_backup_line(ps, NULL, NULL, &dl); - prot_free(ps); - ps = NULL; - if (c == EOF) { - xsyslog(LOG_ERR, "IOERROR: parse_backup_line failed", - "guid=<%s> chunk=<%d> backup=<%s>", - message_guid_encode(&msgid->guid), - chunk->id, - backup->data_fname); - r = IMAP_IOERROR; - } - } - gzuc_member_end(gzuc, NULL); - if (r) goto next_msgid; - - /* A single backup line contains many messages, so process - * them all while they're already decompressed. - * Tricksy loop construct so we can unstitch safely. - */ - next = dl->head; - while ((di = next)) { - struct message_guid *guid = NULL; - struct sync_msgid *found_msgid = NULL; - - next = di->next; - - if (!dlist_tofile(di, NULL, &guid, NULL, NULL)) - continue; - - found_msgid = msgid_lookup(msgid_list, guid); - if (!found_msgid) - continue; - - /* found one we want, move to upload list */ - dlist_unstitch(dl, di); - dlist_stitch(upload, di); - - /* set the destination partition */ - if (di->part) free(di->part); - di->part = xstrdup(partition); - - /* flag that we're sending it */ - found_msgid->need_upload = 0; - msgid_list->toupload--; - } - -next_msgid: - if (dl) { - dlist_unlink_files(dl); - dlist_free(&dl); - } - - if (chunk) backup_chunk_free(&chunk); - if (message) backup_message_free(&message); - } - - if (gzuc) gzuc_free(&gzuc); - - *uploadp = upload; - return 0; -} diff --git a/backup/lcb_sqlconsts.c b/backup/lcb_sqlconsts.c deleted file mode 100644 index 87ceaa34fad..00000000000 --- a/backup/lcb_sqlconsts.c +++ /dev/null @@ -1,604 +0,0 @@ -/* lcb_sqlconsts.c -- backup index sql constants - * - * Copyright (c) 1994-2015 Carnegie Mellon University. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * 3. The name "Carnegie Mellon University" must not be used to - * endorse or promote products derived from this software without - * prior written permission. For permission or any legal - * details, please contact - * Carnegie Mellon University - * Center for Technology Transfer and Enterprise Creation - * 4615 Forbes Avenue - * Suite 302 - * Pittsburgh, PA 15213 - * (412) 268-7393, fax: (412) 268-7395 - * innovation@andrew.cmu.edu - * - * 4. Redistributions of any form whatsoever must retain the following - * acknowledgment: - * "This product includes software developed by Computing Services - * at Carnegie Mellon University (http://www.cmu.edu/computing/)." - * - * CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO - * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY - * AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE - * FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN - * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING - * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - * - */ - -#include "backup/lcb_sqlconsts.h" - -/* n.b. this will collapse internal whitespace, and swallow leading/trailing - * whitespace. be careful when concatenating QUOTE()d literals. - */ -#define QUOTE(...) #__VA_ARGS__ - -const int backup_index_version = 3; - -const char backup_index_initsql[] = QUOTE( - CREATE TABLE chunk( - id INTEGER PRIMARY KEY ASC, - ts_start INTEGER, - ts_end INTEGER, - offset INTEGER unique, - length INTEGER, - file_sha1 TEXT, - data_sha1 TEXT - ); - - CREATE TABLE message( - id INTEGER PRIMARY KEY ASC, - guid CHAR UNIQUE NOT NULL, - partition CHAR, - chunk_id INTEGER REFERENCES chunk(id), - offset INTEGER, - size INTEGER - ); - CREATE INDEX IF NOT EXISTS idx_msg_guid ON message(guid); - - CREATE TABLE mailbox( - id INTEGER PRIMARY KEY ASC, - last_chunk_id INTEGER NOT NULL REFERENCES chunk(id), - uniqueid CHAR UNIQUE NOT NULL, - mboxname CHAR NOT NULL, - mboxtype CHAR, - last_uid INTEGER, - highestmodseq INTEGER, - recentuid INTEGER, - recenttime INTEGER, - last_appenddate INTEGER, - pop3_last_login INTEGER, - pop3_show_after INTEGER, - uidvalidity INTEGER, - partition CHAR, - acl CHAR, - options CHAR, - sync_crc INTEGER, - sync_crc_annot INTEGER, - quotaroot CHAR, - xconvmodseq INTEGER, - annotations CHAR, - deleted INTEGER - ); - CREATE INDEX IF NOT EXISTS idx_bmb_uid ON mailbox(uniqueid); - - CREATE TABLE mailbox_message( - id INTEGER PRIMARY KEY ASC, - mailbox_id INTEGER NOT NULL REFERENCES mailbox(id), - message_id INTEGER NOT NULL REFERENCES message(id), - last_chunk_id INTEGER NOT NULL REFERENCES chunk(id), - uid INTEGER NOT NULL, - modseq INTEGER, - last_updated INTEGER, - flags CHAR, - internaldate INTEGER, - annotations CHAR, - expunged INTEGER, - UNIQUE (mailbox_id, message_id) - ); - CREATE INDEX IF NOT EXISTS idx_mbr_uid ON mailbox_message(uid); - - CREATE TABLE subscription( - id INTEGER PRIMARY KEY ASC, - last_chunk_id INTEGER NOT NULL REFERENCES chunk(id), - mboxname CHAR UNIQUE NOT NULL, - unsubscribed INTEGER - ); - CREATE INDEX IF NOT EXISTS idx_sub_mbx ON subscription(mboxname); - - CREATE TABLE seen( - id INTEGER PRIMARY KEY ASC, - last_chunk_id INTEGER NOT NULL REFERENCES chunk(id), - uniqueid CHAR UNIQUE NOT NULL, - lastread INTEGER, - lastuid INTEGER, - lastchange INTEGER, - seenuids CHAR - ); - CREATE INDEX IF NOT EXISTS idx_seen_unq ON seen(uniqueid); - - CREATE TABLE sieve( - id INTEGER PRIMARY KEY ASC, - chunk_id INTEGER NOT NULL REFERENCES chunk(id), - last_update INTEGER, - filename CHAR NOT NULL, - guid CHAR NOT NULL, - offset INTEGER, - deleted INTEGER - ); - CREATE INDEX IF NOT EXISTS idx_siv_fn ON sieve(filename); -); - -const char backup_index_upgrade_v2[] = QUOTE( - CREATE TABLE subscription( - id INTEGER PRIMARY KEY ASC, - last_chunk_id INTEGER NOT NULL REFERENCES chunk(id), - mboxname CHAR UNIQUE NOT NULL, - unsubscribed INTEGER - ); - CREATE INDEX IF NOT EXISTS idx_sub_mbx ON subscription(mboxname); -); - -const char backup_index_upgrade_v3[] = QUOTE( - CREATE TABLE seen( - id INTEGER PRIMARY KEY ASC, - last_chunk_id INTEGER NOT NULL REFERENCES chunk(id), - uniqueid CHAR UNIQUE NOT NULL, - lastread INTEGER, - lastuid INTEGER, - lastchange INTEGER, - seenuids CHAR - ); - CREATE INDEX IF NOT EXISTS idx_seen_unq ON seen(uniqueid); -); - -const char backup_index_upgrade_v4[] = QUOTE( - CREATE TABLE sieve( - id INTEGER PRIMARY KEY ASC, - chunk_id INTEGER NOT NULL REFERENCES chunk(id), - last_update INTEGER, - filename CHAR NOT NULL, - guid CHAR NOT NULL, - offset INTEGER, - deleted INTEGER - ); - CREATE INDEX IF NOT EXISTS idx_siv_fn ON sieve(filename); -); - -const struct sqldb_upgrade backup_index_upgrade[] = { - { 2, backup_index_upgrade_v2, NULL }, - { 3, backup_index_upgrade_v3, NULL }, - { 4, backup_index_upgrade_v4, NULL }, - { 0, NULL, NULL } /* leave me last */ -}; - -const char backup_index_start_sql[] = QUOTE( - INSERT INTO chunk ( ts_start, offset, file_sha1 ) - VALUES ( :ts_start, :offset, :file_sha1 ); -); - -const char backup_index_end_sql[] = QUOTE( - UPDATE chunk SET - ts_end = :ts_end, - length = :length, - data_sha1 = :data_sha1 - WHERE id = :id; -); - -#define CHUNK_SELECT_FIELDS QUOTE( \ - id, ts_start, ts_end, offset, length, file_sha1, data_sha1 \ -) - -const char backup_index_chunk_select_all_sql[] = - "SELECT " CHUNK_SELECT_FIELDS - " FROM chunk" - ";" -; - -const char backup_index_chunk_select_live_sql[] = - "SELECT " CHUNK_SELECT_FIELDS - " FROM chunk" - " WHERE id IN (" - " SELECT last_chunk_id" - " FROM mailbox" - " WHERE deleted IS NULL OR deleted > :since" - " UNION" - " SELECT last_chunk_id" - " FROM mailbox_message" - " WHERE expunged IS NULL OR expunged > :since" - " UNION" - " SELECT chunk_id" - " FROM message AS m" - " JOIN mailbox_message AS mm" - " ON m.id = mm.message_id" - " AND (mm.expunged IS NULL OR mm.expunged > :since)" - " UNION" - " SELECT last_chunk_id" - " FROM subscription" - " WHERE unsubscribed IS NULL OR unsubscribed > :since" - " UNION" - " SELECT last_chunk_id" - " FROM seen" - " UNION" - " SELECT chunk_id" - " FROM sieve" - " WHERE deleted IS NULL or deleted > :since" - " )" - ";" -; - -const char backup_index_chunk_select_latest_sql[] = - "SELECT " CHUNK_SELECT_FIELDS - " FROM chunk" - " WHERE id = (SELECT MAX(id) FROM chunk)" - ";" -; - -const char backup_index_chunk_select_id_sql[] = - "SELECT " CHUNK_SELECT_FIELDS - " FROM chunk" - " WHERE id = :id" - ";" -; - -const char backup_index_mailbox_update_sql[] = QUOTE( - UPDATE mailbox SET - last_chunk_id = :last_chunk_id, - mboxname = :mboxname, - mboxtype = :mboxtype, - last_uid = :last_uid, - highestmodseq = :highestmodseq, - recentuid = :recentuid, - recenttime = :recenttime, - last_appenddate = :last_appenddate, - pop3_last_login = :pop3_last_login, - pop3_show_after = :pop3_show_after, - uidvalidity = :uidvalidity, - partition = :partition, - acl = :acl, - options = :options, - sync_crc = :sync_crc, - sync_crc_annot = :sync_crc_annot, - quotaroot = :quotaroot, - xconvmodseq = :xconvmodseq, - annotations = :annotations, - deleted = :deleted - WHERE uniqueid = :uniqueid; -); - -/* FIXME use uniqueid when sync proto contains it */ -const char backup_index_mailbox_rename_sql[] = QUOTE( - UPDATE mailbox SET - mboxname = :newmboxname, - partition = :partition, - uidvalidity = :uidvalidity - WHERE mboxname = :oldmboxname; -); - -const char backup_index_mailbox_delete_sql[] = QUOTE( - UPDATE mailbox SET - deleted = :deleted - WHERE mboxname = :mboxname; -); - -const char backup_index_mailbox_insert_sql[] = QUOTE( - INSERT INTO mailbox ( - last_chunk_id, uniqueid, mboxname, mboxtype, last_uid, - highestmodseq, recentuid, recenttime, last_appenddate, - pop3_last_login, pop3_show_after, uidvalidity, partition, - acl, options, sync_crc, sync_crc_annot, quotaroot, - xconvmodseq, annotations, deleted - ) - VALUES ( - :last_chunk_id, :uniqueid, :mboxname, :mboxtype, :last_uid, - :highestmodseq, :recentuid, :recenttime, :last_appenddate, - :pop3_last_login, :pop3_show_after, :uidvalidity, :partition, - :acl, :options, :sync_crc, :sync_crc_annot, :quotaroot, - :xconvmodseq, :annotations, :deleted - ); -); - -#define MAILBOX_SELECT_FIELDS QUOTE( \ - m.id, m.last_chunk_id, uniqueid, mboxname, mboxtype, last_uid, \ - highestmodseq, recentuid, recenttime, last_appenddate, \ - pop3_last_login, pop3_show_after, uidvalidity, m.partition, acl,\ - options, sync_crc, sync_crc_annot, quotaroot, xconvmodseq, \ - m.annotations, deleted \ -) - -const char backup_index_mailbox_select_all_sql[] = - "SELECT " MAILBOX_SELECT_FIELDS - " FROM mailbox AS m" - ";" -; - -const char backup_index_mailbox_select_mboxname_sql[] = - "SELECT " MAILBOX_SELECT_FIELDS - " FROM mailbox AS m" - " WHERE mboxname = :mboxname" - ";" -; - -const char backup_index_mailbox_select_uniqueid_sql[] = - "SELECT " MAILBOX_SELECT_FIELDS - " FROM mailbox AS m" - " WHERE uniqueid = :uniqueid" - ";" -; - -const char backup_index_mailbox_select_chunkid_sql[] = - "SELECT " MAILBOX_SELECT_FIELDS - " FROM mailbox AS m" - " WHERE last_chunk_id = :last_chunk_id" - ";" -; - -const char backup_index_mailbox_select_message_guid_sql[] = - "SELECT " MAILBOX_SELECT_FIELDS - " FROM mailbox AS m" - " JOIN mailbox_message AS mm" - " ON mm.mailbox_id = m.id" - " JOIN message AS msg" - " ON mm.message_id = msg.id" - " WHERE msg.guid = :guid" - ";" -; - -const char backup_index_mailbox_message_update_sql[] = QUOTE( - UPDATE mailbox_message SET - last_chunk_id = :last_chunk_id, - uid = :uid, - modseq = :modseq, - last_updated = :last_updated, - flags = :flags, - internaldate = :internaldate, - annotations = :annotations, - expunged = COALESCE( - (SELECT expunged FROM mailbox_message - WHERE mailbox_id = :mailbox_id - AND message_id = :message_id), - :expunged - ) - WHERE mailbox_id = :mailbox_id AND message_id = :message_id; -); - -const char backup_index_mailbox_message_insert_sql[] = QUOTE( - INSERT INTO mailbox_message ( - mailbox_id, message_id, last_chunk_id, uid, - modseq, last_updated, flags, internaldate, - annotations, expunged - ) - VALUES ( - :mailbox_id, :message_id, :last_chunk_id, :uid, - :modseq, :last_updated, :flags, :internaldate, - :annotations, :expunged - ); -); - -#define MAILBOX_MESSAGE_SELECT_FIELDS QUOTE( \ - r.id as id, mailbox_id, mb.uniqueid as mailbox_uniqueid, message_id, \ - r.last_chunk_id, uid, modseq, last_updated, flags, internaldate, \ - m.guid as guid, m.size as size, r.annotations, \ - expunged \ -) - -#define MAILBOX_MESSAGE_SELECT_JOIN QUOTE( \ - mailbox_message as r \ - JOIN message as m \ - ON r.message_id = m.id \ - JOIN mailbox as mb \ - on r.mailbox_id = mb.id \ -) - -const char backup_index_mailbox_message_select_mailbox_sql[] = - "SELECT " MAILBOX_MESSAGE_SELECT_FIELDS - " FROM " MAILBOX_MESSAGE_SELECT_JOIN - " WHERE mailbox_id = :mailbox_id" - ";" -; - -const char backup_index_mailbox_message_select_chunkid_sql[] = - "SELECT " MAILBOX_MESSAGE_SELECT_FIELDS - " FROM " MAILBOX_MESSAGE_SELECT_JOIN - " WHERE r.last_chunk_id = :last_chunk_id" - ";" -; - -const char backup_index_mailbox_message_select_all_sql[] = - "SELECT " MAILBOX_MESSAGE_SELECT_FIELDS - " FROM " MAILBOX_MESSAGE_SELECT_JOIN - ";" -; - -const char backup_index_mailbox_message_select_one_sql[] = - "SELECT " MAILBOX_MESSAGE_SELECT_FIELDS - " FROM " MAILBOX_MESSAGE_SELECT_JOIN - " WHERE mb.uniqueid = :uniqueid" - " AND m.guid = :guid" - ";" -; - -const char backup_index_mailbox_message_expunge_sql[] = QUOTE( - UPDATE mailbox_message - SET expunged = :expunged - WHERE mailbox_id = :mailbox_id AND uid = :uid; -); - -const char backup_index_message_update_sql[] = QUOTE( - UPDATE message SET - guid = :guid, - partition = :partition, - chunk_id = :chunk_id, - offset = :offset, - size = :size - WHERE guid = :guid; -); - -const char backup_index_message_insert_sql[] = QUOTE( - INSERT INTO message ( - guid, partition, chunk_id, offset, size - ) - VALUES ( - :guid, :partition, :chunk_id, :offset, :size - ); -); - -#define MESSAGE_SELECT_FIELDS QUOTE( \ - m.id, guid, partition, chunk_id, offset, size \ -) - -const char backup_index_message_select_all_sql[] = - "SELECT " MESSAGE_SELECT_FIELDS - " FROM message AS m" - " ORDER BY id" - ";" -; - -const char backup_index_message_select_guid_sql[] = - "SELECT " MESSAGE_SELECT_FIELDS - " FROM message AS m" - " WHERE guid = :guid" - ";" -; - -const char backup_index_message_select_chunkid_sql[] = - "SELECT " MESSAGE_SELECT_FIELDS - " FROM message AS m" - " WHERE chunk_id = :chunk_id" - ";" -; - -const char backup_index_message_select_live_chunkid_sql[] = - "SELECT DISTINCT " MESSAGE_SELECT_FIELDS - " FROM message AS m" - " JOIN mailbox_message AS mm" - " ON m.id = mm.message_id" - " AND (mm.expunged IS NULL OR mm.expunged > :since)" - " WHERE chunk_id = :chunk_id" - ";" -; - -const char backup_index_seen_update_sql[] = QUOTE( - UPDATE seen SET - last_chunk_id = :last_chunk_id, - lastread = :lastread, - lastuid = :lastuid, - lastchange = :lastchange, - seenuids = :seenuids - WHERE uniqueid = :uniqueid; -); - -const char backup_index_seen_insert_sql[] = QUOTE( - INSERT INTO seen ( - last_chunk_id, uniqueid, lastread, lastuid, lastchange, seenuids - ) - VALUES ( - :last_chunk_id, :uniqueid, :lastread, :lastuid, :lastchange, :seenuids - ); -); - -#define SEEN_SELECT_FIELDS QUOTE( \ - seen.id, seen.last_chunk_id, uniqueid, lastread, \ - lastuid, lastchange, seenuids \ -) - -const char backup_index_seen_select_all_sql[] = - "SELECT " SEEN_SELECT_FIELDS - " FROM seen AS seen" - " ORDER BY seen.id" - ";" -; - -const char backup_index_seen_select_chunkid_sql[] = - "SELECT " SEEN_SELECT_FIELDS - " FROM seen AS seen" - " WHERE last_chunk_id = :last_chunk_id" - " ORDER BY seen.id" - ";" -; - -const char backup_index_subscription_update_sql[] = QUOTE( - UPDATE subscription SET - last_chunk_id = :last_chunk_id, - unsubscribed = :unsubscribed - WHERE mboxname = :mboxname; -); - -const char backup_index_subscription_insert_sql[] = QUOTE( - INSERT INTO subscription ( - last_chunk_id, mboxname, unsubscribed - ) - VALUES ( - :last_chunk_id, :mboxname, :unsubscribed - ); -); - -#define SUBSCRIPTION_SELECT_FIELDS QUOTE( \ - sub.id, sub.last_chunk_id, mboxname, unsubscribed \ -) - -const char backup_index_subscription_select_all_sql[] = - "SELECT " SUBSCRIPTION_SELECT_FIELDS - " FROM subscription AS sub" - " ORDER BY sub.id" - ";" -; - -const char backup_index_subscription_select_chunkid_sql[] = - "SELECT " SUBSCRIPTION_SELECT_FIELDS - " FROM subscription AS sub" - " WHERE last_chunk_id = :last_chunk_id" - " ORDER BY sub.id" - ";" -; - -const char backup_index_sieve_insert_sql[] = QUOTE( - INSERT INTO sieve ( - chunk_id, last_update, filename, guid, offset - ) - VALUES ( - :chunk_id, :last_update, :filename, :guid, :offset - ); -); - -const char backup_index_sieve_delete_sql[] = QUOTE( - UPDATE sieve SET - deleted = :deleted - WHERE filename = :filename - AND deleted IS NULL; -); - -#define SIEVE_SELECT_FIELDS QUOTE( \ - sieve.id, sieve.chunk_id, last_update, filename, \ - guid, offset, deleted \ -) - -const char backup_index_sieve_select_all_sql[] = - "SELECT " SIEVE_SELECT_FIELDS - " FROM sieve AS sieve" - " ORDER BY sieve.id" - ";" -; - -const char backup_index_sieve_select_chunkid_sql[] = - "SELECT " SIEVE_SELECT_FIELDS - " FROM sieve AS sieve" - " WHERE chunk_id = :chunk_id" - " ORDER BY sieve.id" - ";" -; diff --git a/backup/lcb_sqlconsts.h b/backup/lcb_sqlconsts.h deleted file mode 100644 index dac02d6be7e..00000000000 --- a/backup/lcb_sqlconsts.h +++ /dev/null @@ -1,103 +0,0 @@ -/* lcb_sqlconsts.h -- backup index sql constants - * - * Copyright (c) 1994-2015 Carnegie Mellon University. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * 3. The name "Carnegie Mellon University" must not be used to - * endorse or promote products derived from this software without - * prior written permission. For permission or any legal - * details, please contact - * Carnegie Mellon University - * Center for Technology Transfer and Enterprise Creation - * 4615 Forbes Avenue - * Suite 302 - * Pittsburgh, PA 15213 - * (412) 268-7393, fax: (412) 268-7395 - * innovation@andrew.cmu.edu - * - * 4. Redistributions of any form whatsoever must retain the following - * acknowledgment: - * "This product includes software developed by Computing Services - * at Carnegie Mellon University (http://www.cmu.edu/computing/)." - * - * CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO - * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY - * AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE - * FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN - * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING - * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - * - */ - -#ifndef BACKUP_LCB_SQLCONSTS_H -#define BACKUP_LCB_SQLCONSTS_H - -#include "lib/sqldb.h" - -extern const char backup_index_initsql[]; - -extern const struct sqldb_upgrade backup_index_upgrade[]; - -extern const int backup_index_version; - -extern const char backup_index_start_sql[]; -extern const char backup_index_end_sql[]; - -extern const char backup_index_chunk_select_all_sql[]; -extern const char backup_index_chunk_select_live_sql[]; -extern const char backup_index_chunk_select_latest_sql[]; -extern const char backup_index_chunk_select_id_sql[]; - -extern const char backup_index_mailbox_update_sql[]; -extern const char backup_index_mailbox_rename_sql[]; -extern const char backup_index_mailbox_delete_sql[]; -extern const char backup_index_mailbox_insert_sql[]; -extern const char backup_index_mailbox_select_all_sql[]; -extern const char backup_index_mailbox_select_mboxname_sql[]; -extern const char backup_index_mailbox_select_uniqueid_sql[]; -extern const char backup_index_mailbox_select_chunkid_sql[]; -extern const char backup_index_mailbox_select_message_guid_sql[]; - -extern const char backup_index_mailbox_message_update_sql[]; -extern const char backup_index_mailbox_message_insert_sql[]; -extern const char backup_index_mailbox_message_select_mailbox_sql[]; -extern const char backup_index_mailbox_message_select_chunkid_sql[]; -extern const char backup_index_mailbox_message_select_all_sql[]; -extern const char backup_index_mailbox_message_select_one_sql[]; -extern const char backup_index_mailbox_message_expunge_sql[]; - -extern const char backup_index_message_update_sql[]; -extern const char backup_index_message_insert_sql[]; -extern const char backup_index_message_select_all_sql[]; -extern const char backup_index_message_select_guid_sql[]; -extern const char backup_index_message_select_chunkid_sql[]; -extern const char backup_index_message_select_live_chunkid_sql[]; - -extern const char backup_index_seen_update_sql[]; -extern const char backup_index_seen_insert_sql[]; -extern const char backup_index_seen_select_all_sql[]; -extern const char backup_index_seen_select_chunkid_sql[]; - -extern const char backup_index_subscription_update_sql[]; -extern const char backup_index_subscription_insert_sql[]; -extern const char backup_index_subscription_select_all_sql[]; -extern const char backup_index_subscription_select_chunkid_sql[]; - -extern const char backup_index_sieve_insert_sql[]; -extern const char backup_index_sieve_delete_sql[]; -extern const char backup_index_sieve_select_all_sql[]; -extern const char backup_index_sieve_select_chunkid_sql[]; - -#endif diff --git a/backup/lcb_verify.c b/backup/lcb_verify.c deleted file mode 100644 index a8dac204c9b..00000000000 --- a/backup/lcb_verify.c +++ /dev/null @@ -1,652 +0,0 @@ -/* lcb_verify.c -- replication-based backup api - verify functions - * - * Copyright (c) 1994-2015 Carnegie Mellon University. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * 3. The name "Carnegie Mellon University" must not be used to - * endorse or promote products derived from this software without - * prior written permission. For permission or any legal - * details, please contact - * Carnegie Mellon University - * Center for Technology Transfer and Enterprise Creation - * 4615 Forbes Avenue - * Suite 302 - * Pittsburgh, PA 15213 - * (412) 268-7393, fax: (412) 268-7395 - * innovation@andrew.cmu.edu - * - * 4. Redistributions of any form whatsoever must retain the following - * acknowledgment: - * "This product includes software developed by Computing Services - * at Carnegie Mellon University (http://www.cmu.edu/computing/)." - * - * CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO - * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY - * AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE - * FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN - * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING - * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - * - */ -#include -#include - -#include "lib/gzuncat.h" -#include "lib/hash.h" -#include "lib/map.h" -#include "lib/xmalloc.h" -#include "lib/xsha1.h" - -#include "backup/backup.h" - -#define LIBCYRUS_BACKUP_SOURCE /* this file is part of libcyrus_backup */ -#include "backup/lcb_internal.h" -#include "backup/lcb_sqlconsts.h" - -static int verify_chunk_checksums(struct backup *backup, struct backup_chunk *chunk, - struct gzuncat *gzuc, int verbose, - FILE *out); -static int verify_chunk_messages(struct backup *backup, struct backup_chunk *chunk, - struct gzuncat *gzuc, unsigned level, - int verbose, FILE *out); -static int verify_chunk_mailbox_links(struct backup *backup, struct backup_chunk *chunk, - struct gzuncat *gzuc, int verbose, - FILE *out); - -EXPORTED int backup_verify(struct backup *backup, unsigned level, int verbose, FILE *out) -{ - struct backup_chunk_list *chunk_list = NULL; - struct gzuncat *gzuc = NULL; - int r = 0; - - /* don't double-verify last checksum when verifying all */ - if ((level & BACKUP_VERIFY_ALL_CHECKSUMS)) - level &= ~BACKUP_VERIFY_LAST_CHECKSUM; - - /* don't double-verify message links when verifying message guids */ - if ((level & BACKUP_VERIFY_MESSAGE_GUIDS)) - level &= ~BACKUP_VERIFY_MESSAGE_LINKS; - - chunk_list = backup_get_chunks(backup); - if (!chunk_list || !chunk_list->count) goto done; - - gzuc = gzuc_new(backup->fd); - if (!gzuc) { - r = -1; - goto done; - } - - if (!r && (level & BACKUP_VERIFY_LAST_CHECKSUM)) - r = verify_chunk_checksums(backup, chunk_list->tail, gzuc, verbose, out); - - if (!r && level > BACKUP_VERIFY_LAST_CHECKSUM) { - struct backup_chunk *chunk = chunk_list->head; - while (!r && chunk) { - if (!r && (level & BACKUP_VERIFY_ALL_CHECKSUMS)) - r = verify_chunk_checksums(backup, chunk, gzuc, verbose, out); - - if (!r && (level & BACKUP_VERIFY_MESSAGES)) - r = verify_chunk_messages(backup, chunk, gzuc, level, verbose, out); - - if (!r && (level & BACKUP_VERIFY_MAILBOX_LINKS)) - r = verify_chunk_mailbox_links(backup, chunk, gzuc, verbose, out); - - chunk = chunk->next; - } - } - -done: - if (gzuc) gzuc_free(&gzuc); - if (chunk_list) backup_chunk_list_free(&chunk_list); - return r; -} - -static int verify_chunk_checksums(struct backup *backup, struct backup_chunk *chunk, - struct gzuncat *gzuc, int verbose, FILE *out) -{ - int r; - - if (out && verbose) - fprintf(out, "checking chunk %d checksums...\n", chunk->id); - - /* validate file-prior-to-this-chunk checksum */ - if (out && verbose > 1) - fprintf(out, " checking file checksum...\n"); - char file_sha1[2 * SHA1_DIGEST_LENGTH + 1]; - sha1_file(backup->fd, backup->data_fname, chunk->offset, file_sha1); - r = strncmp(chunk->file_sha1, file_sha1, sizeof(file_sha1)); - if (r) { - syslog(LOG_DEBUG, "%s: %s (chunk %d) file checksum mismatch: %s on disk, %s in index", - __func__, backup->data_fname, chunk->id, file_sha1, chunk->file_sha1); - if (out) - fprintf(out, "file checksum mismatch for chunk %d: %s on disk, %s in index\n", - chunk->id, file_sha1, chunk->file_sha1); - goto done; - } - - /* validate data-within-this-chunk checksum */ - // FIXME length and data_sha1 are set at backup_append_end. - // detect and correctly report case where this hasn't occurred. - if (out && verbose > 1) - fprintf(out, " checking data length\n"); - char buf[8192]; /* FIXME whatever */ - size_t len = 0; - SHA1_CTX sha_ctx; - SHA1Init(&sha_ctx); - gzuc_member_start_from(gzuc, chunk->offset); - while (!gzuc_member_eof(gzuc)) { - ssize_t n = gzuc_read(gzuc, buf, sizeof(buf)); - if (n >= 0) { - SHA1Update(&sha_ctx, buf, n); - len += n; - } - } - gzuc_member_end(gzuc, NULL); - if (len != chunk->length) { - syslog(LOG_DEBUG, "%s: %s (chunk %d) data length mismatch: " - SIZE_T_FMT " on disk," - SIZE_T_FMT " in index", - __func__, backup->data_fname, chunk->id, len, chunk->length); - if (out) - fprintf(out, "data length mismatch for chunk %d: " - SIZE_T_FMT " on disk," - SIZE_T_FMT " in index\n", - chunk->id, len, chunk->length); - r = -1; - goto done; - } - - if (out && verbose > 1) - fprintf(out, " checking data checksum...\n"); - unsigned char sha1_raw[SHA1_DIGEST_LENGTH]; - char data_sha1[2 * SHA1_DIGEST_LENGTH + 1]; - SHA1Final(sha1_raw, &sha_ctx); - r = bin_to_hex(sha1_raw, SHA1_DIGEST_LENGTH, data_sha1, BH_LOWER); - assert(r == 2 * SHA1_DIGEST_LENGTH); - r = strncmp(chunk->data_sha1, data_sha1, sizeof(data_sha1)); - if (r) { - syslog(LOG_DEBUG, "%s: %s (chunk %d) data checksum mismatch: %s on disk, %s in index", - __func__, backup->data_fname, chunk->id, data_sha1, chunk->data_sha1); - if (out) - fprintf(out, "data checksum mismatch for chunk %d: %s on disk, %s in index\n", - chunk->id, data_sha1, chunk->data_sha1); - goto done; - } - -done: - syslog(LOG_DEBUG, "%s: checksum %s!", __func__, r ? "failed" : "passed"); - if (out && verbose) - fprintf(out, "%s\n", r ? "error" : "ok"); - return r; -} - -static ssize_t _prot_fill_cb(unsigned char *buf, size_t len, void *rock) -{ - struct gzuncat *gzuc = (struct gzuncat *) rock; - return gzuc_read(gzuc, buf, len); -} - -struct verify_message_rock { - struct gzuncat *gzuc; - int verify_guid; - struct dlist *cached_dlist; - off_t cached_offset; - int verbose; - FILE *out; -}; - -static int _verify_message_cb(const struct backup_message *message, void *rock) -{ - struct verify_message_rock *vmrock = (struct verify_message_rock *) rock; - struct dlist *dl = NULL; - struct dlist *di = NULL; - FILE *out = vmrock->out; - int r; - - /* cache the dlist so that multiple reads from the same offset don't - * cause expensive reverse seeks in decompression stream - */ - if (!vmrock->cached_dlist || vmrock->cached_offset != message->offset) { - if (vmrock->cached_dlist) { - dlist_unlink_files(vmrock->cached_dlist); - dlist_free(&vmrock->cached_dlist); - } - - r = gzuc_seekto(vmrock->gzuc, message->offset); - if (r) return r; - - struct protstream *ps = prot_readcb(_prot_fill_cb, vmrock->gzuc); - prot_setisclient(ps, 1); /* don't sync literals */ - r = parse_backup_line(ps, NULL, NULL, &dl); - - if (r == EOF) { - const char *error = prot_error(ps); - if (error && 0 != strcmp(error, PROT_EOF_STRING)) { - syslog(LOG_ERR, - "%s: error reading message %i at offset " OFF_T_FMT ", byte %i: %s", - __func__, message->id, message->offset, prot_bytes_in(ps), error); - if (out) - fprintf(out, "error reading message %i at offset " OFF_T_FMT ", byte %i: %s", - message->id, message->offset, prot_bytes_in(ps), error); - } - prot_free(ps); - return r; - } - - prot_free(ps); - - vmrock->cached_dlist = dl; - vmrock->cached_offset = message->offset; - } - else { - dl = vmrock->cached_dlist; - } - - r = strcmp(dl->name, "MESSAGE"); - if (r) return r; - - r = -1; - for (di = dl->head; di; di = di->next) { - struct message_guid *guid = NULL; - const char *fname = NULL; - - if (!dlist_tofile(di, NULL, &guid, NULL, &fname)) - continue; - - r = message_guid_cmp(guid, message->guid); - if (!r) { - if (vmrock->verify_guid) { - const char *msg_base = NULL; - size_t msg_len = 0; - struct message_guid computed_guid; - int fd; - - fd = open(fname, O_RDWR); - if (fd != -1) { - map_refresh(fd, 1, &msg_base, &msg_len, MAP_UNKNOWN_LEN, fname, NULL); - - message_guid_generate(&computed_guid, msg_base, msg_len); - r = message_guid_cmp(&computed_guid, message->guid); - if (r && out) - fprintf(out, "guid mismatch for message %i\n", message->id); - - map_free(&msg_base, &msg_len); - close(fd); - } - else { - xsyslog(LOG_ERR, "IOERROR: open failed", - "filename=<%s>", fname); - if (out) - fprintf(out, "error reading staging file for message %i\n", message->id); - r = -1; - } - } - break; - } - } - - return r; -} - -/* verify that each message exists within the chunk the index claims */ -static int verify_chunk_messages(struct backup *backup, struct backup_chunk *chunk, - struct gzuncat *gzuc, unsigned level, - int verbose, FILE *out) -{ - int r; - - struct verify_message_rock vmrock = { - gzuc, - (level & BACKUP_VERIFY_MESSAGE_GUIDS), - NULL, - 0, - verbose, - out, - }; - - if (out && verbose) - fprintf(out, "checking chunk %d messages...\n", chunk->id); - - r = gzuc_member_start_from(gzuc, chunk->offset); - if (!r) { - r = backup_message_foreach(backup, chunk->id, NULL, - _verify_message_cb, &vmrock); - gzuc_member_end(gzuc, NULL); - } - - if (vmrock.cached_dlist) { - dlist_unlink_files(vmrock.cached_dlist); - dlist_free(&vmrock.cached_dlist); - } - - syslog(LOG_DEBUG, "%s: chunk %d %s!", __func__, chunk->id, - r ? "failed" : "passed"); - if (out && verbose) - fprintf(out, "%s\n", r ? "error" : "ok"); - - return r; -} - -static int mailbox_matches(const struct backup_mailbox *mailbox, - struct dlist *dlist) -{ - const char *mboxname = NULL; - uint32_t last_uid = 0; - modseq_t highestmodseq = 0; - uint32_t recentuid = 0; - time_t recenttime = 0; - time_t last_appenddate = 0; - uint32_t uidvalidity = 0; - const char *partition = NULL; - const char *acl = NULL; - const char *options = NULL; - modseq_t xconvmodseq = 0; - struct synccrcs synccrcs = { 0, 0 }; - - if (!dlist_getatom(dlist, "MBOXNAME", &mboxname) - || strcmp(mboxname, mailbox->mboxname) != 0) - return 0; - - if (!dlist_getnum32(dlist, "LAST_UID", &last_uid) - || last_uid != mailbox->last_uid) - return 0; - - if (!dlist_getnum64(dlist, "HIGHESTMODSEQ", &highestmodseq) - || highestmodseq != mailbox->highestmodseq) - return 0; - - if (!dlist_getnum32(dlist, "RECENTUID", &recentuid) - || recentuid != mailbox->recentuid) - return 0; - - if (!dlist_getdate(dlist, "RECENTTIME", &recenttime) - || recenttime != mailbox->recenttime) - return 0; - - if (!dlist_getdate(dlist, "LAST_APPENDDATE", &last_appenddate) - || last_appenddate != mailbox->last_appenddate) - return 0; - - if (!dlist_getnum32(dlist, "UIDVALIDITY", &uidvalidity) - || uidvalidity != mailbox->uidvalidity) - return 0; - - if (!dlist_getatom(dlist, "PARTITION", &partition) - || strcmp(partition, mailbox->partition) != 0) - return 0; - - if (!dlist_getatom(dlist, "ACL", &acl) - || strcmp(acl, mailbox->acl) != 0) - return 0; - - if (!dlist_getatom(dlist, "OPTIONS", &options) - || strcmp(options, mailbox->options) != 0) - return 0; - - /* optional */ - dlist_getnum64(dlist, "XCONVMODSEQ", &xconvmodseq); - if (xconvmodseq != mailbox->xconvmodseq) - return 0; - - /* CRCs */ - dlist_getnum32(dlist, "SYNC_CRC", &synccrcs.basic); - dlist_getnum32(dlist, "SYNC_CRC_ANNOT", &synccrcs.annot); - if (synccrcs.basic != mailbox->sync_crc) - return 0; - if (synccrcs.annot != mailbox->sync_crc_annot) - return 0; - - syslog(LOG_DEBUG, "%s: %s matches!", __func__, mailbox->uniqueid); - return 1; -} - -static int mailbox_message_matches(const struct backup_mailbox_message *mailbox_message, - struct dlist *dlist) -{ - modseq_t modseq; - uint32_t last_updated; - uint32_t internaldate; - uint32_t size; - struct message_guid *guid; - - if (!dlist_getnum64(dlist, "MODSEQ", &modseq) - || modseq != mailbox_message->modseq) - return 0; - - if (!dlist_getnum32(dlist, "LAST_UPDATED", &last_updated) - || (time_t) last_updated != mailbox_message->last_updated) - return 0; - - if (!dlist_getnum32(dlist, "INTERNALDATE", &internaldate) - || (time_t) internaldate != mailbox_message->internaldate) - return 0; - - if (!dlist_getnum32(dlist, "SIZE", &size) - || size != mailbox_message->size) - return 0; - - if (!dlist_getguid(dlist, "GUID", &guid) - || !message_guid_equal(guid, &mailbox_message->guid)) - return 0; - - syslog(LOG_DEBUG, "%s: %s:%u matches!", __func__, - mailbox_message->mailbox_uniqueid, mailbox_message->uid); - return 1; -} - -/* verify that the matching MAILBOX exists within the claimed chunk - * for each mailbox or mailbox_message in the index - */ -static int verify_chunk_mailbox_links(struct backup *backup, struct backup_chunk *chunk, - struct gzuncat *gzuc, int verbose, FILE *out) -{ - /* - * get list of mailboxes in chunk - * get list of mailbox_messages in chunk - * index mailboxes list by uniqueid - * index mailbox_messages list by uniqueid:uid - * open chunk - * foreach line in chunk - * read dlist - * skip if it's not a mailbox - * if details in dlist match details in mailbox - * remove from mailbox list/index - * foreach record in dlist - * if details in dlist match details in mailbox_message - * remove from mailbox_message list/index - * failed if either list of mailboxes or list of mailbox_messages is not empty - */ - - struct backup_mailbox_list *mailbox_list = NULL; - struct backup_mailbox_message_list *mailbox_message_list = NULL; - hash_table mailbox_list_index = HASH_TABLE_INITIALIZER; - hash_table mailbox_message_list_index = HASH_TABLE_INITIALIZER; - struct backup_mailbox *mailbox = NULL; - struct backup_mailbox_message *mailbox_message = NULL; - int r; - - if (out && verbose) - fprintf(out, "checking chunk %d mailbox links...\n", chunk->id); - - mailbox_list = backup_get_mailboxes(backup, chunk->id, BACKUP_MAILBOX_NO_RECORDS); - mailbox_message_list = backup_get_mailbox_messages(backup, chunk->id); - - if (mailbox_list->count == 0 && mailbox_message_list->count == 0) { - /* nothing we care about in this chunk */ - free(mailbox_list); - free(mailbox_message_list); - if (out && verbose) - fprintf(out, "ok\n"); - return 0; - } - - /* XXX consider whether the two hashes should use pools */ - - if (mailbox_list->count) { - /* build an index of the mailbox list */ - construct_hash_table(&mailbox_list_index, mailbox_list->count, 0); - mailbox = mailbox_list->head; - while (mailbox) { - hash_insert(mailbox->uniqueid, mailbox, &mailbox_list_index); - mailbox = mailbox->next; - } - } - - if (mailbox_message_list->count) { - /* build an index of the mailbox message list */ - construct_hash_table(&mailbox_message_list_index, - mailbox_message_list->count, 0); - mailbox_message = mailbox_message_list->head; - while (mailbox_message) { - char keybuf[1024]; // FIXME whatever - snprintf(keybuf, sizeof(keybuf), "%s:%d", - mailbox_message->mailbox_uniqueid, mailbox_message->uid); - hash_insert(keybuf, mailbox_message, &mailbox_message_list_index); - mailbox_message = mailbox_message->next; - } - } - - r = gzuc_member_start_from(gzuc, chunk->offset); - if (r) { - syslog(LOG_ERR, "%s: error reading chunk %i at offset " OFF_T_FMT ": %s", - __func__, chunk->id, chunk->offset, zError(r)); - if (out) - fprintf(out, "error reading chunk %i at offset " OFF_T_FMT ": %s", - chunk->id, chunk->offset, zError(r)); - goto done; - } - struct protstream *ps = prot_readcb(_prot_fill_cb, gzuc); - prot_setisclient(ps, 1); /* don't sync literals */ - - struct buf cmd = BUF_INITIALIZER; - while (1) { - struct dlist *dl = NULL; - struct dlist *record = NULL; - struct dlist *di = NULL; - const char *uniqueid = NULL; - - int c = parse_backup_line(ps, NULL, &cmd, &dl); - if (c == EOF) { - const char *error = prot_error(ps); - if (error && 0 != strcmp(error, PROT_EOF_STRING)) { - syslog(LOG_ERR, - "%s: error reading chunk %i data at offset " OFF_T_FMT ", byte %i: %s", - __func__, chunk->id, chunk->offset, prot_bytes_in(ps), error); - if (out) - fprintf(out, "error reading chunk %i data at offset " OFF_T_FMT ", byte %i: %s", - chunk->id, chunk->offset, prot_bytes_in(ps), error); - r = EOF; - } - break; - } - - if (strcmp(buf_cstring(&cmd), "APPLY") != 0) - goto next_line; - - if (strcmp(dl->name, "MAILBOX") != 0) - goto next_line; - - if (!dlist_getatom(dl, "UNIQUEID", &uniqueid)) - goto next_line; - - if (mailbox_list->count) { - mailbox = (struct backup_mailbox *) hash_lookup(uniqueid, &mailbox_list_index); - - if (mailbox && mailbox_matches(mailbox, dl)) { - backup_mailbox_list_remove(mailbox_list, mailbox); - hash_del(uniqueid, &mailbox_list_index); - backup_mailbox_free(&mailbox); - } - } - - if (mailbox_message_list->count) { - if (!dlist_getlist(dl, "RECORD", &record)) - goto next_line; - - for (di = record->head; di; di = di->next) { - char keybuf[1024]; // FIXME whatever - uint32_t uid; - - if (!dlist_getnum32(di, "UID", &uid)) - continue; - - snprintf(keybuf, sizeof(keybuf), "%s:%d", uniqueid, uid); - mailbox_message = (struct backup_mailbox_message *) hash_lookup( - keybuf, &mailbox_message_list_index); - - if (!mailbox_message) - continue; - - if (!mailbox_message_matches(mailbox_message, di)) - continue; - - backup_mailbox_message_list_remove(mailbox_message_list, mailbox_message); - hash_del(keybuf, &mailbox_message_list_index); - backup_mailbox_message_free(&mailbox_message); - } - } - -next_line: - if (dl) { - dlist_unlink_files(dl); - dlist_free(&dl); - } - } - buf_free(&cmd); - - prot_free(ps); - gzuc_member_end(gzuc, NULL); - - /* anything left in either of the lists is missing from the chunk data. bad! */ - mailbox = mailbox_list->head; - while (mailbox) { - syslog(LOG_DEBUG, "%s: chunk %d missing mailbox data for %s (%s)", - __func__, chunk->id, mailbox->uniqueid, mailbox->mboxname); - if (out) - fprintf(out, "chunk %d missing mailbox data for %s (%s)\n", - chunk->id, mailbox->uniqueid, mailbox->mboxname); - mailbox = mailbox->next; - } - - mailbox_message = mailbox_message_list->head; - while (mailbox_message) { - syslog(LOG_DEBUG, "%s: chunk %d missing mailbox_message data for %s uid %u", - __func__, chunk->id, mailbox_message->mailbox_uniqueid, - mailbox_message->uid); - if (out) - fprintf(out, "chunk %d missing mailbox_message data for %s uid %u\n", - chunk->id, mailbox_message->mailbox_uniqueid, - mailbox_message->uid); - mailbox_message = mailbox_message->next; - } - - if (!r) r = mailbox_list->count || mailbox_message_list->count ? -1 : 0; - -done: - free_hash_table(&mailbox_list_index, NULL); - free_hash_table(&mailbox_message_list_index, NULL); - - backup_mailbox_list_empty(mailbox_list); - free(mailbox_list); - - backup_mailbox_message_list_empty(mailbox_message_list); - free(mailbox_message_list); - - syslog(LOG_DEBUG, "%s: chunk %d %s!", __func__, chunk->id, - r ? "failed" : "passed"); - if (out && verbose) - fprintf(out, "%s\n", r ? "error" : "ok"); - return r; -} diff --git a/backup/restore.c b/backup/restore.c deleted file mode 100644 index 27fcff00f43..00000000000 --- a/backup/restore.c +++ /dev/null @@ -1,942 +0,0 @@ -/* restore.c -- tool for restoring from replication-based backups - * - * Copyright (c) 1994-2016 Carnegie Mellon University. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * 3. The name "Carnegie Mellon University" must not be used to - * endorse or promote products derived from this software without - * prior written permission. For permission or any legal - * details, please contact - * Carnegie Mellon University - * Center for Technology Transfer and Enterprise Creation - * 4615 Forbes Avenue - * Suite 302 - * Pittsburgh, PA 15213 - * (412) 268-7393, fax: (412) 268-7395 - * innovation@andrew.cmu.edu - * - * 4. Redistributions of any form whatsoever must retain the following - * acknowledgment: - * "This product includes software developed by Computing Services - * at Carnegie Mellon University (http://www.cmu.edu/computing/)." - * - * CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO - * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY - * AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE - * FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN - * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING - * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - * - */ -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -#include "imap/global.h" -#include "imap/imap_err.h" -#include "imap/imap_proxy.h" -#include "imap/mboxname.h" -#include "imap/message_guid.h" -#include "imap/sync_support.h" - -#include "backup/backup.h" - -static struct namespace restore_namespace; - -EXPORTED void fatal(const char *s, int code) -{ - fprintf(stderr, "Fatal error: %s\n", s); - syslog(LOG_ERR, "Fatal error: %s", s); - exit(code); -} - -static const char *argv0 = NULL; -static void usage(void) -{ - fprintf(stderr, "Usage:\n"); - fprintf(stderr, " %s [options] server [mode] backup [mboxname | uniqueid | guid]...\n", argv0); - - fprintf(stderr, "\n%s\n", - "Options:\n" - " -A [acl] # apply specified acl to restored mailboxes\n" - " -C alt_config # alternate config file\n" - " -D # don't trim deletedprefix before restoring\n" - " -F input-file # read mailboxes/messages from file rather than argv\n" - " -L # local mailbox operations only (no mupdate)\n" - " -M mboxname # restore messages to specified mailbox\n" - " -P partition # restore mailboxes to specified partition\n" - " -U # try to preserve uniqueid, uid, modseq, etc\n" - " -X # don't restore expunged messages\n" - " -a # try to restore all mailboxes in backup\n" - " -n # calculate work required but don't perform restoration\n" - " -r # recurse into submailboxes\n" - " -v # verbose (repeat for more verbosity)\n" - " -w seconds # wait before starting (useful for attaching a debugger)\n" - " -x # only restore expunged messages\n" - " -z # require compression (abort if compression unavailable)\n" - ); - - fprintf(stderr, "%s\n", - "Modes:\n" - " -f # specified backup interpreted as filename\n" - " -m # specified backup interpreted as mboxname\n" - " -u # specified backup interpreted as userid (default)\n" - ); - - exit(EX_USAGE); -} - -static void save_argv0(const char *s) -{ - const char *slash = strrchr(s, '/'); - if (slash) - argv0 = slash + 1; - else - argv0 = s; -} - -static struct sasl_callback mysasl_cb[] = { - { SASL_CB_GETOPT, (mysasl_cb_ft *) &mysasl_config, NULL }, - { SASL_CB_CANON_USER, (mysasl_cb_ft *) &mysasl_canon_user, NULL }, - { SASL_CB_LIST_END, NULL, NULL } -}; - -enum restore_mode { - RESTORE_MODE_UNSPECIFIED = 0, - RESTORE_MODE_FILENAME, - RESTORE_MODE_MBOXNAME, - RESTORE_MODE_USERNAME, -}; - -enum restore_expunged_mode { - RESTORE_EXPUNGED_OKAY = 0, - RESTORE_EXPUNGED_EXCLUDE, - RESTORE_EXPUNGED_ONLY, -}; - -struct restore_options { - const char *override_acl; - const char *override_mboxname; - const char *override_partition; - enum restore_expunged_mode expunged_mode; - int do_submailboxes; - int keep_uidvalidity; - int require_compression; - int trim_deletedprefix; - int verbose; -}; - -#define HEX_DIGITS "0123456789abcdefghijklmnopqrstuvwxyz" - -static int restore_add_object(const char *object_name, - const struct restore_options *options, - struct backup *backup, - struct backup_mailbox_list *mailbox_list, - struct sync_folder_list *reserve_folder_list, - struct sync_reserve_list *reserve_list); -static int restore_add_mailbox(const struct backup_mailbox *mailbox, - const struct restore_options *options, - struct backup_mailbox_list *mailbox_list, - struct sync_folder_list *reserve_folder_list, - struct sync_reserve_list *reserve_list); -static int restore_add_message(const struct backup_message *message, - const struct backup_mailbox_list *message_mailboxes, - const struct restore_options *options, - struct backup_mailbox_list *mailbox_list, - struct sync_folder_list *reserve_folder_list, - struct sync_reserve_list *reserve_list); - -static struct sync_folder_list *restore_make_reserve_folder_list( - struct backup *backup); - - -int main(int argc, char **argv) -{ - save_argv0(argv[0]); - - const char *alt_config = NULL; - const char *input_file = NULL; - const char *backup_name = NULL; - const char *servername = NULL; - enum restore_mode mode = RESTORE_MODE_UNSPECIFIED; - int local_only = 0; - int wait = 0; - int do_nothing = 0; - int do_all_mailboxes = 0; - - struct restore_options options = {0}; - options.expunged_mode = RESTORE_EXPUNGED_OKAY; - options.trim_deletedprefix = 1; - - struct backup *backup = NULL; - mbname_t *mbname = NULL; - struct backup_mailbox_list *mailbox_list = NULL; - struct sync_folder_list *reserve_folder_list = NULL; - struct sync_reserve_list *reserve_list = NULL; - struct sync_client_state sync_cs = SYNC_CLIENT_STATE_INITIALIZER; - struct dlist *upload = NULL; - int opt, r = 0; - - /* keep this in alphabetical order */ - static const char short_options[] = ":A:C:DF:LM:P:UXaf:m:nru:vw:xz"; - - static const struct option long_options[] = { - { "override-acl", optional_argument, NULL, 'A' }, - /* n.b. no long option for -C */ - { "keep-deletedprefix", no_argument, NULL, 'D' }, - { "input-file", required_argument, NULL, 'F' }, - { "local-only", no_argument, NULL, 'L' }, - { "dest-mailbox", required_argument, NULL, 'M' }, - { "dest-partition", required_argument, NULL, 'P' }, - { "keep-uidvalidity", no_argument, NULL, 'U' }, - { "skip-expunged", no_argument, NULL, 'X' }, - { "all-mailboxes", no_argument, NULL, 'a' }, - { "file", required_argument, NULL, 'f' }, - { "mailbox", required_argument, NULL, 'm' }, - { "dry-run", no_argument, NULL, 'n' }, - { "recursive", no_argument, NULL, 'r' }, - { "userid", required_argument, NULL, 'u' }, - { "verbose", no_argument, NULL, 'v' }, - { "delayed-startup", required_argument, NULL, 'w' }, - { "only-expunged", no_argument, NULL, 'x' }, - { "require-compression", no_argument, NULL, 'z' }, - - { 0, 0, 0, 0 }, - }; - - while (-1 != (opt = getopt_long(argc, argv, - short_options, long_options, NULL))) - { - switch (opt) { - case 'A': - if (options.keep_uidvalidity) usage(); - options.override_acl = optarg; - break; - case 'C': - alt_config = optarg; - break; - case 'D': - /* XXX does this clash with keep_uidvalidity? */ - options.trim_deletedprefix = 0; - break; - case 'F': - if (do_all_mailboxes) usage(); - input_file = optarg; - break; - case 'L': - local_only = 1; - break; - case 'M': - if (options.keep_uidvalidity) usage(); - options.override_mboxname = optarg; - break; - case 'P': - if (options.keep_uidvalidity) usage(); - options.override_partition = optarg; - break; - case 'U': - if (options.override_acl || options.override_mboxname || options.override_partition) - usage(); - options.keep_uidvalidity = 1; - break; - case 'X': - if (options.expunged_mode != RESTORE_EXPUNGED_OKAY) usage(); - options.expunged_mode = RESTORE_EXPUNGED_EXCLUDE; - break; - case 'a': - if (input_file) usage(); - do_all_mailboxes = 1; - break; - case 'f': - if (mode != RESTORE_MODE_UNSPECIFIED) usage(); - mode = RESTORE_MODE_FILENAME; - backup_name = optarg; - break; - case 'm': - if (mode != RESTORE_MODE_UNSPECIFIED) usage(); - mode = RESTORE_MODE_MBOXNAME; - backup_name = optarg; - break; - case 'n': - do_nothing = 1; - break; - case 'r': - options.do_submailboxes = 1; - break; - case 'u': - if (mode != RESTORE_MODE_UNSPECIFIED) usage(); - mode = RESTORE_MODE_USERNAME; - backup_name = optarg; - break; - case 'v': - options.verbose++; - break; - case 'w': - wait = atoi(optarg); - if (wait < 0) usage(); - break; - case 'x': - if (options.expunged_mode != RESTORE_EXPUNGED_OKAY) usage(); - options.expunged_mode = RESTORE_EXPUNGED_ONLY; - break; - case 'z': - options.require_compression = 1; - break; - case ':': - if (optopt == 'A') options.override_acl = ""; - else usage(); - break; - default: - usage(); - break; - } - } - - /* we need a server name */ - if (optind == argc) usage(); - servername = argv[optind++]; - - /* we need a source of backup data */ - if (mode == RESTORE_MODE_UNSPECIFIED) { - if (optind == argc) usage(); - backup_name = argv[optind++]; - mode = RESTORE_MODE_USERNAME; - } - - /* we need either an input file or some objects to restore */ - if (!do_all_mailboxes && !input_file && optind == argc) usage(); - /* and we can't have both because i said */ - if ((input_file || do_all_mailboxes) && optind < argc) usage(); - - /* okay, arguments seem sane, we are go */ - cyrus_init(alt_config, "restore", 0, 0); - - if ((r = mboxname_init_namespace(&restore_namespace, NAMESPACE_OPTION_ADMIN))) { - fatal(error_message(r), EX_CONFIG); - } - mboxevent_setnamespace(&restore_namespace); - - /* load the SASL plugins */ - global_sasl_init(1, 0, mysasl_cb); - - /* wait here for gdb attach */ - if (wait) { - fprintf(stderr, "Waiting for %d seconds for gdb attach...\n", wait); - sleep(wait); - } - - /* open backup */ - switch (mode) { - case RESTORE_MODE_FILENAME: - r = backup_open_paths(&backup, backup_name, NULL, - BACKUP_OPEN_NONBLOCK, BACKUP_OPEN_NOCREATE); - break; - case RESTORE_MODE_MBOXNAME: - mbname = mbname_from_extname(backup_name, &restore_namespace, NULL); - if (!mbname) usage(); - r = backup_open(&backup, mbname, - BACKUP_OPEN_NONBLOCK, BACKUP_OPEN_NOCREATE); - mbname_free(&mbname); - break; - case RESTORE_MODE_USERNAME: - mbname = mbname_from_userid(backup_name); - if (!mbname) usage(); - r = backup_open(&backup, mbname, - BACKUP_OPEN_NONBLOCK, BACKUP_OPEN_NOCREATE); - mbname_free(&mbname); - break; - default: - usage(); - break; - } - - if (r) goto done; - - /* scan for objects to restore: - * mailboxes will have all messages added, modulo expunged_mode - * messages will be added individually with appropriate folder - */ - mailbox_list = xzmalloc(sizeof *mailbox_list); - reserve_folder_list = restore_make_reserve_folder_list(backup); - reserve_list = sync_reserve_list_create(SYNC_MSGID_LIST_HASH_SIZE); - - if (input_file) { - char buf[MAX_MAILBOX_NAME + 2]; // \n\0 - size_t len; - FILE *f; - - if (0 != strcmp(input_file, "-")) { - f = fopen(input_file, "r"); - if (!f) { - fprintf(stderr, "fopen %s: %s\n", input_file, strerror(errno)); - goto done;// FIXME shut_down? - } - } - else { - f = stdin; - } - - while (fgets(buf, sizeof(buf), f)) { - len = strlen(buf); - - if (len > 0 && buf[len - 1] == '\n') - buf[--len] = '\0'; - - if (len == 0 || buf[0] == '#') - continue; - - r = restore_add_object(buf, &options, backup, mailbox_list, - reserve_folder_list, reserve_list); - - // FIXME r - } - fclose(f); - } - else if (do_all_mailboxes) { - struct backup_mailbox_list *all_mailboxes; - struct backup_mailbox *mailbox; - - all_mailboxes = backup_get_mailboxes(backup, 0, - BACKUP_MAILBOX_ALL_RECORDS); - - for (mailbox = all_mailboxes->head; mailbox; mailbox = mailbox->next) { - restore_add_mailbox(mailbox, &options, mailbox_list, - reserve_folder_list, reserve_list); - } - - backup_mailbox_list_empty(all_mailboxes); - free(all_mailboxes); - } - else { - int i; - - for (i = optind; i < argc; i++) { - r = restore_add_object(argv[i], &options, backup, mailbox_list, - reserve_folder_list, reserve_list); - - // FIXME r - } - } - - if (do_nothing) { - if (options.verbose) - fprintf(stderr, "do nothing (-n) specified, exiting now\n"); - goto done; - } - - /* connect to destination */ - sync_cs.servername = servername; - sync_cs.flags = options.verbose ? SYNC_FLAG_VERBOSE : 0; - sync_connect(&sync_cs); - - if (!sync_cs.backend) { - // FIXME - r = -1; - goto done; - } - - /* do the restore */ - struct sync_reserve *reserve; - for (reserve = reserve_list->head; reserve; reserve = reserve->next) { - /* send APPLY RESERVE and parse missing lists */ - r = sync_reserve_partition(&sync_cs, reserve->part, - reserve_folder_list, - reserve->list); - if (r) goto done; - - /* send APPLY MESSAGEs */ - r = backup_prepare_message_upload(backup, - reserve->part, - reserve->list, - &sync_msgid_lookup, - &upload); - if (r) goto done; - /* upload in small(ish) blocks to avoid timeouts */ - while (upload->head) { - struct dlist *block = dlist_splice(upload, 1024); - sync_send_apply(block, sync_cs.backend->out); - r = sync_parse_response("MESSAGE", sync_cs.backend->in, NULL); - dlist_unlink_files(block); - dlist_free(&block); - if (r) goto done; - } - } - - /* sync_prepare_dlists needs to upload messages per-mailbox, because - * it needs the mailbox to find the filename for the message. but - * we have no such limitation, so we can upload messages while - * looping over the reserve_list instead, above. - * - * alternatively, if we do it on a per-mailbox basis then we can limit - * the hit on the staging directory to only a mailbox worth of messages - * at a time. but we don't have a logical grouping that would make - * this coherent - */ - - /* send RESTORE MAILBOXes */ - struct backup_mailbox *mailbox; - for (mailbox = mailbox_list->head; mailbox; mailbox = mailbox->next) { - /* XXX filter the mailbox records based on reserve/missing/upload */ - - /* XXX does this sensibly handle mailbox objs with empty values? */ - struct dlist *dl = backup_mailbox_to_dlist(mailbox); - if (!dl) continue; - - if (local_only) { - free(dl->name); - dl->name = xstrdup("LOCAL_MAILBOX"); - } - - sync_send_restore(dl, sync_cs.backend->out); - r = sync_parse_response("MAILBOX", sync_cs.backend->in, NULL); - dlist_free(&dl); - if (r) goto done; - } - -done: - if (r) - fprintf(stderr, "%s: %s\n", backup_name, error_message(r)); - - /* release lock asap */ - if (backup) - backup_close(&backup); - - if (upload) { - dlist_unlink_files(upload); - dlist_free(&upload); - } - - if (mailbox_list) { - backup_mailbox_list_empty(mailbox_list); - free(mailbox_list); - } - - if (reserve_folder_list) - sync_folder_list_free(&reserve_folder_list); - - if (reserve_list) - sync_reserve_list_free(&reserve_list); - - sync_disconnect(&sync_cs); - free(sync_cs.backend); - - backup_cleanup_staging_path(); - cyrus_done(); - - exit(r ? EX_TEMPFAIL : EX_OK); -} - -static void my_mailbox_list_add(struct backup_mailbox_list *mailbox_list, - struct backup_mailbox *mailbox) -{ - struct backup_mailbox *tmp; - - for (tmp = mailbox_list->head; tmp; tmp = tmp->next) { - if (0 == strcmp(tmp->mboxname, mailbox->mboxname)) break; - } - - if (tmp) { - /* mailbox already in our list -- append the records to it */ - if (mailbox->records && mailbox->records->count) { - if (!tmp->records) { - tmp->records = mailbox->records; - mailbox->records = NULL; - } - else if (!tmp->records->head) { - tmp->records->head = mailbox->records->head; - tmp->records->tail = mailbox->records->tail; - tmp->records->count = mailbox->records->count; - memset(mailbox->records, 0, sizeof *mailbox->records); - } - else { - tmp->records->tail->next = mailbox->records->head; - tmp->records->tail = mailbox->records->tail; - tmp->records->count += mailbox->records->count; - memset(mailbox->records, 0, sizeof *mailbox->records); - } - } - - /* release the mailbox we were given, since we're not holding it */ - backup_mailbox_free(&mailbox); - } - else { - /* not already in our list -- just add it */ - backup_mailbox_list_add(mailbox_list, mailbox); - } -} - -static struct sync_folder_list *restore_make_reserve_folder_list( - struct backup *backup) -{ - struct sync_folder_list *folder_list = sync_folder_list_create(); - struct backup_mailbox_list *mailboxes = NULL; - struct backup_mailbox *iter; - - mailboxes = backup_get_mailboxes(backup, 0, BACKUP_MAILBOX_NO_RECORDS); - - if (mailboxes) { - for (iter = mailboxes->head; iter; iter = iter->next) { - const struct synccrcs synccrcs = { 0, 0 }; - - /* we only care about mboxname here */ - sync_folder_list_add(folder_list, NULL, iter->mboxname, - 0, NULL, NULL, 0, 0, 0, 0, synccrcs, - 0, 0, 0, 0, NULL, 0, 0, 0, 0); - } - - backup_mailbox_list_empty(mailboxes); - free(mailboxes); - } - - return folder_list; -} - -static void apply_mailbox_options(struct backup_mailbox *mailbox, - const struct restore_options *options) -{ - if (options->override_mboxname) { - if (options->verbose) { - fprintf(stderr, "%s: overriding mboxname with %s\n", - mailbox->mboxname, options->override_mboxname); - } - if (mailbox->mboxname) free(mailbox->mboxname); - mailbox->mboxname = xstrdup(options->override_mboxname); - } - - if (options->override_partition) { - if (options->verbose) { - fprintf(stderr, "%s: overriding partition with %s (was %s)\n", - mailbox->mboxname, options->override_partition, mailbox->partition); - } - if (mailbox->partition) free(mailbox->partition); - mailbox->partition = xstrdup(options->override_partition); - } - - if (options->override_acl) { - if (options->verbose) { - fprintf(stderr, "%s: overriding acl with '%s' (was '%s')\n", - mailbox->mboxname, options->override_acl, mailbox->acl); - } - if (mailbox->acl) free(mailbox->acl); - - /* treat empty override string as no ACL, resulting in the mailbox - * being restored with the default ACL for its owner. */ - if (*options->override_acl) { - mailbox->acl = xstrdup(options->override_acl); - } - else { - mailbox->acl = NULL; - } - } - - if (!options->keep_uidvalidity) { - if (options->verbose) { - fprintf(stderr, "%s: not trying to keep uidvalidity\n", - mailbox->mboxname); - } - if (mailbox->uniqueid) free(mailbox->uniqueid); - mailbox->uniqueid = NULL; - mailbox->highestmodseq = 0; - mailbox->uidvalidity = 0; - } - else if (options->verbose) { - fprintf(stderr, "%s: trying to keep uidvalidity(%u), " - "uniqueid(%s), highestmodseq(" MODSEQ_FMT ")\n", - mailbox->mboxname, - mailbox->uidvalidity, - mailbox->uniqueid, - mailbox->highestmodseq); - } - - if (mailbox->mboxname && options->trim_deletedprefix) { - mbname_t *mbname = mbname_from_intname(mailbox->mboxname); - if (mbname_isdeleted(mbname)) { - char *freeme = mailbox->mboxname; - mbname_set_isdeleted(mbname, 0); - mailbox->mboxname = xstrdup(mbname_intname(mbname)); - if (options->verbose) { - fprintf(stderr, "%s: removing deletedprefix (was %s)\n", - mailbox->mboxname, freeme); - } - free(freeme); - } - mbname_free(&mbname); - } - - if (options->expunged_mode != RESTORE_EXPUNGED_OKAY) { - struct backup_mailbox_message *iter, *tmp, *next; - - next = mailbox->records->head; - while ((iter = next)) { - next = iter->next; - tmp = NULL; - - switch (options->expunged_mode) { - case RESTORE_EXPUNGED_EXCLUDE: - if (iter->expunged) { - tmp = backup_mailbox_message_list_remove(mailbox->records, iter); - - if (options->verbose) { - fprintf(stderr, "%s: excluding expunged message: %s\n", - mailbox->mboxname, - message_guid_encode(&iter->guid)); - } - } - break; - case RESTORE_EXPUNGED_ONLY: - if (!iter->expunged) { - tmp = backup_mailbox_message_list_remove(mailbox->records, iter); - - if (options->verbose) { - fprintf(stderr, "%s: excluding unexpunged message: %s\n", - mailbox->mboxname, - message_guid_encode(&iter->guid)); - } - } - break; - default: - break; - } - - if (tmp) - backup_mailbox_message_free(&tmp); - } - } -} - -static int restore_add_mailbox(const struct backup_mailbox *mailbox, - const struct restore_options *options, - struct backup_mailbox_list *mailbox_list, - struct sync_folder_list *reserve_folder_list, - struct sync_reserve_list *reserve_list) -{ - struct backup_mailbox *clone = backup_mailbox_clone(mailbox); - - apply_mailbox_options(clone, options); - - /* populate reserve list */ - if (clone->records) { - struct sync_msgid_list *msgid_list = NULL; - struct backup_mailbox_message *record = NULL; - - msgid_list = sync_reserve_partlist(reserve_list, clone->partition); - for (record = clone->records->head; record; record = record->next) { - sync_msgid_insert(msgid_list, &record->guid); - } - } - - /* add to folder list if not already-extant folder */ - if (options->override_mboxname) { - const struct synccrcs synccrcs = {0, 0}; - sync_folder_list_add(reserve_folder_list, NULL, clone->mboxname, - 0, NULL, NULL, 0, 0, 0, 0, synccrcs, - 0, 0, 0, 0, NULL, 0, 0, 0, 0); - } - - /* populate mailbox list */ - my_mailbox_list_add(mailbox_list, clone); - - return 0; -} - -static int restore_add_message(const struct backup_message *message, - const struct backup_mailbox_list *message_mailboxes, - const struct restore_options *options, - struct backup_mailbox_list *mailbox_list, - struct sync_folder_list *reserve_folder_list, - struct sync_reserve_list *reserve_list) -{ - struct sync_msgid_list *msgid_list = NULL; - - if (options->override_mboxname) { - /* create a mailbox... */ - struct backup_mailbox *mailbox = xzmalloc(sizeof *mailbox); - apply_mailbox_options(mailbox, options); - if (!mailbox->partition) - mailbox->partition = xstrdup(message->partition); - - /* ... containing this message */ - struct backup_mailbox_message *mailbox_message = - xzmalloc(sizeof *mailbox_message); - - mailbox_message->guid = *message->guid; - mailbox_message->size = message->length; - - mailbox->records = xzmalloc(sizeof *mailbox->records); - mailbox->records->head = mailbox->records->tail = mailbox_message; - mailbox->records->count = 1; - - /* add to reserve list */ - msgid_list = sync_reserve_partlist(reserve_list, mailbox->partition); - sync_msgid_insert(msgid_list, message->guid); - - /* add to reserve folder list */ - const struct synccrcs synccrcs = {0, 0}; - sync_folder_list_add(reserve_folder_list, NULL, mailbox->mboxname, - 0, NULL, NULL, 0, 0, 0, 0, synccrcs, - 0, 0, 0, 0, NULL, 0, 0, 0, 0); - - /* add to mailbox list */ - my_mailbox_list_add(mailbox_list, mailbox); - } - else if (message_mailboxes) { - struct backup_mailbox *iter; - - for (iter = message_mailboxes->head; iter; iter = iter->next) { - struct backup_mailbox *clone = backup_mailbox_clone(iter); - apply_mailbox_options(clone, options); - - /* add to reserve list */ - msgid_list = sync_reserve_partlist(reserve_list, clone->partition); - sync_msgid_insert(msgid_list, message->guid); - - /* add to mailbox list */ - my_mailbox_list_add(mailbox_list, clone); - } - } - - return 0; -} - -struct submailbox_rock { - const char *prefix; - size_t prefix_len; - const struct restore_options *options; - struct backup_mailbox_list *mailbox_list; - struct sync_folder_list *reserve_folder_list; - struct sync_reserve_list *reserve_list; -}; - -static int submailbox_cb(const struct backup_mailbox *mailbox, void *rock) -{ - struct submailbox_rock *smbrock = (struct submailbox_rock *) rock; - - if (0 == strncmp(smbrock->prefix, mailbox->mboxname, smbrock->prefix_len)) { - return restore_add_mailbox(mailbox, - smbrock->options, - smbrock->mailbox_list, - smbrock->reserve_folder_list, - smbrock->reserve_list); - } - - return 0; -} - -static int restore_add_object(const char *object_name, - const struct restore_options *options, - struct backup *backup, - struct backup_mailbox_list *mailbox_list, - struct sync_folder_list *reserve_folder_list, - struct sync_reserve_list *reserve_list) -{ - struct backup_mailbox *mailbox = NULL; - struct backup_message *message = NULL; - struct message_guid tmp_guid; - size_t len; - int r; - - /* try to work out what we're restoring */ - len = strlen(object_name); - if (len == 24 && strspn(object_name, HEX_DIGITS) == len) { - /* looks like a non-libuuid uniqueid */ - mailbox = backup_get_mailbox_by_uniqueid(backup, object_name, - BACKUP_MAILBOX_ALL_RECORDS); - } - else if (len == 36 && strspn(object_name, "-" HEX_DIGITS) == len) { - /* looks like a libuuid uniqueid */ - mailbox = backup_get_mailbox_by_uniqueid(backup, object_name, - BACKUP_MAILBOX_ALL_RECORDS); - } - else if (message_guid_decode(&tmp_guid, object_name)) { - /* looks like it's a message guid */ - message = backup_get_message(backup, &tmp_guid); - } - else if (strchr(object_name, '.')) { - /* has a dot, might be an mboxname */ - mbname_t *mbname = mbname_from_extname(object_name, - &restore_namespace, NULL); - mailbox = backup_get_mailbox_by_name(backup, mbname, - BACKUP_MAILBOX_ALL_RECORDS); - mbname_free(&mbname); - } - else { - /* not sure what it is, guess mboxname? */ - mbname_t *mbname = mbname_from_extname(object_name, - &restore_namespace, NULL); - mailbox = backup_get_mailbox_by_name(backup, mbname, - BACKUP_MAILBOX_ALL_RECORDS); - mbname_free(&mbname); - } - - /* add it to the restore lists */ - if (mailbox) { - r = restore_add_mailbox(mailbox, options, mailbox_list, - reserve_folder_list, reserve_list); - - if (!r && options->do_submailboxes) { - char prefix[MAX_MAILBOX_NAME + 1]; - int len; - - len = snprintf(prefix, sizeof(prefix), "%s.", mailbox->mboxname); - - /* can only be submailboxes if parent's name is short enough... */ - if (len < MAX_MAILBOX_NAME) { - struct submailbox_rock rock = { - prefix, - strlen(prefix), - options, - mailbox_list, - reserve_folder_list, - reserve_list, - }; - - r = backup_mailbox_foreach(backup, 0, - BACKUP_MAILBOX_ALL_RECORDS, - submailbox_cb, &rock); - } - } - - backup_mailbox_free(&mailbox); - } - else if (message) { - struct backup_mailbox_list *mailboxes = NULL; - - if (!options->override_mboxname) - mailboxes = backup_get_mailboxes_by_message(backup, message, - BACKUP_MAILBOX_MATCH_RECORDS); - r = restore_add_message(message, mailboxes, options, mailbox_list, - reserve_folder_list, reserve_list); - - if (mailboxes) { - backup_mailbox_list_empty(mailboxes); - free(mailboxes); - } - backup_message_free(&message); - } - else { - r = IMAP_MAILBOX_NONEXISTENT; - } - - return r; -} diff --git a/doc/internal/backup b/doc/internal/backup deleted file mode 100644 index 6728f29324d..00000000000 --- a/doc/internal/backup +++ /dev/null @@ -1,903 +0,0 @@ -Notes for backup implementation - -Backup index database (one per user): - -chunk: - int id - timestamp ts - int offset - int length - text file_sha1 -> sha1 of (compressed) data prior to this chunk - text data_sha1 -> sha1 of (uncompressed) data contained in this chunk - -mailbox: - int id - int last_chunk_id -> chunk that knows the current state - char uniqueid -> unique - char mboxname -> altered by a rename - char mboxtype - int last_uid - int highestmodseq - int recentuid - timestamp recenttime - timestamp last_appenddate - timestamp pop3_last_login - timestamp pop3_show_after - timestamp uidvalidity - char partition - char acl - char options - int sync_crc - int sync_crc_annot - char quotaroot - int xconvmodseq - char annotations - timestamp deleted -> time that it was unmailbox'd, or NULL if still alive - -message: - int id - char guid - char partition -> this is used to set the spool directory for the temp file - we might not need it - int chunk_id - int offset -> offset within chunk of dlist containing this message - int size -> size of this message (n.b. not length of dlist) - -mailbox_message: - int mailbox_id - int message_id - int last_chunk_id -> chunk that has a RECORD in a MAILBOX for this - int uid - int modseq - timestamp last_updated - char flags - timestamp internaldate - int size - char annotations - timestamp expunged -> time that it was expunged, or NULL if still alive - -subscription: - int last_chunk_id -> chunk that knows the current state - char mboxname -> no linkage to mailbox table, users can be sub'd to nonexistent - timestamp unsubscribed -> time that it was unsubscribed, or NULL if still alive - -seen: - int last_chunk_id -> chunk that knows the current state - char uniqueid -> mailbox (not necessarily ours) this applies to - timestamp lastread - int lastuid - timestamp lastchange - char seenuids -> a uid sequence encoded as a string - -sieve: - int chunk_id - timestamp last_update - char filename - char guid - int offset -> offset within chunk of the dlist containing this script - timestamp deleted -> time that it was deleted, or NULL if still alive - - -sieve scripts and messages are both identified by a GUID -but APPLY SIEVE doesn't take a GUID, it seems to be generated locally? -the GUID in the response to APPLY SIEVE is generated in the process of -reading the script from disk (sync_sieve_list_generate) - -can't activate scripts because only bytecode files are activated, but -we neither receive bytecode files over sync protocol nor do we compile -them ourselves. - -possibly reduce index size by breaking deleted/expunged values into their -own tables, such that we only store a deleted value for things that are -actually deleted. use left join + is null to find undeleted content - - -messages --------- - -APPLY MESSAGE is a list of messages, not necessarily only one message. -Actually, it's a list of messages for potentially multiple users, but we avoid -this by rejecting GET MESSAGES requests that span multiple users (so that -sync_client retries at USER level, and so we only see APPLY MESSAGE requests -for a single user). - -Cheap first implementation is to index the start/end of the entire APPLY -MESSAGE command identically for each message within it, and at restore time -we grab that chunk and loop over it looking for the correct guid. - -Ideal implementation would be to index the offset and length of each message -exactly (even excluding the dlist wrapper), but this is rather complicated -by the dlist API. - -For now, we just index the offset of the dlist entry for the message, -and we can parse the pure message data back out later from that, when -we need to. Slightly less efficient on reads, but works->good->fast. We -need to loop over the entries in the MESSAGE dlist to find the one with the -desired GUID. - -The indexed length needs to be the length of the message, not the length of the -dlist wrapper, because we need to know this cheaply to supply RECORDs in -MAILBOX responses. - - -renames -------- - -APPLY RENAME %(OLDMBOXNAME old NEWMBOXNAME new PARTITION p UIDVALIDITY 123) - -We identify mboxes by uniqueid, so when we start seeing sync data for the same -uniqueid with a new mboxname we just transparently update it anyway, without -needing to handle the APPLY RENAME. Not sure if this is a problem... Do we -need to record an mbox's previous names somehow? - -I think it's possible to use this to rename a USER though, something like: - -APPLY RENAME %(OLDMBOXNAME example.com!user.smithj NEWMBOXNAME example.com!user.jsmith ...) - --- in which case, without special handling of the RENAME command itself, there -will be a backup for the old user that ends with the RENAME, and a backup of -the new user that (probably) duplicates everything again (except for stuff -that's been expunged). - -And if someone else gets given the original name, like - -APPLY RENAME %(OLDMBOXNAME example.com!user.samantha-mithj NEWMBOXNAME example.com!user.smithj ...) - -Then anything that was expunged from the original user but still available in -backup disappears? Or the two backups get conflated, and samantha can -"restore" the original smithj's old mail? - -Uggh. - -if there's a mailboxes database pointing to the backup files, then the backup -file names don't need to be based on the userid, they could e.g. be based on -the user's inbox's uniqueid. this would make it easier to deal with user -renames because the backup filename wouldn't need to change. but this depends -on the uniqueid(s) in question being present on most areas of the sync -protocol, otherwise when starting a backup of a brand new user we won't be -able to tell where to store it. workaround in the meantime could be to make -some kind of backup id from the mailboxes database, and base the filename on -this. - -actually, using "some kind of backup id from the mailboxes database" is probably -the best solution. otherwise the lock complexity of renaming a user while making -sure their new backup filename doesn't already exist is frightful. - -maybe do something with mkstemp()? - -furthermore: what if a mailbox is moved from one user to another? like: - -APPLY RENAME %(OLD... example.com!user.foo.something NEW... example.com!user.bar.something ...) - -when a different-uid rename IS a rename of a user (and not just a folder -being moved to a different user), what does it look like? -* does it do a single APPLY RENAME for the user, and expect their folders to - shake out of that? -* does it do an APPLY RENAME for each of their folders? - -in the latter case, we need to append each of those RENAMEs to the old backup -so they can take effect correctly, and THEN rename the backup file itself. but -how to tell when the appends are finished? - -how can we tell the difference between folder(s) moved to a different user vs -user has been renamed? - -there is a setting: 'allowusermoves: 0' which, when enabled, allows users to -be renamed via IMAP rename/xfer commands. but the default is that this is -disabled. we could initially require this to be disabled while using backups... - -not sure what the workflow looks like for renaming a user if this is not enabled. - -not sure what the sync flow looks like in either case. - -looking at sync_apply_rename and mboxlist_renamemailbox, it seems like we'll -see an APPLY RENAME for each affected mbox when a recursive rename is occurring. - -there doesn't seem to be anything preventing user/a/foo -> user/b/foo in the -general (non-INBOX) case. - -renames might be a little easier to handle if the index replicated the mailbox -hierarchy rather than just being a flat structure. though this adds complexity -wrt hiersep handling. something like: - - mailbox: - mboxname # just the name of this mbox - parent_id # fk to parent mailbox - full_mboxname # cached value, parent.full_mboxname + mboxname - - -locking -------- - -just use a normal flock/fcntl lock on the data file and only open the index -if that lock succeeded - -backup: needs to append foo and update foo.index -reindex: only needs to read foo, but needs a write lock to prevent writes - while it does so. needs to write to (replace) foo.index -compact: needs to re-write foo and foo.index -restore: needs to read - - -verifying index ---------------- - -how to tell whether the .index file is the correct one for the backup data it -ostensibly represents? - -one way to do this would be to have backup_index_end() store a checksum of -the corresponding data contents in the index. - -when opening a backup, verify this checksum against the data, and refuse to -load the index if it doesn't match. - -- sha1sum of (compressed) contents of file prior to each chunk - -how to tell whether the chunk data is any good? store a checksum of the chunk -contents along with the rest of the chunk index - -- sha1sum of (uncompressed) contents of each chunk - - -mailboxes database ------------------- - -bron reckons use twoskip for this -userid -> backup_filename - -lib/cyrusdb module implements this, look into that - -look at conversations db code to see how to use it - -need a tool: - * given a user, show their backup filename - * dump/undump - * rebuild based on files discovered in backup directory - -where does this fit into the locking scheme? - - -reindex -------- - -* convert user mailbox name to backup name -* complain if there's no backup data file? -* lock, rename .index to .index.old, init new .index -* foreach file chunk: -* timestamp is from first line in chunk -* complain if timestamp has gone backwards? -* index records from chunk -* unlock -* clean up .index.old - -on error: -* discard partial new index -* restore .index.old -* bail out - - -backupd -------- - -cmdloop: -* (periodic cleanup) -* read command, determine backup name -* already holding lock ? bump timestamp : obtain lock -* write data to gzname, flush immediately -* index data - -periodic cleanup: -* check timestamp of each held lock -* if stale (define: stale?), release -* FIXME if we've appended more than the chunk size we would compact to, release - -sync restart: -* release each held lock - -exit: -* release each held lock - -need a "backup_index_abort" to complete the backup_index_start/end set. -_start should create a transaction, _end should commit it, and _abort should -roll it back. then, if backupd fails to write to the gzip file for some -reason, the (now invalid) index info we added can be discarded too. - -flushing immediately on write results in poor gzip compression, but for -incremental backups that's not a problem. when the compact process hits the -file it will recompress the data more efficiently. - - -questions ---------- -* what does it look like when uidvalidity changes? - - -restore -------- - -restoration is effectively a reverse-direction replication (replicating TO master), -which means we can't necessarily supply things like uid, modseq, etc without racing -against normal message arrivals. so instead we add an extra command to the protocol -to restore a message to a folder but let the destination determine the tasty bits. - -protocol flow looks something like: - -c: APPLY RESERVE ... # as usual -s: * MISSING (foo bar) -s: OK -c: APPLY MESSAGE ... # as usual -s: OK -c: RESTORE MAILBOX ... # new sync proto command -s: OK - -we introduce a new command, RESTORE MAILBOX, which is similar to the existing -APPLY MAILBOX. it specifies, for a mailbox, the mailbox state plus the message -records relevant to the restore. - -the imapd/sync_server receiving the RESTORE command creates the mailbox if necessary, -and then adds the message records to it as new records (i.e. generating new uid etc). -this will end up generating new events in the backup channel's sync log, and then the -messages will be backed up again with their new uids, etc. additional wire transfer -of message data should be avoided by keeping the same guid. - -if the mailbox already exists but its uniqueid does not match the one from the backup, -then what? this probably means user has deleted folder and contents, then made new -folder with same name. so it's probably v common for mailbox uniqueid to not match -like this. so we don't care about special handling for this case. just add any -messages that aren't already there. - -if the mailbox doesn't already exist on the destination (e.g. if rebuilding a server -from backups) then it's safe and good to reuse uidvalidity, uniqueid, uid, modseq etc, -such that connecting clients can preserve their state. so the imapd/sync_server -receiving the restore request accepts these fields as optional, but only preserves -them if it's safe to do so. - -* restore: sbin program for selecting and restoring messages - -restore command needs options: -+ whether or not to trim deletedprefix off mailbox names to be restored -+ whether or not to restore uniqueid, highestmodseq, uid and so on -+ whether or not to limit to/exclude expunged messages -+ whether or not to restore sub-mailboxes -+ sync_client-like options (servername, local_only, partition, ...) -+ user/mailbox/backup file(s) to restore from -+ mailbox to restore to (override location in backup) -+ override acl? - -can we heuristically determine whether an argument is an mboxname, uniqueid or guid? - => libuuid uniqueid is 36 bytes of hyphen (at fixed positions) and hex digits - => non-libuuid uniqueid is 24 bytes of hex digits - => mboxname usually contains at least one . somewhere - => guid is 40 bytes of hex digits - -usage: - restore [options] server [mode] backup [mboxname | uniqueid | guid]... - -options: - -A acl # apply specified acl to restored mailboxes - -C alt_config # alternate config file - -D # don't trim deletedprefix before restoring - -F input-file # read mailboxes/messages from file rather than argv - -L # local mailbox operations only (no mupdate) - -M mboxname # restore messages to specified mailbox - -P partition # restore mailboxes to specified partition - -U # try to preserve uniqueid, uid, modseq, etc - -X # don't restore expunged messages - -a # try to restore all mailboxes in backup - -n # calculate work required but don't perform restoration - -r # recurse into submailboxes - -v # verbose - -w seconds # wait before starting (useful for attaching a debugger) - -x # only restore expunged messages (not sure if useful?) - -z # require compression (abort if compression unavailable) - -mode: - -f # specified backup interpreted as filename - -m # specified backup interpreted as mboxname - -u # specified backup interpreted as userid (default) - - -compact --------- - -# finding messages that are to be kept (either exist as unexpunged somewhere, -# or exist as expunged but more recently than threshold) -# (to get unique rows, add "distinct" and remove mm.expunged from fields) -sqlite> select m.*, mm.expunged from message as m join mailbox_message as mm on m.id = mm.message_id and (mm.expunged is null or mm.expunged > 1437709300); -id|guid|partition|chunk_id|offset|length|expunged -1|1c7cca361502dfed2d918da97e506f1c1e97dfbe|default|1|458|2159| -1|1c7cca361502dfed2d918da97e506f1c1e97dfbe|default|1|458|2159|1446179047 -1|1c7cca361502dfed2d918da97e506f1c1e97dfbe|default|1|458|2159|1446179047 - -# finding chunks that are still needed (due to containing last state -# of mailbox or mailbox_message, or containing a message) -sqlite> select * from chunk where id in (select last_chunk_id from mailbox where deleted is null or deleted > 1437709300 union select last_chunk_id from mailbox_message where expunged is null or expunged > 1437709300 union select chunk_id from message as m join mailbox_message as mm on m.id = mm.message_id and (mm.expunged is null or mm.expunged > 1437709300)); -id|timestamp|offset|length|file_sha1|data_sha1 -1|1437709276|0|3397|da39a3ee5e6b4b0d3255bfef95601890afd80709|6836d0110252d08a0656c14c2d2d314124755491 -3|1437709355|1977|2129|fee183c329c011ead7757f59182116500776eaaf|a5677cfa1f5f7b627763652f4bb9b99f5970748c -4|1437709425|2746|1719|3d9f02135bf964ff0b6a917921b862c3420e48f0|7b64ec321457715ee61fe238f178f5d72adaef64 -5|1437709508|3589|2890|0cee599b1573110fee428f8323690cbcb9589661|90d104346ef3cba9e419461dd26045035f4cba02 - -remember: a single APPLY MESSAGE line can contain many messages! - -thoughts: -* need a heuristic for quickly determining whether a backup needs to be compacted - -> sum(chunks to discard, chunks to combine, chunks to split) > threshold - -> can we detect chunks that are going to significantly reduce in size as - result of discarding individual lines? -* "quick" vs "full" compaction - -settings: - -* backup retention period -* chunk combination size (byte length or elapsed time) - -combining chunks: -* size threshold below which adjacent chunks can be joined -* size threshold above which chunks should be split -* duration threshold below which adjacent chunks can be joined -* duration threshold above which chunks should be split -backup_min_chunk_size: 0 for no minimum -backup_max_chunk_size: 0 for no maximum -backup_min_chunk_duration: 0 for no minimum -backup_max_chunk_duration: 0 for no maximum -priority: size or duration?? - -data we absolutely need to keep: - -* the most recent APPLY MAILBOX for each mailbox we're keeping (mailbox state) -* the APPLY MAILBOX containing the most recent RECORD for each message we're keeping (record state) -* the APPLY MESSAGE for each message we're keeping (message data) - -data that we should practically keep: - -* all APPLY MAILBOXes for a given mailbox from the chunk identified as its last -* all APPLY MAILBOXes containing a RECORD for a given message from the chunk identified as its last -* the APPLY MESSAGE for each message we're keeping - -four kinds of compaction (probably at least two simultaneously): - -* removing unused chunks -* combining adjacent chunks into a single chunk (for better gz compression) -* removing unused message lines from within a chunk (important after combining) -* removing unused messages from within a message line - -"unused messages" - messages for which all records have been expunged for longer - than the retention period -"unused chunks" - chunks which contain only unused messages - -algorithm: - -* open (and lock) backup and backup.new (or bail out) -* use backup index to identify chunks we still need -* create a chunk in backup.new -* foreach chunk we still need: -* foreach line in the chunk: -* next line if we don't need to keep it -* create new line -* foreach message in line: -* if we still need the message, or if we're not doing message granularity -* add the message to the new line -* write and index tmp line to backup.new -* if the new chunk is big enough, or if we're not combining -* end chunk and start a new one -* end the new chunk -* rename backup->backup.old, backup.new->backup -* close (and unlock) backup.old and backup - - -command line locking utility -------- - -command line utility to lock a backup (for e.g. safely poking around in the -.index on a live system). - -example failure: -$ctl_backups lock -f /path/to/backup -* Trying to obtain lock on /path/to/backup... -NO some error - - -example success: -$ctl_backups lock -f /path/to/backup -* Trying to obtain lock on /path/to/backup... -[potentially a delay here if we need to wait for another process to release the lock] -OK locked -[waits for its stdin to close, then unlocks and exits] - -if you need to rummage around in backup.index, run this program in another -shell, do your work, then ^D it when you're finished. - -you could also call this from e.g. perl over a bidirectional pipe - wait to -read "OK locked", then you've got your lock. close the pipe to unlock when -you're finished working. if you don't read "OK locked" before the pipe closes -then something went wrong and you didn't get the lock. - -specify backups by -f filename, -m mailbox, -u userid -default run mode as above --s to fork an sqlite of the index (and unlock when it exits) --x to fork a command of your choosing (and unlock when it exits) - - -reconstruct ------------ - -rebuilding backups.db from on disk files - -scan each backup partition for backup files: - * skip timestamped files (i.e. backups from compact/reindex) - * skip .old files (old backups from reindex) - * .index files => skip??? - * skip unreadable files - * skip empty files - * skip directories etc - -what's the correct procedure for repopulating a cyrus database? -keep copy of the previous (presumably broken) one? - -trim off mkstemp suffix (if any) to find userid -can we use a recognisable character to delimit the mkstemp suffix? - -what if there's multiple backup files for a given userid? precedence? - -verify found backups before recording. reindex? - -locking? what if something has a filename and does stuff with it while -reconstruct runs? - -backupd always uses db for opens, so as long as reconstruct keeps the db -locked while it works, the db won't clash. but backupd might have backups -still open from before reconstruct started, which it will write to quite -happily, even though reconstruct might decide that some other file is the -correct one for that user... - -a backup server would generally be used only for backups, and sync_client -is quite resilient when the destination isn't there, so it's actually -no problem to just shut down cyrus while reconstruct runs. no outage to -user-facing services, just maybe some sync backlog to catch up on once -cyrus is restarted. - - -ctl_backups -------------- - -sbin tool for mass backup/index/database operations - -needs: - * rebuild backups.db from disk contents - * list backups/info - * rename a backup - * delete a backup - * verify a backup (check all sha1's, not just most recent) - -not sure if these should be included, or separate tools: - * reindex a backup (or more) - * compact a backup (or more) - * lock a backup - * some sort of rolling compaction? - -usage: - ctl_backups [options] reconstruct # reconstruct backups.db from disk files - ctl_backups [options] list [list_opts] [[mode] backup...] # list backup info for given/all users - ctl_backups [options] move new_fname [mode] backup # rename a backup (think about this more) - ctl_backups [options] delete [mode] backup # delete a backup - ctl_backups [options] verify [mode] backup... # verify specified backups - ctl_backups [options] reindex [mode] backup... # reindex specified backups - ctl_backups [options] compact [mode] backup... # compact specified backups - ctl_backups [options] lock [lock_opts] [mode] backup # lock specified backup - -options: - -C alt_config # alternate config file - -F # force (run command even if not needed) - -S # stop on error - -v # verbose - -w # wait for locks (i.e. don't skip locked backups) - -mode: - -A # all known backups (not valid for single backup commands) - -D # specified backups interpreted as domains (nvfsbc) - -P # specified backups interpreted as userid prefixes (nvfsbc) - -f # specified backups interpreted as filenames - -m # specified backups interpreted as mboxnames - -u # specified backups interpreted as userids (default) - -lock_opts: - -c # exclusively create backup - -s # lock backup and open index in sqlite - -x cmd # lock backup and execute cmd - -p # lock backup and wait for eof on stdin (default) - -list_opts: - -t [hours] # "stale" (no update in hours) backups only (default: 24) - - -cyr_backup ------- - -sbin tool for inspecting backups - -needs: - * better name? - * list stuff - * show stuff - * dump stuff - * restore? - -* should lock/move/delete (single backup commands) from ctl_backups be moved here? - -usage: - cyr_backup [options] [mode] backup list [all | chunks | mailboxes | messages]... - cyr_backup [options] [mode] backup show chunks [id...] - cyr_backup [options] [mode] backup show messages [guid...] - cyr_backup [options] [mode] backup show mailboxes [mboxname | uniqueid]... - cyr_backup [options] [mode] backup dump [dump_opts] chunk id - cyr_backup [options] [mode] backup dump [dump_opts] message guid - cyr_backup [options] [mode] backup json [chunks | mailboxes | messages]... - -options: - -C alt_config # alternate config file - -v # verbose - -mode: - -f # backup interpreted as filename - -m # backup interpreted as mboxname - -u # backup interpreted as userid (default) - -commands: - list: table of contents, one per line - show: indexed details of listed items, one per paragraph, detail per line - dump: relevant contents from backup stream - json: indexed details of listed items in json format - -dump options: - -o filename # dump to named file instead of stdout - - -partitions ----------- - -not enough information in sync protocol to handle partitions easily? - -we know what the partition is when we do an APPLY operation (mailbox, message, -etc), but the initial GET operations don't include it. so we need to already -know where the appropriate backup is partitioned in order to find the backup -file in order to look inside it to respond to the GET request - -if we have a mailboxes database (indexed by mboxname, uniqueid and userid) then -maybe that would make it feasible? if it's not in the mailboxes database then -we don't have a backup for it yet, so we respond accordingly, and get sent -enough information to create it. - -does that mean the backup api needs to take an mbname on open, and it handles -the job of looking it up in the mailboxes database to find the appropriate -thing to open? - -can we use sqlite for such a database, or is the load on it going to be too -heavy? locking? we have lots of database formats up our sleeves here, so -even though we use sqlite for the backup index there isn't any particular -reason we're beholden to it for the mailboxes db too - -if we have a mailboxes db then we need a reconstruct tool for that, too - -what if we support multiple backup partitions, but don't expect these -to necessarily correspond with mailbox partitions. they're just for spreading -disk usage around. - -* when creating a backup for a previously-unseen user we'd pick a random - partition to put them on -* ctl_backups would need a command to move an existing backup to a - given partition -* ctl_backups would need a command to pre-create a user backup on a - given partition for initial distribution -* instead of "backup_data_path" setting, have one-or-more - "backuppartition-" settings, ala partition- and friends - -see imap/partlist.[ch] for partition list management stuff. it's complicated -and doesn't have a test suite, so maybe save this implementation until needed. - -but... maybe rename backup_data_path to backuppartition-default in the meantime, -so that when we do add this it's not a complicated reconfig to update? - -partlist_local_select (and lazy-loaded partlist_local_init) are where the -mailbox partitions come from (see also mboxlist_create_partition), do something -similar for backup partitions - - -data corruption ---------------- - -backups.db: - * can be reconstructed from on disk files at any time - * how to detect corruption? does cyrus_db detect/repair on its own? - -backup indexes: - * can be reindexed at any time from backup data - * how to detect corruption? assume sqlite will notice, complain? - -backup data: - * what's zlib's failure mode? do we lose the entire chunk or just the corrupt bit? - * verify will notice sha1sum mismatches - * dlist format will reject some kinds of corruption (but not all) - * reindex: should skip unparseable dlist lines - * message data has its own checksums (guid) - * reindex: should skip messages that don't match their own checksums - * compact: "full" compact will only keep useful data according to index - * backupd: will sync anything that's in user mailbox but not in backup index - -i think this means that if a message or mailbox state becomes corrupted in -the backup data file, and it still exists in the user's real mailbox, you -recover from the corruption by reindexing and then letting the sync process -copy the missing data back in again. and you can tidy up the data file by -running a compact over it. - -you detect data corruption in most recent chunk reactively as soon as the -backup system needs to open it again (quick verify on open) - -you detect data corruption in older chunks reactively by trying to restore from -it. may be too late: if a message needs restoring it's because user mailbox no -longer has it - -you detect data corruption preemptively by running the verify tool over it. -recommend scheduling this in EVENTS/cron? - -if data corruption occurs in message that's no longer in user's mailbox, that -message is lost. it was going to be deleted from the backup after $retention -period anyway (by compact), but if it needs restoring in the meantime, sorry - - -installation instructions -------------------------- - -(obviously, most of this won't work at this point, because the code doesn't -exist. but this is, approximately, where things are heading.) - -on your backup server: - * compile with --enable-backup configure option and install - * imapd.conf: - backuppartition-default: /var/spool/backup # FIXME better example - backup_db: twoskip - backup_db_path: /var/imap/backups.db - backup_staging_path: /var/spool/backup - backup_retention_days: 7 - * cyrus.conf SERVICES: - backupd cmd="backupd" listen="csync" prefork=0 - (remove other services, most likely) - (should i create a master/conf/backup.conf example file?) - * cyrus.conf EVENTS: - compact cmd="ctl_backups compact -A" at=0400 - * start server as usual - * do i want a special port for backupd? - -on your imap server: - * imapd.conf: - sync_log_channels: backup - sync_log: 1 - backup_sync_host: backup-server.example.com - backup_sync_port: csync - backup_sync_authname: ... - backup_sync_password: ... - backup_sync_repeat_interval: ... # seconds, smaller value = livelier backups but more i/o - backup_sync_shutdown_file: .... - * cyrus.conf STARTUP: - backup_sync cmd="sync_client -r -n backup" - * cyrus.conf SERVICES: - restored cmd="restored" [...] - * start/restart master - -files and such: - {configdirectory}/backups.db - database mapping userids to backup locations - {backuppartition-name}//_XXXXXX - backup data stream for userid - {backuppartition-name}//_XXXXXX.index - index into userid's backup data stream - -do i want rhost in the path? - * protects from issue if multiple servers are trying to back up their own version of same user - (though this is its own problem that the backup system shouldn't have to compensate for) - * but makes location of undifferentiated user unpredictable - * so probably not, actually - - -chatting about implementation 20/10 ------------------------------------ -09:54 elliefm_ -here's a fun sync question -APPLY MESSAGE provides a list of messages -can a single APPLY MESSAGE contain messages for multiple mailboxes and/or users? -my first hunch is that it doesn't cross users, since the broadest granularity for a single sync run is USER -10:06 kmurchison -We'd have to check with Bron, but I *think* messages can cross mailboxes for a single user -10:06 brong_ -yes -APPLY MESSAGE just adds it to the reserve list -10:07 elliefm_ -nah apply message uploads the message, APPLY RESERVE adds it to the reserve list :P -10:07 brong_ -same same -APPLY RESERVE copies it from a local mailbox -APPLY MESSAGE uploads it -10:07 elliefm_ -yep -10:07 brong_ -they both wind up in the reserve list -10:07 elliefm_ -ahh i see what you mean, gotcha -10:07 brong_ -until you send a RESTART -ideally you want it reserve in the same partition, but it will copy the message over if it's not on the same partition -there's no restriction on which mailbox it came from/went to -good for user renames, and good for an append to a bunch of mailboxes in different users / shared space all at once -(which LMTP can do) -10:10 elliefm_ -i can handle the case where a single APPLY MESSAGE contains messages for multiple mailboxes belonging to the same user -but i'm in trouble if a single APPLY MESSAGE can contain messages belonging to different users -10:14 brong_ -elliefm_: why? -10:14 brong_ -you don't have to keep them if they aren't used -10:15 elliefm_ -for backups - when i see the apply, i need to know which user's backup to add it to. that's easy enough if it doesn't cross users but gets mega fiddly if it does -i'm poking around in sync client to see if it's likely to be an issue or not -11:00 brong__ -elliefm_: I would stage it, and add it to users as it gets refcounted in by an index file -11:07 elliefm_ -that's pretty much what we do for ordinary sync and delivery stuff yeah? -11:08 brong__ -yep -and it's what the backup thing does -11:09 elliefm_ -i'm pretty sure that APPLY RESERVE and APPLY MESSAGE don't give a damn about users, they're just "here's every message you might not have already had since last time we spoke" and it lets the APPLY MAILBOX work out where to attach them later -11:09 brong__ -yep -11:09 elliefm_ -so yeah, i'll need to do something here -i've been working so far on the idea that a single user's backup consists of 1) an append-only gzip stream of the sync protocol chat that built it, and 2) an index that tracks current state of mailboxes, and offsets within (1) of message data -that gets us good compression (file per user, not file per message), and if the index gets corrupted or lost, it's rebuildable purely from (1), it doesn't need a live copy of the original mailbox -11:12 brong_ -yep, that all works -11:12 elliefm_ -(so if you lose your imap server, you're not unable to rebuild a broken index on the backup) -11:13 brong_ -it's easy enough to require the sync protocol stream to only contain messages per user -though "apply reserve" is messy -because you need to return "yes, I have that message" -11:13 elliefm_ -with that implementation i can't (easily) keep user.a's messages from not existing in user.b's data stream (though they won't be indexed) -11:14 brong_ -I'm not too adverse to the idea of just unpacking each message as it comes off the wire into a temporary directory -11:14 elliefm_ -(because at the time i'm receiving the sync data i don't know which it needs to go in, so if they come in in the same reserve i'd need to append them to both data streams) -which isn't a huge problem, just… irks me a bit -11:14 brong_ -and then reading the indexes as they come in, checking against the state DB to see if we already have them, and streaming them into the gzip if they aren't there yet -what we can do is something like the current format, where files go into a tar -11:16 elliefm_ -i guess the fiddly bit there is that there's one more moving part to keep synchronised across failure states -a backup for a single user becomes 1) data stream + 2) any messages that were uploaded but not yet added to a mailbox + 3) index (which doesn't know what to do with (2)) -which in the general case is fine, the next sync will update the mailboxes, which will push (2) into (1) and index it nicely, and on we go -but it's just a little bit more mess if there's a failure that you need to recover from between those states — it's no longer a simple case of "it's in the backup and we know everything about it" or "it doesn't exist", there's a third case of "well we might have the data but don't really know what to do with it" -the other fiddly bit is that the process of appending to the data stream is suddenly in the business of crafting output rather than simply dumping what it gets, which isn't really burdensome, but it is one more little crack for bugs to crawl into -i guess in terms of sync protocol, one thing i could do on my end is identify apply operations that seem to contain multiple users' data, and just return an error on those. the sync client on the other end will promote them until they're eventually user syncs, which i think are always user granularity -11:50 elliefm_ -i think for now, first stage implementation will be to stream the reserve/message commands in full to every user backup they might apply to. and optimising that down so that each stream only contains messages belonging to that user can be a future optimisation - - -todo list ---------- - -* clean up error handling -* perl tool to anonymise sync proto talk -* verification step to check entire data stream for errors (even chunks that aren't indexed) -* prot_fill_cb: extra argument to pass back an error string to prot_fill -* ctl_backups verify: set level -* backupd: don't block on locked backups, return mailbox locked -- but sync_client doesn't handle this -* test multiple backup partitions -* configure: error if backups requested and we don't have zlib -* valgrind -* finish reconstruct -* compact: split before append? - -compact implementation steps: - 1 remove unused chunks, keep everything else as is - 2 join adjacent chunks if small enough, split large chunks - 3 parse/rebuild message lines - 4 discard unused mailbox lines diff --git a/doc/legacy/install-backups.html b/doc/legacy/install-backups.html deleted file mode 100644 index df9c50b0b1b..00000000000 --- a/doc/legacy/install-backups.html +++ /dev/null @@ -1,400 +0,0 @@ - - -Installing Cyrus Backups - - -

Installing Cyrus Backups

- -

Cyrus Backups are a replication-based backup service for Cyrus IMAP servers. -This is currently an experimental feature. If you have the resources to try -it out alongside your existing backup solutions, feedback would be appreciated.

- -

Introduction and assumptions

- -

This document is intended to be a guide to the configuration and administration -of Cyrus Backups.

- -

This document is a work in progress and at this point is incomplete.

- -

This document assumes that you are familiar with compiling, installing, -configuring and maintaining Cyrus IMAP servers generally, and will only discuss -backup-related portions in detail.

- -

This document assumes a passing familiarity with Cyrus Replication.

- -

Limitations

- -

Cyrus Backups are experimental and incomplete.

- -

The following components exist and appear to work:

-
    -
  • backupd, and therefore inbound replication
    • -
  • autovivification of backup storage for new users, with automatic partition selection
    • -
  • rebuilding of backup indexes from backup data files
    • -
  • compaction of backup files to remove stale data and combine chunks for better compression
    • -
  • deep verification of backup file/index state
    • -
  • examination of backup data
    • -
  • locking tool, for safe non-cyrus operations on backup files
    • -
- -

The following components don't yet exist in a workable state -- these tasks must be - massaged through manually (with care):

-

    -
  • recovery of data back into a Cyrus IMAP server
    • -
  • reconstruct of backups.db from backup files
    • -
- -

The following types of information are currently backed up

-
    -
  • mailbox state and annotations
    • -
  • messages
    • -
  • mailbox message records, flags, and annotations
    • -
- -

The following types of information are not currently backed up

-
    -
  • sieve scripts
    • -
  • subscriptions
    • -
  • quota information
    • -
  • seen data (?) other than the basic \Seen flag attached to a mailbox message record
    • -
- -

Architecture

- -

Cyrus Backups are designed to run on one or more standalone, dedicated backup -servers, with suitably-sized storage partitions. These servers generally do not -run an IMAP daemon, nor do they have conventional mailbox storage.

- -

Your Cyrus IMAP servers synchronise mailbox state to the Cyrus Backup server(s) -using the Cyrus replication (aka sync, aka csync) protocol.

- -

Backup data is stored in two files per user: a data file, containing gzipped -chunks of replication commands; and an SQLite database, which indexes the current -state of the backed up data. User backup files are stored in a hashed subdirectory -of their containing partition.

- -

A twoskip database, backups.db, stores mappings of users to their backup -file locations

- -

Installation

- -

Requirements

- -
    -
  • At least one Cyrus IMAP server, serving and storing user data.
    • -
  • At least one machine which will become the first backup server.
    • -
- -

Cyrus Backups server

- -
    -
  1. Compile cyrus with the --enable-backup configure option and install it.
    2. -
  2. Set up an imapd.conf file for it with the following options (default values shown): -
    -
    backup_db: twoskip
    -
    backup_db_path: {configdirectory}/backups.db
    -
    The backups db contains a mapping of user ids to their backup locations
    - -
    backup_staging_path: {temp_path}/backup
    -
    Directory to use for staging message files during backup operations. The - replication protocol will transfer as many as 1024 messages in a single sync - operation, so, conservatively, this directory needs to contain enough storage - for 1024 * your maximum message size * number of running backupd's, plus some - wiggle room.
    - -
    backup_retention_days: 7
    -
    Number of days for which backup data (messages etc) should be kept within the - backup storage after the corresponding item has been deleted/expunged from - the Cyrus IMAP server.
    - -
    backuppartition-name: /path/to/this/partition
    -
    You need at least one backuppartition-name to store backup data. - These work similarly to regular/archive IMAP partitions, but note that there - is no relationship between backup partition names and regular/archive partition - names. New users will be have their backup storage provisioned according to - the usual partition selection rules.
    - -
    backup_compact_minsize: 0
    -
    The ideal minimum data chunk size within backup files, in kB. The compact tool - will try to combine chunks that are smaller than this into neighbouring chunks. - Larger values tend to yield better compression ratios, but if the data is - corrupted on disk, the entire chunk will become unreadable. Zero turns this - behaviour off.
    - -
    backup_compact_maxsize: 0
    -
    The ideal maximum data chunk size within backup files, in kB. The compact tool - will try to split chunks that are larger than this into multiple smaller - chunks. Zero turns this behaviour off.
    - -
    backup_compact_work_threshold: 1
    -
    The number of chunks within a backup file that must obviously need compaction - before the compact tool will attempt to compact the file. Larger values are - expected to reduce compaction I/O load at the expense of delayed recovery - of storage space.
    -
    3. -
  3. Create a user for authenticating to the backup system, and add it to the - admins setting in imapd.conf
    4. -
  4. Add appropriate sasl_* settings for your authentication method to - imapd.conf
    5. -
  5. Set up a cyrus.conf file for it: -
      -
    • In the SERVICES section, arrange for backupd to run:
      - backupd cmd="backupd" listen="csync" prefork=0
      • -
    • You probably don't need any other SERVICES entries
      • -
    • In the EVENTS section, arrange for compaction to occur at some interval(s)
      - compact cmd="ctl_backups compact -A" at=0400
      • -
    -
    6. -
  6. Start up the server, and use synctest to verify that you can authenticate to backupd
    7. -
- -

Cyrus IMAP servers

- -

Your Cyrus IMAP servers must be running version 3 or later of Cyrus, -and must have been compiled with the --enable-replication configure option. -It does not need to be recompiled with the --enable-backup option.

- -

It's recommended to set up a dedicated replication channel for backups, -so that your backup replication can coexist independently of your other -replication configurations

- -

Add settings to imapd.conf like:

- -
-
sync_log_channels: channel
-
Add a new channel "channel" to whatever was already here. Suggest - calling this "backup"
- -
sync_log: 1
-
Enable sync log if you want rolling replication to the backup server - (and if it wasn't already)
- -
channel_sync_host: backup-server.example.com
-
The host name of your Cyrus Backup server
- -
channel_sync_port: csync
-
The port on which your Cyrus Backup server's backupd process listens
- -
channel_sync_authname: ...
-
channel_sync_password: ...
-
Credentials for authenticating to the Cyrus Backup server
- -
channel_sync_repeat_interval: 1
-
Minimum time in seconds between rolling replication runs. Smaller value - means livelier backups but more network I/O. Larger value reduces I/O.
-
- -

Update cyrus.conf to arrange for replication to occur. If you want to use -rolling replication, add a sync_client invocation to the SERVICES section specifying -(at least) the -r and -n channel options.

- -

If you want to use scheduled replication, add sync_client invocations to the EVENTS -section (or cron, etc), specifying at least the -n channel option -(to use the channel-specific configuration), plus whatever other options you need for -selecting users to back up. See the sync_client manpage for details.

- -

Administration

- -

Storage requirements

- -

It's not really known yet how to predict the storage requirements for a backup -server. Experimentation in dev environment suggests around 20-40% compressed backup -file size relative to the backed up data, depending on compact settings, but this is -with relatively tiny mailboxes and non-pathological data.

- -

The backup staging spool conservatively needs to be large enough to hold an entire -sync's worth of message files at once. Which is your maximum message size * 1024 messages -* the number of backupd processes you're running, plus some wiggle room probably. In -practice it'll probably not hit this limit unless someone is trying to. (Most users, -I suspect, don't have 1024 maximum-sized messages in their account, or don't receive -them all at once anyway.)

- -

Certain invocations of ctl_backups and cyr_backup also require staging spool space, -due to the way replication protocol (and thus backup data) parsing handles messages. -So keep this in mind I suppose.

- -

Initial backups

- -

Once a Cyrus Backup system is configured and running, new users that are created on -the IMAP servers will be backed up seamlessly without administrator intervention.

- -

The very first backup taken of a pre-existing mailbox will be big -- the entire mailbox -in one hit. It's suggested that, when initially provisioning a Cyrus Backup server for -an existing Cyrus IMAP environment, that the sync_client commands be run carefully, for -a small group of mailboxes at a time, until all/most of your mailboxes have been backed up -at least once. Also run ctl_backups compact on the backups, to break up big -chunks, if you wish. Only then should you enable rolling/scheduled replication.

- -

Restoring from backups

- -

The restore tool will restore mailboxes and messages from a specified backup to a -specified destination server. The destination server must be running a -replication-capable imapd or sync_server. The restore tool should be run from the -backup server containing the specified backup.

- -

File locking

- -

All backupd/ctl_backups/cyr_backup operations first obtain a lock on the relevant -backup file. ctl_backups and cyr_backup will try to do this without blocking (unless -told otherwise), whereas backupd will never block.

- -

Moving backup files to different backup partitions

- -

There's no tool for this (yet). To do it manually, stop backupd, copy the files -to the new partition, then use cyr_dbtool to update the user's backups.db -entry to point to the new location. Run ctl_backups verify on both -the new filename (-f mode) and the user's userid (-u mode) -to ensure everything is okay, then restart backupd.

- -

Provoking a backup for a particular user/user group/everyone/etc right now

- -

Just run sync_client by hand with appropriate options (as cyrus user, of -course). See its man page for ways of specifying items to replicate.

- -

What about tape backups?

- -

As long as backupd, ctl_backups and cyr_backup are not currently running (and assuming -no-one's poking around in things otherwise), it's safe to take/restore a filesystem -snapshot of backup partitions. So to schedule, say, a nightly tape dump of your Cyrus -Backup server, make your cron job shut down Cyrus, make the copy, then restart Cyrus.

- -

Meanwhile, your Cyrus IMAP servers are still online and available. Regular backups -will resume once your backupd is running again.

- -

If you can work at a finer granularity than file system, you don't need to shut down -backupd. Just use ctl_backups lock to hold a lock on each backup while you -work with its files, and the rest of the backup system will work around that.

- -

Restoring is more complicated, depending on what you actually need to do: when you -restart the backupd after restoring a filesystem snapshot, the next time your Cyrus IMAP -server replicates to it, the restored backups will be brought up to date. Probably not -what you wanted -- so don't restart backupd until you've done whatever you were doing.

- -

Multiple IMAP servers, one backup server

- -

This is fine, as long as each user being backed up is only being backed up by one -server (or they are otherwise synchronised). If IMAP servers have different ideas -about the state of a user's mailboxes, one of those will be in sync with the backup -server and the other will get a lot of replication failures.

- -

Multiple IMAP servers, multiple backup servers

- -

Make sure your sync_client configuration(s) on each IMAP server knows which users -are being backed up to which backup servers, and selects them appropriately. See the -sync_client man page for options for specifying users, and run it as an -event (rather than rolling).

- -

Or just distribute it at server granularity, such that backup server A serves -IMAP servers A, B and C, and backup server B serves IMAP servers D, E, F, etc.

- -

One IMAP server, multiple backup servers

- -

Configure one channel plus one rolling sync_client per backup server, and your -IMAP server can be more or less simultaneously backed up to multiple backup -destinations.

- -

Reducing load

- -

To reduce load on your client-facing IMAP servers, configure sync log chaining -on their replicas and let those take the load of replicating to the backup servers.

- -

To reduce network traffic, do the same thing, specifically using replicas -that are already co-located with the backup server.

- -

Other setups

- -

The use of the replication protocol and sync_client allows a lot of interesting -configuration possibilities to shake out. Have a rummage in the sync_client -man page for inspiration.

- -

Tools

- -

ctl_backups

- -

This tool is generally for mass operations that require few/fixed arguments -across multiple/all backups

- -

Supported operations:

-
-
compact
-
Reduce backups' disk usage by: -
    -
  • combining small chunks for better gzip compression -- especially important - for hot backups, which produce many tiny chunks
    • -
  • removing deleted content that has passed its retention period
    • -
- Note that the original backup/index files are preserved (with a timestamped filename), - so that in the event of compact bugs/failures, data is not lost. But this also means - that compact actually increases disk usage in practice, until the old files - are cleaned up. This will probably be automated in some way once things are stable - and reliable.
- -
list
-
List known backups. Add more -v's for more detail.
- -
lock
-
Lock a single backup, so you can safely work on it with non-cyrus tools.
- -
reindex
-
Regenerate indexes for backups from their data files. Useful if index becomes - corrupted by some bug, or invalidated by working on data with non-cyrus tools. - Note that the original index file is preserved (with a timestamped filename), - so that in the event of reindex bugs/failures, data is not lost. But this also - means that reindex increases disk usage in practice, until the old files - are cleaned up. This will probably be automated in some way once things are stable - and reliable.
- -
verify
-
Deep verification of backups. Verifies that: -
    -
  • Checksums for each chunk in index match data
    • -
  • Mailbox states are in the chunk that the index says they're in
    • -
  • Mailbox states match indexed states
    • -
  • Messages are in the chunk the index says they're in
    • -
  • Message data checksum matches indexed checksums
    • -
-
-

See the ctl_backups man page for more information.

- -

cyr_backup

- -

This tool is generally for operations on a single mailbox that require multiple -additional arguments

- -

Supported operations

-
-
list [ chunks | mailboxes | messages | all ]
-
Line-per-item listing of information stored in a backup.
- -
show [ chunks | mailboxes | messages ] items...
-
Paragraph-per-item listing of information for specified items. - Chunk items are specified by id, mailboxes by mboxname or uniqueid, messages by guid.
- -
dump [ chunk | message ] item
-
Full dump of one item. chunk dumps the uncompressed content of a chunk (i.e. - a bunch of sync protocol commands). message dumps a raw rfc822 message - (useful for manually restoring until a proper restore tool appears)
-
- -

- -

See the cyr_backup man page for more information.

- -

restore

- -

This tool is for restoring mail from backup files.

- -

Required arguments are a destination server (in ip:port or host:port format), a backup file, -and mboxnames, uniqueids or guids specifying the mailboxes or messages to be restored.

- -

If the target mailbox does not already exist on the destination server, options are available -to preserve the mailbox and message properties as they existed in the backup. This is useful for -rebuilding a lost server from backups, such that client state remains consistent.

- -

If the target mailbox already exists on the destination server, restored messages will be -assigned new, unused uids and will appear to the client as new messages.

- -

See the restore man page for more information.

- - - diff --git a/docsrc/imap/concepts/deployment/databases.rst b/docsrc/imap/concepts/deployment/databases.rst index 654ac8252d4..f3563ac91f0 100644 --- a/docsrc/imap/concepts/deployment/databases.rst +++ b/docsrc/imap/concepts/deployment/databases.rst @@ -34,7 +34,6 @@ One per system: * `PTS cache (ptscache.db)`_ * `STATUS cache (statuscache.db)`_ * `User Access (user_deny.db)`_ -* `Backups (backups.db)`_ * `News database (fetchnews.db)`_ * `Zoneinfo db (zoneinfo.db)`_ @@ -209,17 +208,6 @@ record is as follows:: File type can be: `flat`_ (default), `skiplist`_, `sql`_, or `twoskip`_. -.. _imap-concepts-deployment-db-backups: - -Backups (backups.db) --------------------- - -This database maps userids to the location of their backup files. It only -exists on Cyrus Backup servers (compiled with the `--enable-backup` configure -option). - -File type can be: `twoskip`_ (default), `skiplist`_, `sql`_, or `twoskip`_. - .. _imap-concepts-deployment-db-conversations: Conversations (.conversations) diff --git a/docsrc/imap/developer/thoughts/backup.rst b/docsrc/imap/developer/thoughts/backup.rst deleted file mode 100644 index ec69b65cbc8..00000000000 --- a/docsrc/imap/developer/thoughts/backup.rst +++ /dev/null @@ -1,925 +0,0 @@ -.. _imap-developer-thoughts-backup: - -.. Note: This document was converted from the original by Nic Bernstein - (Onlight). Any formatting mistakes are my fault and not the - original author's. - -Notes for backup implementation -=============================== - -Backup index database (one per user): - -chunk:: - - int id - timestamp ts - int offset - int length - text file_sha1 -> sha1 of (compressed) data prior to this chunk - text data_sha1 -> sha1 of (uncompressed) data contained in this chunk - -mailbox:: - - int id - int last_chunk_id -> chunk that knows the current state - char uniqueid -> unique - char mboxname -> altered by a rename - char mboxtype - int last_uid - int highestmodseq - int recentuid - timestamp recenttime - timestamp last_appenddate - timestamp pop3_last_login - timestamp pop3_show_after - timestamp uidvalidity - char partition - char acl - char options - int sync_crc - int sync_crc_annot - char quotaroot - int xconvmodseq - char annotations - timestamp deleted -> time that it was unmailbox'd, or NULL if still alive - -message:: - - int id - char guid - char partition -> this is used to set the spool directory for the temp file - we might not need it - int chunk_id - int offset -> offset within chunk of dlist containing this message - int size -> size of this message (n.b. not length of dlist) - -mailbox_message:: - - int mailbox_id - int message_id - int last_chunk_id -> chunk that has a RECORD in a MAILBOX for this - int uid - int modseq - timestamp last_updated - char flags - timestamp internaldate - int size - char annotations - timestamp expunged -> time that it was expunged, or NULL if still alive - -subscription:: - - int last_chunk_id -> chunk that knows the current state - char mboxname -> no linkage to mailbox table, users can be sub'd to nonexistent - timestamp unsubscribed -> time that it was unsubscribed, or NULL if still alive - -seen:: - - int last_chunk_id -> chunk that knows the current state - char uniqueid -> mailbox (not necessarily ours) this applies to - timestamp lastread - int lastuid - timestamp lastchange - char seenuids -> a uid sequence encoded as a string - -sieve:: - - int chunk_id - timestamp last_update - char filename - char guid - int offset -> offset within chunk of the dlist containing this script - timestamp deleted -> time that it was deleted, or NULL if still alive - - -sieve scripts and messages are both identified by a GUID -but APPLY SIEVE doesn't take a GUID, it seems to be generated locally? -the GUID in the response to APPLY SIEVE is generated in the process of -reading the script from disk (sync_sieve_list_generate) - -can't activate scripts because only bytecode files are activated, but -we neither receive bytecode files over sync protocol nor do we compile -them ourselves. - -possibly reduce index size by breaking deleted/expunged values into their -own tables, such that we only store a deleted value for things that are -actually deleted. use left join + is null to find undeleted content - -messages --------- - -APPLY MESSAGE is a list of messages, not necessarily only one message. -Actually, it's a list of messages for potentially multiple users, but we avoid -this by rejecting GET MESSAGES requests that span multiple users (so that -sync_client retries at USER level, and so we only see APPLY MESSAGE requests -for a single user). - -Cheap first implementation is to index the start/end of the entire APPLY -MESSAGE command identically for each message within it, and at restore time -we grab that chunk and loop over it looking for the correct guid. - -Ideal implementation would be to index the offset and length of each message -exactly (even excluding the dlist wrapper), but this is rather complicated -by the dlist API. - -For now, we just index the offset of the dlist entry for the message, -and we can parse the pure message data back out later from that, when -we need to. Slightly less efficient on reads, but works->good->fast. We -need to loop over the entries in the MESSAGE dlist to find the one with the -desired GUID. - -The indexed length needs to be the length of the message, not the length of the -dlist wrapper, because we need to know this cheaply to supply RECORDs in -MAILBOX responses. - -renames -------- - -APPLY RENAME %(OLDMBOXNAME old NEWMBOXNAME new PARTITION p UIDVALIDITY 123) - -We identify mboxes by uniqueid, so when we start seeing sync data for the same -uniqueid with a new mboxname we just transparently update it anyway, without -needing to handle the APPLY RENAME. Not sure if this is a problem... Do we -need to record an mbox's previous names somehow? - -I think it's possible to use this to rename a USER though, something like: - -APPLY RENAME %(OLDMBOXNAME example.com!user.smithj NEWMBOXNAME example.com!user.jsmith ...) - --- in which case, without special handling of the RENAME command itself, there -will be a backup for the old user that ends with the RENAME, and a backup of -the new user that (probably) duplicates everything again (except for stuff -that's been expunged). - -And if someone else gets given the original name, like - -APPLY RENAME %(OLDMBOXNAME example.com!user.samantha-mithj NEWMBOXNAME example.com!user.smithj ...) - -Then anything that was expunged from the original user but still available in -backup disappears? Or the two backups get conflated, and samantha can -"restore" the original smithj's old mail? - -Uggh. - -if there's a mailboxes database pointing to the backup files, then the backup -file names don't need to be based on the userid, they could e.g. be based on -the user's inbox's uniqueid. this would make it easier to deal with user -renames because the backup filename wouldn't need to change. but this depends -on the uniqueid(s) in question being present on most areas of the sync -protocol, otherwise when starting a backup of a brand new user we won't be -able to tell where to store it. workaround in the meantime could be to make -some kind of backup id from the mailboxes database, and base the filename on -this. - -actually, using "some kind of backup id from the mailboxes database" is probably -the best solution. otherwise the lock complexity of renaming a user while making -sure their new backup filename doesn't already exist is frightful. - -maybe do something with mkstemp()? - -furthermore: what if a mailbox is moved from one user to another? like: - -APPLY RENAME %(OLD... example.com!user.foo.something NEW... example.com!user.bar.something ...) - -when a different-uid rename IS a rename of a user (and not just a folder -being moved to a different user), what does it look like? -* does it do a single APPLY RENAME for the user, and expect their folders to shake out of that? -* does it do an APPLY RENAME for each of their folders? - -in the latter case, we need to append each of those RENAMEs to the old backup -so they can take effect correctly, and THEN rename the backup file itself. but -how to tell when the appends are finished? - -how can we tell the difference between folder(s) moved to a different user vs -user has been renamed? - -there is a setting: 'allowusermoves: 0' which, when enabled, allows users to -be renamed via IMAP rename/xfer commands. but the default is that this is -disabled. we could initially require this to be disabled while using backups... - -not sure what the workflow looks like for renaming a user if this is not enabled. - -not sure what the sync flow looks like in either case. - -looking at sync_apply_rename and mboxlist_renamemailbox, it seems like we'll -see an APPLY RENAME for each affected mbox when a recursive rename is occurring. - -there doesn't seem to be anything preventing user/a/foo -> user/b/foo in the -general (non-INBOX) case. - -renames might be a little easier to handle if the index replicated the mailbox -hierarchy rather than just being a flat structure. though this adds complexity -wrt hiersep handling. something like: - -mailbox: - - mboxname - # just the name of this mbox - - parent_id - # fk to parent mailbox - - full_mboxname - # cached value, parent.full_mboxname + mboxname - -locking -------- - -just use a normal flock/fcntl lock on the data file and only open the index -if that lock succeeded - -* backup: needs to append foo and update foo.index -* reindex: only needs to read foo, but needs a write lock to prevent - writes while it does so. needs to write to (replace) foo.index -* compact: needs to re-write foo and foo.index -* restore: needs to read - - -verifying index ---------------- - -how to tell whether the .index file is the correct one for the backup data it -ostensibly represents? - -one way to do this would be to have backup_index_end() store a checksum of -the corresponding data contents in the index. - -when opening a backup, verify this checksum against the data, and refuse to -load the index if it doesn't match. - -- sha1sum of (compressed) contents of file prior to each chunk - -how to tell whether the chunk data is any good? store a checksum of the chunk -contents along with the rest of the chunk index - -- sha1sum of (uncompressed) contents of each chunk - - -mailboxes database ------------------- - -bron reckons use twoskip for this -userid -> backup_filename - -lib/cyrusdb module implements this, look into that - -look at conversations db code to see how to use it - -need a tool: -* given a user, show their backup filename -* dump/undump -* rebuild based on files discovered in backup directory - -where does this fit into the locking scheme? - - -reindex -------- - -* convert user mailbox name to backup name -* complain if there's no backup data file? -* lock, rename .index to .index.old, init new .index -* foreach file chunk: -* timestamp is from first line in chunk -* complain if timestamp has gone backwards? -* index records from chunk -* unlock -* clean up .index.old - -on error: -* discard partial new index -* restore .index.old -* bail out - - -backupd -------- - -cmdloop: -* (periodic cleanup) -* read command, determine backup name -* already holding lock ? bump timestamp : obtain lock -* write data to gzname, flush immediately -* index data - -periodic cleanup: -* check timestamp of each held lock -* if stale (define: stale?), release -* FIXME if we've appended more than the chunk size we would compact to, release - -sync restart: -* release each held lock - -exit: -* release each held lock - -need a "backup_index_abort" to complete the backup_index_start/end set. -_start should create a transaction, _end should commit it, and _abort should -roll it back. then, if backupd fails to write to the gzip file for some -reason, the (now invalid) index info we added can be discarded too. - -flushing immediately on write results in poor gzip compression, but for -incremental backups that's not a problem. when the compact process hits the -file it will recompress the data more efficiently. - - -questions ---------- -* what does it look like when uidvalidity changes? - - -restore -------- - -restoration is effectively a reverse-direction replication (replicating TO master), -which means we can't necessarily supply things like uid, modseq, etc without racing -against normal message arrivals. so instead we add an extra command to the protocol -to restore a message to a folder but let the destination determine the tasty bits. - -protocol flow looks something like: - -c: APPLY RESERVE ... # as usual -s: * MISSING (foo bar) -s: OK -c: APPLY MESSAGE ... # as usual -s: OK -c: RESTORE MAILBOX ... # new sync proto command -s: OK - -we introduce a new command, RESTORE MAILBOX, which is similar to the existing -APPLY MAILBOX. it specifies, for a mailbox, the mailbox state plus the message -records relevant to the restore. - -the imapd/sync_server receiving the RESTORE command creates the mailbox if necessary, -and then adds the message records to it as new records (i.e. generating new uid etc). -this will end up generating new events in the backup channel's sync log, and then the -messages will be backed up again with their new uids, etc. additional wire transfer -of message data should be avoided by keeping the same guid. - -if the mailbox already exists but its uniqueid does not match the one from the backup, -then what? this probably means user has deleted folder and contents, then made new -folder with same name. so it's probably v common for mailbox uniqueid to not match -like this. so we don't care about special handling for this case. just add any -messages that aren't already there. - -if the mailbox doesn't already exist on the destination (e.g. if rebuilding a server -from backups) then it's safe and good to reuse uidvalidity, uniqueid, uid, modseq etc, -such that connecting clients can preserve their state. so the imapd/sync_server -receiving the restore request accepts these fields as optional, but only preserves -them if it's safe to do so. - -* restore: sbin program for selecting and restoring messages - -restore command needs options: -+ whether or not to trim deletedprefix off mailbox names to be restored -+ whether or not to restore uniqueid, highestmodseq, uid and so on -+ whether or not to limit to/exclude expunged messages -+ whether or not to restore sub-mailboxes -+ sync_client-like options (servername, local_only, partition, ...) -+ user/mailbox/backup file(s) to restore from -+ mailbox to restore to (override location in backup) -+ override acl? - -can we heuristically determine whether an argument is an mboxname, uniqueid or guid? - => libuuid uniqueid is 36 bytes of hyphen (at fixed positions) and hex digits - => non-libuuid uniqueid is 24 bytes of hex digits - => mboxname usually contains at least one . somewhere - => guid is 40 bytes of hex digits - -usage: - restore [options] server [mode] backup [mboxname | uniqueid | guid]... - -options: - -A acl # apply specified acl to restored mailboxes - -C alt_config # alternate config file - -D # don't trim deletedprefix before restoring - -F input-file # read mailboxes/messages from file rather than argv - -L # local mailbox operations only (no mupdate) - -M mboxname # restore messages to specified mailbox - -P partition # restore mailboxes to specified partition - -U # try to preserve uniqueid, uid, modseq, etc - -X # don't restore expunged messages - -a # try to restore all mailboxes in backup - -n # calculate work required but don't perform restoration - -r # recurse into submailboxes - -v # verbose - -w seconds # wait before starting (useful for attaching a debugger) - -x # only restore expunged messages (not sure if useful?) - -z # require compression (abort if compression unavailable) - -mode: - -f # specified backup interpreted as filename - -m # specified backup interpreted as mboxname - -u # specified backup interpreted as userid (default) - - -compact --------- - -# finding messages that are to be kept (either exist as unexpunged somewhere, -# or exist as expunged but more recently than threshold) -# (to get unique rows, add "distinct" and remove mm.expunged from fields) -sqlite> select m.*, mm.expunged from message as m join mailbox_message as mm on m.id = mm.message_id and (mm.expunged is null or mm.expunged > 1437709300); -id|guid|partition|chunk_id|offset|length|expunged -1|1c7cca361502dfed2d918da97e506f1c1e97dfbe|default|1|458|2159| -1|1c7cca361502dfed2d918da97e506f1c1e97dfbe|default|1|458|2159|1446179047 -1|1c7cca361502dfed2d918da97e506f1c1e97dfbe|default|1|458|2159|1446179047 - -# finding chunks that are still needed (due to containing last state -# of mailbox or mailbox_message, or containing a message) -sqlite> select * from chunk where id in (select last_chunk_id from mailbox where deleted is null or deleted > 1437709300 union select last_chunk_id from mailbox_message where expunged is null or expunged > 1437709300 union select chunk_id from message as m join mailbox_message as mm on m.id = mm.message_id and (mm.expunged is null or mm.expunged > 1437709300)); -id|timestamp|offset|length|file_sha1|data_sha1 -1|1437709276|0|3397|da39a3ee5e6b4b0d3255bfef95601890afd80709|6836d0110252d08a0656c14c2d2d314124755491 -3|1437709355|1977|2129|fee183c329c011ead7757f59182116500776eaaf|a5677cfa1f5f7b627763652f4bb9b99f5970748c -4|1437709425|2746|1719|3d9f02135bf964ff0b6a917921b862c3420e48f0|7b64ec321457715ee61fe238f178f5d72adaef64 -5|1437709508|3589|2890|0cee599b1573110fee428f8323690cbcb9589661|90d104346ef3cba9e419461dd26045035f4cba02 - -remember: a single APPLY MESSAGE line can contain many messages! - -thoughts: - -* need a heuristic for quickly determining whether a backup needs to be compacted - - * sum(chunks to discard, chunks to combine, chunks to split) > threshold - * can we detect chunks that are going to significantly reduce in size as result of discarding individual lines? - -* "quick" vs "full" compaction - -settings: - -* backup retention period -* chunk combination size (byte length or elapsed time) - -combining chunks: -* size threshold below which adjacent chunks can be joined -* size threshold above which chunks should be split -* duration threshold below which adjacent chunks can be joined -* duration threshold above which chunks should be split -backup_min_chunk_size: 0 for no minimum -backup_max_chunk_size: 0 for no maximum -backup_min_chunk_duration: 0 for no minimum -backup_max_chunk_duration: 0 for no maximum -priority: size or duration?? - -data we absolutely need to keep: - -* the most recent APPLY MAILBOX for each mailbox we're keeping (mailbox state) -* the APPLY MAILBOX containing the most recent RECORD for each message we're keeping (record state) -* the APPLY MESSAGE for each message we're keeping (message data) - -data that we should practically keep: - -* all APPLY MAILBOXes for a given mailbox from the chunk identified as its last -* all APPLY MAILBOXes containing a RECORD for a given message from the chunk identified as its last -* the APPLY MESSAGE for each message we're keeping - -four kinds of compaction (probably at least two simultaneously): - -* removing unused chunks -* combining adjacent chunks into a single chunk (for better gz compression) -* removing unused message lines from within a chunk (important after combining) -* removing unused messages from within a message line - -"unused messages" - messages for which all records have been expunged for longer - than the retention period - -"unused chunks" - chunks which contain only unused messages - -algorithm: - -* open (and lock) backup and backup.new (or bail out) -* use backup index to identify chunks we still need -* create a chunk in backup.new -* foreach chunk we still need: -* foreach line in the chunk: -* next line if we don't need to keep it -* create new line -* foreach message in line: -* if we still need the message, or if we're not doing message granularity -* add the message to the new line -* write and index tmp line to backup.new -* if the new chunk is big enough, or if we're not combining -* end chunk and start a new one -* end the new chunk -* rename backup->backup.old, backup.new->backup -* close (and unlock) backup.old and backup - - -command line locking utility ----------------------------- - -command line utility to lock a backup (for e.g. safely poking around in the -.index on a live system). - -example failure: -$ctl_backups lock -f /path/to/backup -* Trying to obtain lock on /path/to/backup... -NO some error - - -example success: -$ctl_backups lock -f /path/to/backup -* Trying to obtain lock on /path/to/backup... -[potentially a delay here if we need to wait for another process to release the lock] -OK locked -[waits for its stdin to close, then unlocks and exits] - -if you need to rummage around in backup.index, run this program in another -shell, do your work, then ^D it when you're finished. - -you could also call this from e.g. perl over a bidirectional pipe - wait to -read "OK locked", then you've got your lock. close the pipe to unlock when -you're finished working. if you don't read "OK locked" before the pipe closes -then something went wrong and you didn't get the lock. - -specify backups by -f filename, -m mailbox, -u userid -default run mode as above --s to fork an sqlite of the index (and unlock when it exits) --x to fork a command of your choosing (and unlock when it exits) - - -reconstruct ------------ - -rebuilding backups.db from on disk files - -scan each backup partition for backup files: - * skip timestamped files (i.e. backups from compact/reindex) - * skip .old files (old backups from reindex) - * .index files => skip??? - * skip unreadable files - * skip empty files - * skip directories etc - -what's the correct procedure for repopulating a cyrus database? -keep copy of the previous (presumably broken) one? - -trim off mkstemp suffix (if any) to find userid -can we use a recognisable character to delimit the mkstemp suffix? - -what if there's multiple backup files for a given userid? precedence? - -verify found backups before recording. reindex? - -locking? what if something has a filename and does stuff with it while -reconstruct runs? - -backupd always uses db for opens, so as long as reconstruct keeps the db -locked while it works, the db won't clash. but backupd might have backups -still open from before reconstruct started, which it will write to quite -happily, even though reconstruct might decide that some other file is the -correct one for that user... - -a backup server would generally be used only for backups, and sync_client -is quite resilient when the destination isn't there, so it's actually -no problem to just shut down cyrus while reconstruct runs. no outage to -user-facing services, just maybe some sync backlog to catch up on once -cyrus is restarted. - - -ctl_backups -------------- - -sbin tool for mass backup/index/database operations - -needs: - * rebuild backups.db from disk contents - * list backups/info - * rename a backup - * delete a backup - * verify a backup (check all sha1's, not just most recent) - -not sure if these should be included, or separate tools: - * reindex a backup (or more) - * compact a backup (or more) - * lock a backup - * some sort of rolling compaction? - -usage: - ctl_backups [options] reconstruct # reconstruct backups.db from disk files - ctl_backups [options] list [list_opts] [[mode] backup...] # list backup info for given/all users - ctl_backups [options] move new_fname [mode] backup # rename a backup (think about this more) - ctl_backups [options] delete [mode] backup # delete a backup - ctl_backups [options] verify [mode] backup... # verify specified backups - ctl_backups [options] reindex [mode] backup... # reindex specified backups - ctl_backups [options] compact [mode] backup... # compact specified backups - ctl_backups [options] lock [lock_opts] [mode] backup # lock specified backup - -options: - -C alt_config # alternate config file - -F # force (run command even if not needed) - -S # stop on error - -v # verbose - -w # wait for locks (i.e. don't skip locked backups) - -mode: - -A # all known backups (not valid for single backup commands) - -D # specified backups interpreted as domains (nvfsbc) - -P # specified backups interpreted as userid prefixes (nvfsbc) - -f # specified backups interpreted as filenames - -m # specified backups interpreted as mboxnames - -u # specified backups interpreted as userids (default) - -lock_opts: - -c # exclusively create backup - -s # lock backup and open index in sqlite - -x cmd # lock backup and execute cmd - -p # lock backup and wait for eof on stdin (default) - -list_opts: - -t [hours] # "stale" (no update in hours) backups only (default: 24) - - -cyr_backup ----------- - -sbin tool for inspecting backups - -needs: - * better name? - * list stuff - * show stuff - * dump stuff - * restore? - -* should lock/move/delete (single backup commands) from ctl_backups be moved here? - -usage: - cyr_backup [options] [mode] backup list [all | chunks | mailboxes | messages]... - cyr_backup [options] [mode] backup show chunks [id...] - cyr_backup [options] [mode] backup show messages [guid...] - cyr_backup [options] [mode] backup show mailboxes [mboxname | uniqueid]... - cyr_backup [options] [mode] backup dump [dump_opts] chunk id - cyr_backup [options] [mode] backup dump [dump_opts] message guid - cyr_backup [options] [mode] backup json [chunks | mailboxes | messages]... - -options: - -C alt_config # alternate config file - -v # verbose - -mode: - -f # backup interpreted as filename - -m # backup interpreted as mboxname - -u # backup interpreted as userid (default) - -commands: - list: table of contents, one per line - show: indexed details of listed items, one per paragraph, detail per line - dump: relevant contents from backup stream - json: indexed details of listed items in json format - -dump options: - -o filename # dump to named file instead of stdout - - -partitions ----------- - -not enough information in sync protocol to handle partitions easily? - -we know what the partition is when we do an APPLY operation (mailbox, message, -etc), but the initial GET operations don't include it. so we need to already -know where the appropriate backup is partitioned in order to find the backup -file in order to look inside it to respond to the GET request - -if we have a mailboxes database (indexed by mboxname, uniqueid and userid) then -maybe that would make it feasible? if it's not in the mailboxes database then -we don't have a backup for it yet, so we respond accordingly, and get sent -enough information to create it. - -does that mean the backup api needs to take an mbname on open, and it handles -the job of looking it up in the mailboxes database to find the appropriate -thing to open? - -can we use sqlite for such a database, or is the load on it going to be too -heavy? locking? we have lots of database formats up our sleeves here, so -even though we use sqlite for the backup index there isn't any particular -reason we're beholden to it for the mailboxes db too - -if we have a mailboxes db then we need a reconstruct tool for that, too - -what if we support multiple backup partitions, but don't expect these -to necessarily correspond with mailbox partitions. they're just for spreading -disk usage around. - -* when creating a backup for a previously-unseen user we'd pick a random - partition to put them on -* ctl_backups would need a command to move an existing backup to a - given partition -* ctl_backups would need a command to pre-create a user backup on a - given partition for initial distribution -* instead of "backup_data_path" setting, have one-or-more - "backuppartition-" settings, ala partition- and friends - -see imap/partlist.[ch] for partition list management stuff. it's complicated -and doesn't have a test suite, so maybe save this implementation until needed. - -but... maybe rename backup_data_path to backuppartition-default in the meantime, -so that when we do add this it's not a complicated reconfig to update? - -partlist_local_select (and lazy-loaded partlist_local_init) are where the -mailbox partitions come from (see also mboxlist_create_partition), do something -similar for backup partitions - - -data corruption ---------------- - -backups.db: - * can be reconstructed from on disk files at any time - * how to detect corruption? does cyrus_db detect/repair on its own? - -backup indexes: - * can be reindexed at any time from backup data - * how to detect corruption? assume sqlite will notice, complain? - -backup data: - * what's zlib's failure mode? do we lose the entire chunk or just the corrupt bit? - * verify will notice sha1sum mismatches - * dlist format will reject some kinds of corruption (but not all) - * reindex: should skip unparseable dlist lines - * message data has its own checksums (guid) - * reindex: should skip messages that don't match their own checksums - * compact: "full" compact will only keep useful data according to index - * backupd: will sync anything that's in user mailbox but not in backup index - -i think this means that if a message or mailbox state becomes corrupted in -the backup data file, and it still exists in the user's real mailbox, you -recover from the corruption by reindexing and then letting the sync process -copy the missing data back in again. and you can tidy up the data file by -running a compact over it. - -you detect data corruption in most recent chunk reactively as soon as the -backup system needs to open it again (quick verify on open) - -you detect data corruption in older chunks reactively by trying to restore from -it. may be too late: if a message needs restoring it's because user mailbox no -longer has it - -you detect data corruption preemptively by running the verify tool over it. -recommend scheduling this in EVENTS/cron? - -if data corruption occurs in message that's no longer in user's mailbox, that -message is lost. it was going to be deleted from the backup after $retention -period anyway (by compact), but if it needs restoring in the meantime, sorry - - -installation instructions -------------------------- - -(obviously, most of this won't work at this point, because the code doesn't -exist. but this is, approximately, where things are heading.) - -on your backup server: - * compile with --enable-backup configure option and install - * imapd.conf: - backuppartition-default: /var/spool/backup # FIXME better example - backup_db: twoskip - backup_db_path: /var/imap/backups.db - backup_staging_path: /var/spool/backup - backup_retention_days: 7 - * cyrus.conf SERVICES: - backupd cmd="backupd" listen="csync" prefork=0 - (remove other services, most likely) - (should i create a master/conf/backup.conf example file?) - * cyrus.conf EVENTS: - compact cmd="ctl_backups compact -A" at=0400 - * start server as usual - * do i want a special port for backupd? - -on your imap server: - * imapd.conf: - sync_log_channels: backup - sync_log: 1 - backup_sync_host: backup-server.example.com - backup_sync_port: csync - backup_sync_authname: ... - backup_sync_password: ... - backup_sync_repeat_interval: ... # seconds, smaller value = livelier backups but more i/o - backup_sync_shutdown_file: .... - * cyrus.conf STARTUP: - backup_sync cmd="sync_client -r -n backup" - * cyrus.conf SERVICES: - restored cmd="restored" [...] - * start/restart master - -files and such: - {configdirectory}/backups.db - database mapping userids to backup locations - {backuppartition-name}//_XXXXXX - backup data stream for userid - {backuppartition-name}//_XXXXXX.index - index into userid's backup data stream - -do i want rhost in the path? - * protects from issue if multiple servers are trying to back up their own version of same user - (though this is its own problem that the backup system shouldn't have to compensate for) - * but makes location of undifferentiated user unpredictable - * so probably not, actually - - -chatting about implementation 20/10 ------------------------------------ - -:: - - 09:54 @elliefm - here's a fun sync question - APPLY MESSAGE provides a list of messages - can a single APPLY MESSAGE contain messages for multiple mailboxes and/or users? - my first hunch is that it doesn't cross users, since the broadest granularity for a single sync run is USER - 10:06 kmurchison - We'd have to check with Bron, but I *think* messages can cross mailboxes for a single user - 10:06 @brong - yes - APPLY MESSAGE just adds it to the reserve list - 10:07 @elliefm - nah apply message uploads the message, APPLY RESERVE adds it to the reserve list :P - 10:07 @brong - same same - APPLY RESERVE copies it from a local mailbox - APPLY MESSAGE uploads it - 10:07 @elliefm - yep - 10:07 @brong - they both wind up in the reserve list - 10:07 @elliefm - ahh i see what you mean, gotcha - 10:07 @brong - until you send a RESTART - ideally you want it reserve in the same partition, but it will copy the message over if it's not on the same partition - there's no restriction on which mailbox it came from/went to - good for user renames, and good for an append to a bunch of mailboxes in different users / shared space all at once - (which LMTP can do) - 10:10 @elliefm - i can handle the case where a single APPLY MESSAGE contains messages for multiple mailboxes belonging to the same user - but i'm in trouble if a single APPLY MESSAGE can contain messages belonging to different users - 10:14 @brong - @elliefm: why? - 10:14 @brong - you don't have to keep them if they aren't used - 10:15 @elliefm - for backups - when i see the apply, i need to know which user's backup to add it to. that's easy enough if it doesn't cross users but gets mega fiddly if it does - i'm poking around in sync client to see if it's likely to be an issue or not - 11:00 @brong_ - @elliefm: I would stage it, and add it to users as it gets refcounted in by an index file - 11:07 @elliefm - that's pretty much what we do for ordinary sync and delivery stuff yeah? - 11:08 @brong_ - yep - and it's what the backup thing does - 11:09 @elliefm - i'm pretty sure that APPLY RESERVE and APPLY MESSAGE don't give a damn about users, they're just "here's every message you might not have already had since last time we spoke" and it lets the APPLY MAILBOX work out where to attach them later - 11:09 @brong_ - yep - 11:09 @elliefm - so yeah, i'll need to do something here - i've been working so far on the idea that a single user's backup consists of 1) an append-only gzip stream of the sync protocol chat that built it, and 2) an index that tracks current state of mailboxes, and offsets within (1) of message data - that gets us good compression (file per user, not file per message), and if the index gets corrupted or lost, it's rebuildable purely from (1), it doesn't need a live copy of the original mailbox - 11:12 @brong - yep, that all works - 11:12 @elliefm - (so if you lose your imap server, you're not unable to rebuild a broken index on the backup) - 11:13 @brong - it's easy enough to require the sync protocol stream to only contain messages per user - though "apply reserve" is messy - because you need to return "yes, I have that message" - 11:13 @elliefm - with that implementation i can't (easily) keep user.a's messages from not existing in user.b's data stream (though they won't be indexed) - 11:14 @brong - I'm not too adverse to the idea of just unpacking each message as it comes off the wire into a temporary directory - 11:14 @elliefm - (because at the time i'm receiving the sync data i don't know which it needs to go in, so if they come in in the same reserve i'd need to append them to both data streams) - which isn't a huge problem, just… irks me a bit - 11:14 @brong - and then reading the indexes as they come in, checking against the state DB to see if we already have them, and streaming them into the gzip if they aren't there yet - what we can do is something like the current format, where files go into a tar - 11:16 @elliefm - i guess the fiddly bit there is that there's one more moving part to keep synchronised across failure states - a backup for a single user becomes 1) data stream + 2) any messages that were uploaded but not yet added to a mailbox + 3) index (which doesn't know what to do with (2)) - which in the general case is fine, the next sync will update the mailboxes, which will push (2) into (1) and index it nicely, and on we go - but it's just a little bit more mess if there's a failure that you need to recover from between those states — it's no longer a simple case of "it's in the backup and we know everything about it" or "it doesn't exist", there's a third case of "well we might have the data but don't really know what to do with it" - the other fiddly bit is that the process of appending to the data stream is suddenly in the business of crafting output rather than simply dumping what it gets, which isn't really burdensome, but it is one more little crack for bugs to crawl into - i guess in terms of sync protocol, one thing i could do on my end is identify apply operations that seem to contain multiple users' data, and just return an error on those. the sync client on the other end will promote them until they're eventually user syncs, which i think are always user granularity - 11:50 @elliefm - i think for now, first stage implementation will be to stream the reserve/message commands in full to every user backup they might apply to. and optimising that down so that each stream only contains messages belonging to that user can be a future optimisation - - -todo list ---------- - -* clean up error handling -* perl tool to anonymise sync proto talk -* verification step to check entire data stream for errors (even chunks that aren't indexed) -* prot_fill_cb: extra argument to pass back an error string to prot_fill -* ctl_backups verify: set level -* backupd: don't block on locked backups, return mailbox locked -- but sync_client doesn't handle this -* test multiple backup partitions -* configure: error if backups requested and we don't have zlib -* valgrind -* finish reconstruct -* compact: split before append? - -compact implementation steps: - 1 remove unused chunks, keep everything else as is - 2 join adjacent chunks if small enough, split large chunks - 3 parse/rebuild message lines - 4 discard unused mailbox lines diff --git a/docsrc/imap/reference/admin.rst b/docsrc/imap/reference/admin.rst index 07da7cb49c0..b361fb67c21 100644 --- a/docsrc/imap/reference/admin.rst +++ b/docsrc/imap/reference/admin.rst @@ -23,7 +23,6 @@ Management admin/access-control admin/quotas admin/sieve - admin/backups admin/nntp admin/protlayer admin/sop diff --git a/docsrc/imap/reference/admin/backups.rst b/docsrc/imap/reference/admin/backups.rst deleted file mode 100644 index fad25d4ff8e..00000000000 --- a/docsrc/imap/reference/admin/backups.rst +++ /dev/null @@ -1,498 +0,0 @@ -.. _cyrus-backups: - -============= -Cyrus Backups -============= - -.. warning:: - This experimental feature is no longer under active development. It - is considered deprecated as of 3.10, and will be removed entirely in - a future version. - -.. contents:: - -Introduction -======================== - -Cyrus Backups are a replication-based backup service for Cyrus IMAP servers. -This is a deprecated experimental feature. - -This document is intended to be a guide to the configuration and -administration of Cyrus Backups. - -This document is a work in progress and at this point is incomplete. - -This document assumes that you are familiar with compiling, installing, -configuring and maintaining Cyrus IMAP servers generally, and will only discuss -backup-related portions in detail. - -This document assumes a passing familiarity with -:ref:`Cyrus Replication `. - -Limitations -=========== - -.. note:: - Cyrus Backups are experimental, incomplete, and deprecated as of 3.10. - -The following components exist and appear to work: - -- backupd, and therefore inbound replication -- autovivification of backup storage for new users, with automatic partition - selection -- rebuilding of backup indexes from backup data files -- compaction of backup files to remove stale data and combine chunks for - better compression -- deep verification of backup file/index state -- examination of backup data -- locking tool, for safe non-cyrus operations on backup files -- recovery of data back into a Cyrus IMAP server - -The following components don't yet exist in a workable state -- these tasks -must be massaged through manually (with care): - -- reconstruct of backups.db from backup files - -The following types of information are currently backed up and recoverable - -- mailbox state and annotations -- messages -- mailbox message records, flags, and annotations - -The following types of information are currently backed up, but tools to -recover them don't yet exist: - -- sieve scripts (but not active script status) -- subscriptions -- seen data - -The following types of information are not currently backed up - -- quota information - -Architecture -============ - -.. note:: - Cyrus Backups are experimental, incomplete, and deprecated as of 3.10. - -Cyrus Backups are designed to run on one or more standalone, dedicated backup -servers, with suitably-sized storage partitions. These servers generally do -not run an IMAP daemon, nor do they have conventional mailbox storage. - -Your Cyrus IMAP servers synchronise mailbox state to the Cyrus Backup server(s) -using the Cyrus replication (aka sync, aka csync) protocol. - -Backup data is stored in two files per user: a data file, containing gzipped -chunks of replication commands; and an SQLite database, which indexes the -current state of the backed up data. User backup files are stored in a hashed -subdirectory of their containing partition. - -A twoskip database, backups.db, stores mappings of users to their backup file -locations - -Installation -============ - -.. note:: - Cyrus Backups are experimental, incomplete, and deprecated as of 3.10. - -Requirements ------------- - -- At least one Cyrus IMAP server, serving and storing user data. -- At least one machine which will become the first backup server. - -Cyrus Backups server --------------------- - -#. Compile cyrus with the ``--enable-backup`` configure option and install it. -#. Set up an :cyrusman:`imapd.conf(5)` file for it with the following options - (default values shown): - - backup\_db: twoskip - The twoskip database format is recommended for backups.db - backup\_db\_path: {configdirectory}/backups.db - The backups db contains a mapping of user ids to their backup locations - backup\_staging\_path: {temp\_path}/backup - Directory to use for staging message files during backup operations. - The replication protocol will transfer as many as 1024 messages in a - single sync operation, so, conservatively, this directory needs to - contain enough storage for 1024 \* your maximum message size \* number - of running backupd's, plus some wiggle room. - backup\_retention\_days: 7 - Number of days for which backup data (messages etc) should be kept - within the backup storage after the corresponding item has been - deleted/expunged from the Cyrus IMAP server. - backuppartition-\ *name*: /path/to/this/partition - You need at least one backuppartition-\ *name* to store backup data. - These work similarly to regular/archive IMAP partitions, but note that - there is no relationship between backup partition names and - regular/archive partition names. New users will be have their backup - storage provisioned according to the usual partition selection rules. - backup\_compact\_minsize: 0 - The ideal minimum data chunk size within backup files, in kB. The - compact tool will try to combine chunks that are smaller than this - into neighbouring chunks. Larger values tend to yield better - compression ratios, but if the data is corrupted on disk, the entire - chunk will become unreadable. Zero turns this behaviour off. - backup\_compact\_maxsize: 0 - The ideal maximum data chunk size within backup files, in kB. The - compact tool will try to split chunks that are larger than this into - multiple smaller chunks. Zero turns this behaviour off. - backup\_compact\_work\_threshold: 1 - The number of chunks within a backup file that must obviously need - compaction before the compact tool will attempt to compact the file. - Larger values are expected to reduce compaction I/O load at the expense - of delayed recovery of storage space. - -#. Create a user for authenticating to the backup system, and add it to the - ``admins`` setting in :cyrusman:`imapd.conf(5)` -#. Add appropriate ``sasl_*`` settings for your authentication method to - :cyrusman:`imapd.conf(5)` -#. Set up a :cyrusman:`cyrus.conf(5)` file for it:: - - START { - # this is required - recover cmd="ctl_cyrusdb -r" - } - - SERVICES { - # backupd is probably the only service entry your backup server needs - backupd cmd="backupd" listen="csync" prefork=0 - } - - EVENTS { - # this is required - checkpoint cmd="ctl_cyrusdb -c" period=30 - - # arrange for compact to run at some interval - compact cmd="ctl_backups compact -A" at=0400 - } - -#. Start up the server, and use :cyrusman:`synctest(1)` to verify that you can - authenticate to backupd - -Cyrus IMAP servers ------------------- - -Your Cyrus IMAP servers must be running version 3 or later of Cyrus, and must -have been compiled with the ``--enable-replication`` configure option. It does -*not* need to be recompiled with the ``--enable-backup`` option. - -It's recommended to set up a dedicated replication channel for backups, so that -your backup replication can coexist independently of your other replication -configurations - -Add settings to :cyrusman:`imapd.conf(5)` like (default values shown): - -*channel*\ \_sync\_host: backup-server.example.com - The host name of your Cyrus Backup server -*channel*\ \_sync\_port: csync - The port on which your Cyrus Backup server's backupd process listens -*channel*\ \_sync\_authname: ... - Credentials for authenticating to the Cyrus Backup server -*channel*\ \_sync\_password: ... - Credentials for authenticating to the Cyrus Backup server - -Using rolling replication -+++++++++++++++++++++++++ - -You can configure backups to use rolling replication. Depending on the sync -repeat interval you configure, this can be used to keep your backups very -current -- potentially as current as your other replicas. - -To configure rolling replication, add additional settings to -:cyrusman:`imapd.conf(5)` like: - -sync\_log: 1 - Enable sync log if it wasn't already. -sync\_log\_channels: *channel* - Add a new channel "*channel*" to whatever was already here. Suggest calling - this "backup" -*channel*\ \_sync\_repeat\_interval: 1 - Minimum time in seconds between rolling replication runs. Smaller value - means livelier backups but more network I/O. Larger value reduces I/O. - -Update :cyrusman:`cyrus.conf(5)` to add a :cyrusman:`sync_client(8)` invocation -to the DAEMON section specifying (at least) the ``-r`` and ``-n channel`` -options. - -See :cyrusman:`imapd.conf(5)` for additional *sync\_* settings that can -be used to affect the replication behaviour. Many can be prefixed with -a channel to limit their affect to only backups, if necessary. - -Using scheduled replication (push) -++++++++++++++++++++++++++++++++++ - -You can configure backups to occur on a schedule determined by the IMAP -server. - -To do this, add :cyrusman:`sync_client(8)` invocations to the EVENTS section -of :cyrusman:`cyrus.conf(5)` (or cron, etc), specifying at least the -``-n channel`` option (to use the channel-specific configuration), plus -whatever other options you need for selecting users to back up. See the -:cyrusman:`sync_client(8)` manpage for details. - -You could also invoke :cyrusman:`sync_client(8)` in a similar way from a -custom script running on the IMAP server. - -Using scheduled replication (pull) -++++++++++++++++++++++++++++++++++ - -You can configure backups to occur on a schedule determined by the -backup server. For example, you may have a custom script that examines -the existing backups, and provokes fresh backups to occur if they are -determined to be out of date. - -To to this, enable XBACKUP on your IMAP server by adding the following -setting to :cyrusman:`imapd.conf(5)`: - -xbackup\_enabled: yes - Enables the XBACKUP command in imapd. - -Your custom script can then authenticate to the IMAP server as an admin -user, and invoke the command ``XBACKUP pattern [channel]``. A replication -of the users or shared mailboxes matching the specified pattern will occur -to the backup server defined by the named channel. If no channel is -specified, default sync configuration will be used. - -For example:: - - C: 1 XBACKUP user.* backup - S: * OK USER anne - S: * OK USER bethany - S: * NO USER cassandane (Operation is not supported on mailbox) - S: * OK USER demi - S: * OK USER ellie - S: 1 OK Completed - -This replicates all users to the channel *backup*. - - -Administration -============== - -.. note:: - Cyrus Backups are experimental, incomplete, and deprecated as of 3.10. - -Storage requirements --------------------- - -It's not really known yet how to predict the storage requirements for a backup -server. Experimentation in dev environment suggests around 20-40% compressed -backup file size relative to the backed up data, depending on compact settings, -but this is with relatively tiny mailboxes and non-pathological data. - -The backup staging spool conservatively needs to be large enough to hold an -entire sync's worth of message files at once. Which is your maximum message -size \* 1024 messages \* the number of backupd processes you're running, plus -some wiggle room probably. In practice it'll probably not hit this limit -unless someone is trying to. (Most users, I suspect, don't have 1024 -maximum-sized messages in their account, or don't receive them all at once -anyway.) - -Certain invocations of ctl\_backups and cyr\_backup also require staging spool -space, due to the way replication protocol (and thus backup data) parsing -handles messages. So keep this in mind I suppose. - -Initial backups ---------------- - -Once a Cyrus Backup system is configured and running, new users that are -created on the IMAP servers will be backed up seamlessly without administrator -intervention. - -The very first backup taken of a pre-existing mailbox will be big -- the entire -mailbox in one hit. It's suggested that, when initially provisioning a Cyrus -Backup server for an existing Cyrus IMAP environment, that the -:cyrusman:`sync_client(8)` commands be run carefully, for a small group of -mailboxes at a time, until all/most of your mailboxes have been backed up at -least once. Also run the :cyrusman:`ctl_backups(8)` ``compact`` command on the -backups, to break up big chunks, if you wish. Only then should you enable -rolling/scheduled replication. - -Restoring from backups ----------------------- - -The :cyrusman:`restore(8)` tool will restore mailboxes and messages from a -specified backup to a specified destination server. The destination server must -be running a replication-capable :cyrusman:`imapd(8)` or -:cyrusman:`sync_server(8)`. The restore tool should be run from the backup -server containing the specified backup. - -File locking ------------- - -All :cyrusman:`backupd(8)`/:cyrusman:`ctl_backups(8)`/:cyrusman:`cyr_backup(8)` -operations first obtain a lock on the relevant backup file. ctl\_backups and -cyr\_backup will try to do this without blocking (unless told otherwise), -whereas backupd will never block. - -Moving backup files to different backup partitions --------------------------------------------------- - -There's no tool for this (yet). To do it manually, stop backupd, copy the files -to the new partition, then use :cyrusman:`cyr_dbtool(8)` to update the user's -backups.db entry to point to the new location. Run the -:cyrusman:`ctl_backups(8)` ``verify`` command on both the new filename (``-f`` -mode) and the user's userid (``-u`` mode) to ensure everything is okay, then -restart backupd. - -Provoking a backup for a particular user/user group/everyone/etc right now --------------------------------------------------------------------------- - -Just run :cyrusman:`sync_client(8)` by hand with appropriate options (as cyrus -user, of course). See its man page for ways of specifying items to replicate. - -If the IMAP server with the user's mail has been configured with the -``xbackup_enabled: yes`` option in :cyrusman:`imapd.conf(5)`, then an admin -user can cause a backup to occur by sending the IMAP server an ``XBACKUP`` -command. - -What about tape backups? ------------------------- - -As long as backupd, ctl\_backups and cyr\_backup are not currently running (and -assuming no-one's poking around in things otherwise), it's safe to take/restore -a filesystem snapshot of backup partitions. So to schedule, say, a nightly tape -dump of your Cyrus Backup server, make your cron job shut down Cyrus, make the -copy, then restart Cyrus. - -Meanwhile, your Cyrus IMAP servers are still online and available. Regular -backups will resume once your backupd is running again. - -If you can work at a finer granularity than file system, you don't need to shut -down backupd. Just use the :cyrusman:`ctl_backups(8)` ``lock`` command to hold -a lock on each backup while you work with its files, and the rest of the backup -system will work around that. - -Restoring is more complicated, depending on what you actually need to do: -when you restart the backupd after restoring a filesystem snapshot, the next -time your Cyrus IMAP server replicates to it, the restored backups will be -brought up to date. Probably not what you wanted -- so don't restart backupd -until you've done whatever you were doing. - -Multiple IMAP servers, one backup server ----------------------------------------- - -This is fine, as long as each user being backed up is only being backed up by -one server (or they are otherwise synchronised). If IMAP servers have different -ideas about the state of a user's mailboxes, one of those will be in sync with -the backup server and the other will get a lot of replication failures. - -Multiple IMAP servers, multiple backup servers ----------------------------------------------- - -Make sure your :cyrusman:`sync_client(8)` configuration(s) on each IMAP server -knows which users are being backed up to which backup servers, and selects -them appropriately. See the :cyrusman:`sync_client(8)` man page for options for -specifying users, and run it as an event (rather than rolling). - -Or just distribute it at server granularity, such that backup server A serves -IMAP servers A, B and C, and backup server B serves IMAP servers D, E, F, etc. - -One IMAP server, multiple backup servers ----------------------------------------- - -Configure one channel plus one rolling :cyrusman:`sync_client(8)` per backup -server, and your IMAP server can be more or less simultaneously backed up to -multiple backup destinations. - -Reducing load -------------- - -To reduce load on your client-facing IMAP servers, configure sync log chaining -on their replicas and let those take the load of replicating to the backup -servers. - -To reduce network traffic, do the same thing, specifically using replicas that -are already co-located with the backup server. - -Other setups ------------- - -The use of the replication protocol and :cyrusman:`sync_client(8)` allows a lot -of interesting configuration possibilities to shake out. Have a rummage in the -:cyrusman:`sync_client(8)` man page for inspiration. - -Tools -===== - -.. note:: - Cyrus Backups are experimental, incomplete, and deprecated as of 3.10. - -ctl\_backups ------------- - -This tool is generally for mass operations that require few/fixed arguments -across multiple/all backups - -Supported operations: - -compact - Reduce backups' disk usage by: - - * combining small chunks for better gzip compression -- especially - important for hot backups, which produce many tiny chunks - * removing deleted content that has passed its retention period -list - List known backups. -lock - Lock a single backup, so you can safely work on it with non-cyrus tools. -reindex - Regenerate indexes for backups from their data files. Useful if index - becomes corrupted by some bug, or invalidated by working on data with - non-cyrus tools. -stat - Show statistics about backups -- disk usage, compression ratio, etc. -verify - Deep verification of backups. Verifies that: - - * Checksums for each chunk in index match data - * Mailbox states are in the chunk that the index says they're in - * Mailbox states match indexed states - * Messages are in the chunk the index says they're in - * Message data checksum matches indexed checksums - -See the :cyrusman:`ctl_backups(8)` man page for more information. - -cyr\_backup ------------ - -This tool is generally for operations on a single mailbox that require multiple -additional arguments - -Supported operations - -list [ chunks \| mailboxes \| messages \| all ] - Line-per-item listing of information stored in a backup. -show [ chunks \| mailboxes \| messages ] items... - Paragraph-per-item listing of information for specified items. Chunk items - are specified by id, mailboxes by mboxname or uniqueid, messages by guid. -dump [ chunk \| message ] item - Full dump of one item. chunk dumps the uncompressed content of a chunk - (i.e. a bunch of sync protocol commands). message dumps a raw rfc822 - message (useful for manually restoring) - -See the :cyrusman:`cyr_backup(8)` man page for more information. - -restore -------- - -This tool is for restoring mail from backup files. - -Required arguments are a destination server (in ip:port or host:port format), -a backup file, and mboxnames, uniqueids or guids specifying the mailboxes or -messages to be restored. - -If the target mailbox does not already exist on the destination server, options -are available to preserve the mailbox and message properties as they existed -in the backup. This is useful for rebuilding a lost server from backups, such -that client state remains consistent. - -If the target mailbox already exists on the destination server, restored -messages will be assigned new, unused uids and will appear to the client as new -messages. - -See the :cyrusman:`restore(8)` man page for more information. diff --git a/docsrc/imap/reference/admin/locations.rst b/docsrc/imap/reference/admin/locations.rst index db5b5470574..2a6aafa24c8 100644 --- a/docsrc/imap/reference/admin/locations.rst +++ b/docsrc/imap/reference/admin/locations.rst @@ -23,7 +23,6 @@ and may be defined for the following types of data: * :ref:`imap-features-mail-spool-partitions` * :ref:`imap-features-metadata-partitions` - * :ref:`cyrus-backups` * :ref:`imap-features-archive-partitions` Please consult the documents linked above for more information on these. diff --git a/docsrc/imap/reference/manpages/systemcommands/backupd.rst b/docsrc/imap/reference/manpages/systemcommands/backupd.rst deleted file mode 100644 index 4c672f2986e..00000000000 --- a/docsrc/imap/reference/manpages/systemcommands/backupd.rst +++ /dev/null @@ -1,98 +0,0 @@ -.. cyrusman:: backupd(8) - -.. _imap-reference-manpages-systemcommands-backupd: - -=========== -**backupd** -=========== - -Cyrus Backups server process - -Synopsis -======== - -.. parsed-literal:: - - **backupd** [ **-C** *config-file* ] [ **-U** *uses* ] [ **-T** *timeout* ] [ **-D** ] - [ **-s** ] [ **-N** ] [ **-p** *ssf* ] - -Description -=========== - -.. note:: - Cyrus Backups are experimental, incomplete, and deprecated as of 3.10. - -**backupd** is the Cyrus Backups server. It accepts Cyrus replication protocol -commands on its standard input and responds on its standard output. It MUST be -invoked by :cyrusman:`master(8)` with those descriptors attached to a -replication client connection, typically :cyrusman:`sync_client(8)`. - -**backupd** |default-conf-text| - -**backupd** is generally configured to run on a dedicated backup server, -containing backup storage, but no IMAP service or mailbox storage. - -You must configure at least one *backuppartition*. User backups will be -distributed among the configured partitions. Note that there is no -relationship between mailbox partitions and *backuppartitions*. - -If the directory ``log``\/*user* exists under the directory specified in the -``configdirectory`` configuration option, then **backupd** will create -protocol telemetry logs for sessions authenticating as *user*. The telemetry -logs will be stored in the ``log``\/*user* directory with a filename of the -**backupd** process-id. - -Options -======= - -.. program:: backupd - -.. option:: -C config-file - - |cli-dash-c-text| - -.. option:: -U uses - - The maximum number of times that the process should be used for new - connections before shutting down. The default is 250. - -.. option:: -T timeout - - The number of seconds that the process will wait for a new - connection before shutting down. Note that a value of 0 (zero) - will disable the timeout. The default is 60. - -.. option:: -D - - Run external debugger specified in debug_command. - -.. option:: -p ssf - - Tell **backupd** that an external layer exists. An *SSF* (security - strength factor) of 1 means an integrity protection layer exists. - Any higher SSF implies some form of privacy protection. - - -Examples -======== - -**backupd** is commonly included in the SERVICES section of -:cyrusman:`cyrus.conf(5)` like so: - -.. parsed-literal:: - SERVICES { - **backupd cmd="backupd" listen="csync" prefork=0** - } - -History -======= - -Files -===== - -See Also -======== - -:cyrusman:`imapd.conf(5)`, -:cyrusman:`master(8)`, -:cyrusman:`sync_client(8)` diff --git a/docsrc/imap/reference/manpages/systemcommands/ctl_backups.rst b/docsrc/imap/reference/manpages/systemcommands/ctl_backups.rst deleted file mode 100644 index c90174ed6ae..00000000000 --- a/docsrc/imap/reference/manpages/systemcommands/ctl_backups.rst +++ /dev/null @@ -1,272 +0,0 @@ -.. cyrusman:: ctl_backups(8) - -.. program:: ctl_backups - -.. _imap-reference-manpages-systemcommands-ctl_backups: - -=============== -**ctl_backups** -=============== - -Perform administrative operations directly on Cyrus backups. - -Synopsis -======== - -.. parsed-literal:: - - **ctl_backups** [OPTIONS] compact [MODE] *backup*... - **ctl_backups** [OPTIONS] list [LIST OPTIONS] [[MODE] *backup*...] - **ctl_backups** [OPTIONS] lock [LOCK OPTIONS] [MODE] *backup* - **ctl_backups** [OPTIONS] reindex [MODE] *backup*... - **ctl_backups** [OPTIONS] stat [MODE] *backup*... - **ctl_backups** [OPTIONS] verify [MODE] *backup*... - -Description -=========== - -.. note:: - Cyrus Backups are experimental, incomplete, and deprecated as of 3.10. - -**ctl_backups** is a tool for performing administrative operations on Cyrus -backups. - -**ctl_backups** |default-conf-text| - -In all invocations, *backup* is interpreted according to the specified MODE. -See :ref:`ctl-backups-modes` below. - -**ctl_backups** provides the following sub-commands: - -.. option:: compact - - Reduce storage required by the named backups. Compact behaviour is - influenced by the **backup_compact_minsize**, **backup_compact_maxsize**, - **backup_compact_work_threshold**, and **backup_retention_days** - configuration settings. See :cyrusman:`imapd.conf(5)` for details. - - This should generally be invoked regularly, such as by adding an - entry to the EVENTS section of :cyrusman:`cyrus.conf(5)`. See - :ref:`ctl-backups-examples` for an example. - - If the **backup_keep_previous** configuration setting is enabled, - compact will preserve the original data and index files (renaming - them with a timestamp). This is useful for debugging. - -.. option:: list - - List backups. See :ref:`ctl-backups-list-options` for options specific - to the **list** sub-command. Columns are separated by tabs, and are: - - * end time of latest chunk - * size of backup data file on disk - * userid to which the backup belongs - * path to backup data file - - If no :ref:`mode ` or backups are specified, lists all - known backups (as if invoked with the **-A** mode). - -.. option:: lock - - Obtain and hold a lock on the named backup. Useful for operating on - Cyrus backup files using non-Cyrus tools (such as UNIX tools or custom - scripts) in relative safety. See :ref:`ctl-backups-lock-options` for details. - -.. option:: reindex - - Rebuild the indexes for the named backups, based on the raw backup data. - This is useful if their index files have been corrupted, or if the index - format has changed. - - If the **backup_keep_previous** configuration setting is enabled, - reindex will preserve the original index file (renaming it with a - timestamp). This is useful for debugging. - -.. option:: stat - - Display stats for the named backups. Columns are separated by tabs, and - are: - - * userid or filename - * compressed (i.e. on disk) size - * uncompressed size - * compactable size - * compression ratio - * utilisation ratio - * start time of latest chunk - * end time of latest chunk - - The compactable size is an approximation of how much uncompressed data would - remain after **compact** is performed. The utilisation ratio is this figure - expressed as a percentage of the uncompressed size. Note that this - approximation is an underestimate. That is to say, a backup that has just - been compacted will probably still report less than 100% utilisation. - -.. option:: verify - - Verify consistency of the named backups by performing deep checks on both - the raw backup data and its index. - -Options -======= - -.. option:: -C config-file - - |cli-dash-c-text| - -.. option:: -F, --force - - Force the operation to occur, even if it is determined to be unnecessary. - This is mostly useful with the **compact** sub-command. - -.. option:: -S, --stop-on-error - - Stop-on-error. With this option, if a sub-command fails for any - particular backup, **ctl_backups** will immediately exit with an error, - without processing further backups. - - The default is to log the error, and continue with the next backup. - -.. option:: -V, --no-verify - - Don't verify backup checksums for read-only operations. - - The read-only operations **list** and **stat** will normally perform a - "quick" verification of the backup file being read, which checks the - checksums of the most recent chunk. This can be slow for backups - whose most recent backup chunk is very large. - - With this option, the verification step will be skipped. - -.. option:: -j, --json - - Produce output in JSON format. The default is plain text. - -.. option:: -v, --verbose - - Increase the verbosity. Can be specified multiple times. - -.. option:: -w, --wait-for-locks - - Wait for locks. With this option, if a backup named on the command line is - locked, execution will block until the lock becomes available. - - The default is to skip backups that are currently locked. - - -.. _ctl-backups-list-options: - -List Options -============ - -Options that apply only to the **list** sub-command. - -.. option:: -t [hours], --stale[=hours] - - List stale backups only, that is, backups that have received no updates - in *hours*. If *hours* is unspecified, it defaults to 24. - -.. _ctl-backups-lock-options: - -Lock Options -============ - -Options that apply only to the **lock** sub-command. - -.. option:: -c, --create - - Exclusively create the named backup while obtaining the lock. Exits - immediately with an error if the named backup already exists. - - When the lock is successfully obtained, continue as per the other options. - -.. option:: -p, --pause - - Locks the named backup, and then waits for EOF on the standard input - stream. Unlocks the backup and exits once EOF is received. This is the - default mode of operation. - -.. option:: -s, --sqlite3 - - Locks the named backup, and with the lock held, opens its index file in - the :manpage:`sqlite3(1)` program. The lock is automatically released when - sqlite3 exits. - -.. option:: -x command, --execute=command - - Locks the named backup, and with the lock held, executes *command* using - **/bin/sh** (as per :manpage:`system(3)`). The lock is automatically - released when *command* completes. - - The filenames of the backup data and index are made available to *command* - in the environment variables **$ctl_backups_lock_data_fname** and - **$ctl_backups_lock_index_fname**, respectively. - -.. _ctl-backups-modes: - -Modes -===== - -.. option:: -A, --all - - Run sub-command over all known backups. - - Known backups are recorded in the database specified by the **backup_db** - and **backup_db_path** configuration options. - -.. option:: -D, --domains - - Backups specified on the command line are interpreted as domains. Run - sub-command over known backups for users in these domains. - -.. option:: -P, --prefixes - - Backups specified on the command line are interpreted as userid prefixes. - Run sub-command over known backups for users matching these prefixes. - -.. option:: -f, --filenames - - Backups specified on the command line are interpreted as filenames. Run - sub-command over the matching backup files. The backup files do not need - to be known about in the backups database. - -.. option:: -m, --mailboxes - - Backups specified on the command line are interpreted as mailbox names. - Run sub-command over known backups containing these mailboxes. - -.. option:: -u, --userids - - Backups specified on the command line are interpreted as userids. Run - sub-command over known backups for matching users. - - This is the default if no mode is specified. - -.. _ctl-backups-examples: - -Examples -======== - -Scheduling **ctl_backups compact** to run each morning using the EVENTS -section of :cyrusman:`cyrus.conf(5)`: - -.. parsed-literal:: - EVENTS { - checkpoint cmd="ctl_cyrusdb -c" period=30 - - **compact cmd="ctl_backups compact -A" at=0400** - } - - -History -======= - -Files -===== - -See Also -======== - -:cyrusman:`imapd.conf(5)`, -:manpage:`sqlite3(1)`, -:manpage:`system(3)` diff --git a/docsrc/imap/reference/manpages/systemcommands/cyr_backup.rst b/docsrc/imap/reference/manpages/systemcommands/cyr_backup.rst deleted file mode 100644 index a45611a638f..00000000000 --- a/docsrc/imap/reference/manpages/systemcommands/cyr_backup.rst +++ /dev/null @@ -1,148 +0,0 @@ -.. cyrusman:: cyr_backup(8) - -.. _imap-reference-manpages-systemcommands-cyr_backup: - -============== -**cyr_backup** -============== - -Inspect contents of Cyrus backups. - -Synopsis -======== - -.. parsed-literal:: - - **cyr_backup** [OPTIONS] [MODE] *backup* list chunks - **cyr_backup** [OPTIONS] [MODE] *backup* list mailboxes - **cyr_backup** [OPTIONS] [MODE] *backup* list messages - **cyr_backup** [OPTIONS] [MODE] *backup* list all - **cyr_backup** [OPTIONS] [MODE] *backup* show mailboxes [ *mboxname* | *uniqueid* ]... - **cyr_backup** [OPTIONS] [MODE] *backup* show messages *guid*... - **cyr_backup** [OPTIONS] [MODE] *backup* dump chunk *id* - **cyr_backup** [OPTIONS] [MODE] *backup* dump message *guid* - **cyr_backup** [OPTIONS] [MODE] *backup* json chunks - **cyr_backup** [OPTIONS] [MODE] *backup* json mailboxes - **cyr_backup** [OPTIONS] [MODE] *backup* json messages - **cyr_backup** [OPTIONS] [MODE] *backup* json headers *guid*... - -Description -=========== - -.. note:: - Cyrus Backups are experimental, incomplete, and deprecated as of 3.10. - -**cyr_backup** is a tool for inspecting the contents of a Cyrus backup. - -**cyr_backup** |default-conf-text| - -In all invocations, *backup* is interpreted according to the specified MODE. -See :ref:`cyr-backup-modes` below. - -**cyr_backup** provides the following sub-commands: - -.. option:: list - - .. parsed-literal:: - **cyr_backup** [OPTIONS] [MODE] *backup* list chunks - **cyr_backup** [OPTIONS] [MODE] *backup* list mailboxes - **cyr_backup** [OPTIONS] [MODE] *backup* list messages - **cyr_backup** [OPTIONS] [MODE] *backup* list all - - List chunks, mailboxes, or messages contained in *backup*. - -.. option:: show - - .. parsed-literal:: - **cyr_backup** [OPTIONS] [MODE] *backup* show mailboxes [ *mboxname* | *uniqueid* ]... - **cyr_backup** [OPTIONS] [MODE] *backup* show messages *guid*... - - **show mailboxes** shows the contents (i.e. the contained messages) of - mailboxes matching each given *mboxname* or *uniqueid* in the named - *backup*. The uid, expunged time, and guid of each message are shown. - - **show messages** shows details of messages matching each given *guid* in - the named *backup*. The guid, containing mailboxes, and MIME headers of - each message are shown. - -.. option:: dump - - .. parsed-literal:: - **cyr_backup** [OPTIONS] [MODE] *backup* dump chunk *id* - **cyr_backup** [OPTIONS] [MODE] *backup* dump message *guid* - - Dump the contents of the specified chunk or message contained in the named - *backup* to standard out. Chunks are output as uncompressed backup data - format (which is a replication protocol stream). Messages are output in - their on-disk format (typically MIME). - - **dump chunk** is mainly useful for diagnostic purposes. - - **dump message** may be useful as part of a manual message restoration - process, when the restoration destination does not support the - :cyrusman:`restore(8)` command. - -.. option:: json - - .. parsed-literal:: - **cyr_backup** [OPTIONS] [MODE] *backup* json chunks - **cyr_backup** [OPTIONS] [MODE] *backup* json mailboxes - **cyr_backup** [OPTIONS] [MODE] *backup* json messages - **cyr_backup** [OPTIONS] [MODE] *backup* json headers *guid*... - - Dump information about the chunks, mailboxes or messages contained in the - named *backup* to standard out, in JSON format. - -.. _cyr-backup-options: - -Options -======= - -.. program:: cyr_backup - -.. option:: -C config-file - - |cli-dash-c-text| - -.. option:: -v, --verbose - - Increase the verbosity. Can be specified multiple times. - -.. _cyr-backup-modes: - -Modes -===== - -.. option:: -f, --filename - - *backup* is interpreted as a filename. The named file does not need to be - known about in the backups database. - -.. option:: -m, --mailbox - - *backup* is interpreted as a mailbox name. There must be a known backup - for the user whose mailbox this is. - - Known backups are recorded in the database specified by the **backup_db** - and **backup_db_path** configuration options. - -.. option:: -u, --userid - - *backup* is interpreted as a userid. There must be a known backup for - the specified user. - - This is the default if no mode is specified. - -Examples -======== - -History -======= - -Files -===== - -See Also -======== - -:cyrusman:`restore(8)` diff --git a/docsrc/imap/reference/manpages/systemcommands/restore.rst b/docsrc/imap/reference/manpages/systemcommands/restore.rst deleted file mode 100644 index f0eb839283c..00000000000 --- a/docsrc/imap/reference/manpages/systemcommands/restore.rst +++ /dev/null @@ -1,249 +0,0 @@ -.. cyrusman:: restore(8) - -.. program:: restore - -.. _imap-reference-manpages-systemcommands-restore: - -=========== -**restore** -=========== - -Restore content from Cyrus backups. - -Synopsis -======== - -.. parsed-literal:: - - **restore** [OPTIONS] *server* [MODE] *backup* [ *mboxname* | *uniqueid* | *guid* ]... - -Description -=========== - -.. note:: - Cyrus Backups are experimental, incomplete, and deprecated as of 3.10. - -**restore** is a tool for restoring messages and mailboxes from a Cyrus backup -to a Cyrus IMAP server. It must be run from the server containing the backup -storage. - -**restore** |default-conf-text| - -*server* specifies the destination server to which content should be restored. -It should be of the form '*host*\ [:\ *port*\ ]', where *host* is either a -hostname, an IPv4 address, or an IPv6 address, and where the optional *port* is -either a known service name (see :manpage:`services(5)`) or a decimal port -number. If *port* is omitted, ``imap`` will be tried first, followed by -``csync``. - -The destination server must point to either an :cyrusman:`imapd(8)` instance -with the replication capability enabled, or a :cyrusman:`sync_server(8)` -instance. In either case it must be Cyrus version 3.0 or newer. - -**restore** will authenticate to the destination server according to the -**restore_authname**, **restore_password** and **restore_realm** configuration -options. The credentials should correspond with one of the destination -server's **admins**. - -*backup* is interpreted according to the specified MODE. -See :ref:`restore-modes` below. - -If neither **-a** nor **-F** options were provided, then the remaining -arguments constitute a list of objects to be restored. These may be mailboxes -(specified by either *mboxname* or *uniqueid*) or messages (specified by their -*guid*). The objects may be specified in any order, and both mailboxes and -individual messages may be restored in one go. :cyrusman:`cyr_backup(8)` can -be used to identify objects to restore from a Cyrus backup. - -Selected mailboxes will have their messages restored to a mailbox of the same -name, which will be created if necessary. Individually-selected messages will -be restored to the mailboxes in which they previously existed. In both cases -the **-M** option can be used to override the destination mailbox (see below), -but note the consequences of doing this when multiple mailbox objects have -been specified, or when the **-r** option is in use. - -Mailboxes that are created during the restoration process will have their ACL -set to the one stored in the backup. The **-A** option can be used to override -this. Mailboxes that are not created during the restoration process (i.e. when -restoring into mailboxes that already exists) will not have their ACLs altered. - -Options -======= - -.. option:: -A [acl], --override-acl[=acl] - - Apply specified *acl* to restored mailboxes, rather than their ACLs as - stored in the backup. - - If *acl* is the empty string (e.g. ``-A ""``) or is unspecified, mailboxes - will be restored with the default ACL for their destination owner. This - is mostly useful when restoring folders from one user's backup into a - different user's mailbox. - -.. option:: -C config-file - - |cli-dash-c-text| - -.. option:: -D, --keep-deletedprefix - - Don't trim **deletedprefix** from mailbox names prior to restoring. This - is mainly useful for rebuilding failed servers, where deleted mailboxes - should be restored as deleted mailboxes, not as new ones. - - The default is to trim the prefix before restoring. - - If the original server from which the backups were produced had - **delete_mode** set to *immediate*, then the mailboxes in the backup will - not have such a prefix, and this option won't have any useful effect. - - See :cyrusman:`imapd.conf(5)` for information about the **deletedprefix** - and **delete_mode** configuration options. - -.. option:: -F input-file, --input-file=input-file - - Get the list of mailboxes or messages from *input-file* instead of from - the command line arguments. - - *input-file* should contain one object specification (either an *mboxname*, - a *uniqueid*, or a *guid*) per line. Empty lines, and lines beginning with - a '#' character, are ignored. - -.. option:: -L, --local-only - - Local operations only. Actions required to restore the requested mailboxes - and messages will be performed on the destination server only. - :cyrusman:`mupdate(8)` actions will not occur. - - The default is for mupdate actions to occur if the destination server is - part of a murder. - - This option has no effect if the destination server is not part of a murder. - -.. option:: -M mboxname, --dest-mailbox=mboxname - - Messages are restored to the mailbox with the specified *mboxname*. If no - mailbox of this name exists, one will be created. - - If multiple mailbox objects are to be restored, whether due to being - specified on the command line, in an *input-file*, or via the **-r** - option, then the collective contents of all such mailboxes will be - restored to the single mailbox *mboxname*. This may not be what you want! - - The default when restoring mailboxes is to restore their respective - contents into mailboxes of the same names. - - The default when restoring individual messages is to restore them into - their original mailboxes. - -.. option:: -P partition, --dest-partition=partition - - Restore mailboxes to the specified *partition* - -.. option:: -U, --keep-uidvalidity - - Try to preserve uidvalidity and other related fields, such that the - restored mailboxes and messages appear like they never left, and IMAP - clients can avoid expensive state updates. - - This can only occur if the mailboxes to be restored **do not** already - exist on the destination server. As such, this option is mainly useful - when rebuilding a failed server. - - If the destination mailboxes already exist, restored messages will be - appended as if newly delivered, regardless of whether the **-U** option - was specified. - -.. option:: -X, --skip-expunged - - Do not restore messages that are marked as expunged in the *backup*. - - See also **-x**. - -.. option:: -a, --all-mailboxes - - Try to restore all mailboxes in the specified *backup*. - -.. option:: -n, --dry-run - - Do nothing. The work required to perform the restoration will be - calculated (and reported depending on verbosity level), but no - restoration will take place, and no connection will be made to - the destination server. - - Note that the *server* argument is still mandatory with this option. - -.. option:: -r, --recursive - - Recurse into submailboxes. When restoring mailboxes, also restore - any mailboxes contained within them. - - The default is to restore only explicitly-specified mailboxes. - -.. option:: -v, --verbose - - Increase the verbosity level. This option can be specified multiple times - for additional verbosity. - -.. option:: -w seconds, --delayed-startup=seconds - - Wait *seconds* before starting. This is useful for attaching a debugger. - -.. option:: -x, --only-expunged - - Only restore messages that are marked as expunged in the *backup*. - - This can be convenient for restoring messages that were accidentally - deleted by the user, without needing to track down individual message - guids. - - See also **-X**. - -.. option:: -z, --require-compression - - Require compression for server connection. The restore will abort - if compression is unavailable. - -.. _restore-modes: - -Modes -===== - -.. option:: -f backup, --file=backup - - *backup* is interpreted as a filename. The named file does not need to be - known about in the backups database. - -.. option:: -m backup, --mailbox=backup - - *backup* is interpreted as a mailbox name. There must be a known backup - for the user whose mailbox this is. - - Known backups are recorded in the database specified by the **backup_db** - and **backup_db_path** configuration options. - -.. option:: -u backup, --userid=backup - - *backup* is interpreted as a userid. There must be a known backup for - the specified user. - - This is the default if no mode is specified. - - -Examples -======== - -History -======= - -Files -===== - -See Also -======== - -:cyrusman:`imapd.conf(5)`, -:manpage:`services(5)`, -:cyrusman:`cyr_backup(8)`, -:cyrusman:`imapd(8)`, -:cyrusman:`mupdate(8)`, -:cyrusman:`sync_server(8)` diff --git a/lib/gzuncat.c b/lib/gzuncat.c deleted file mode 100644 index 867d71baffc..00000000000 --- a/lib/gzuncat.c +++ /dev/null @@ -1,353 +0,0 @@ -/* gzuncat.c - read individual members from concatenated gzip files - * - * Copyright (c) 2015 Carnegie Mellon University. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * 3. The name "Carnegie Mellon University" must not be used to - * endorse or promote products derived from this software without - * prior written permission. For permission or any legal - * details, please contact - * Carnegie Mellon University - * Center for Technology Transfer and Enterprise Creation - * 4615 Forbes Avenue - * Suite 302 - * Pittsburgh, PA 15213 - * (412) 268-7393, fax: (412) 268-7395 - * innovation@andrew.cmu.edu - * - * 4. Redistributions of any form whatsoever must retain the following - * acknowledgment: - * "This product includes software developed by Computing Services - * at Carnegie Mellon University (http://www.cmu.edu/computing/)." - * - * CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO - * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY - * AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE - * FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN - * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING - * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -#include - -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "lib/xmalloc.h" - -#include "lib/gzuncat.h" - -/* - * current_offset, next_offset, and member_eof together indicate the state of - * the reader: - * - * when not reading a member (after gzuc_new or gzuc_member_end): - * - * current_offset = -1 - * next_offset = start point for next call to gzuc_start() - * member_eof = undefined - * - * when reading a member (after gzuc_member_start): - * - * current_offset = file offset of the start of the member being read - * next_offset = -1 - * member_eof = 0 - * - * when reaching the end of a member (before gzuc_member_end): - * - * current_offset = file offset of the start of the member being read - * next_offset = -1 - * member_eof = 1 - */ - -static const size_t default_in_buf_size = 16 * 1024; - -struct gzuncat { - int fd; - off_t current_offset; - off_t next_offset; - int member_eof; - int file_eof; - z_stream strm; - unsigned char *in_buf; - size_t in_buf_size; - size_t bytes_read; -}; - -EXPORTED struct gzuncat *gzuc_new(int fd) -{ - if (fd < 0) return NULL; - - struct gzuncat *gz = xmalloc(sizeof(*gz)); - - gz->fd = fd; - gz->current_offset = -1; - gz->next_offset = 0; - gz->member_eof = -1; - gz->file_eof = 0; - gz->in_buf = NULL; - gz->in_buf_size = default_in_buf_size; - gz->bytes_read = 0; - - return gz; -} - -EXPORTED int gzuc_set_bufsize(struct gzuncat *gz, size_t size) -{ - if (gz->in_buf) return -1; - if (!size) return -1; - - gz->in_buf_size = size; - - return 0; -} - -static int _inflate_init(z_stream *strm, unsigned char *in_buf) -{ - strm->zalloc = Z_NULL; - strm->zfree = Z_NULL; - strm->opaque = Z_NULL; - strm->avail_in = 0; - strm->next_in = in_buf; - - // 15 = support maximum window size - // 16 = decode gzip format - return inflateInit2(strm, 15 + 16); -} - -EXPORTED int gzuc_member_start_from(struct gzuncat *gz, off_t offset) -{ - if (gz->current_offset >= 0 || offset < 0) { - errno = EINVAL; - return Z_ERRNO; - } - - if (!gz->in_buf) - gz->in_buf = xmalloc(gz->in_buf_size); - - memset(gz->in_buf, 0, gz->in_buf_size); - - off_t p = lseek(gz->fd, offset, SEEK_SET); - if (p < 0) return Z_ERRNO; - - int r = _inflate_init(&gz->strm, gz->in_buf); - if (r) return r; - - // anything else to initialise? - - gz->current_offset = offset; - gz->next_offset = -1; - gz->member_eof = 0; - gz->file_eof = 0; - gz->bytes_read = 0; - - return 0; -} - -EXPORTED int gzuc_member_start(struct gzuncat *gz) -{ - return gzuc_member_start_from(gz, gz->next_offset); -} - -EXPORTED int gzuc_member_end(struct gzuncat *gz, off_t *offset) -{ - if (gz->next_offset >= 0) return -1; - - int r = 0; - - if (gz->file_eof) goto done; - - if (gz->current_offset >= 0) { - char discard[16 * 1024]; - while ((r = gzuc_read(gz, discard, sizeof(discard))) > 0); - - /* don't set next_offset if we're at end of underlying file */ - if (gz->file_eof) goto done; - /* nor if there was an error */ - if (r < 0) goto done; - } - - /* we're now at the start of the next member */ - gz->next_offset = lseek(gz->fd, 0, SEEK_CUR); - -done: - inflateEnd(&gz->strm); - gz->current_offset = -1; - gz->member_eof = -1; - gz->bytes_read = 0; - if (!r && offset) *offset = gz->next_offset; - return r; -} - -EXPORTED void gzuc_free(struct gzuncat **gzp) -{ - if (!gzp) return; - if (!*gzp) return; - - struct gzuncat *gz = *gzp; - *gzp = NULL; - - if (gz->current_offset >= 0) - inflateEnd(&gz->strm); - - if (gz->in_buf) { - free(gz->in_buf); - } - - free(gz); -} - -EXPORTED int gzuc_member_eof(struct gzuncat *gz) -{ - if (gz->member_eof == 1) return 1; - if (gz->current_offset < 0) return 1; - if (gz->strm.avail_in) return 0; - return gz->file_eof; -} - -EXPORTED int gzuc_eof(struct gzuncat *gz) -{ - return gz->file_eof; -} - -EXPORTED ssize_t gzuc_read(struct gzuncat *gz, void *buf, size_t count) -{ - if (gz->current_offset < 0) return -1; - if (gz->member_eof == 1) return 0; - if (gz->file_eof == 1) return 0; - - gz->strm.avail_out = count; - gz->strm.next_out = buf; - - ssize_t uncompressed = 0; - int r = 0; - - memset(buf, 0, count); - - do { - // read some more input if we need it - if (!gz->strm.avail_in) { - r = read(gz->fd, gz->in_buf, gz->in_buf_size); - - if (r < 0) { - syslog(LOG_ERR, "IOERROR: %s: read %d: %m", __func__, gz->fd); - return r; - } - else if (r == 0) { - gz->file_eof = 1; - break; - } - else { - gz->strm.avail_in = r; - gz->strm.next_in = gz->in_buf; - } - } - - r = inflate(&gz->strm, Z_SYNC_FLUSH /* FIXME what */); - uncompressed = count - gz->strm.avail_out; - - if (r == Z_OK) { - continue; - } - else if (r == Z_STREAM_END) { - // if we get to the end of the gzip member, and there's still data avail_in the stream - // object, then we've read too much (we're starting to see the next section of the file) - // so we need to seek back to the right spot and update next_offset - if (gz->strm.avail_in) { - off_t p = lseek(gz->fd, 0 - (off_t) gz->strm.avail_in, SEEK_CUR); - if (p < 0) { - syslog(LOG_ERR, "IOERROR: %s: lseek %d: %m", __func__, gz->fd); - return -1; - } - gz->strm.avail_in = 0; - gz->strm.next_in = gz->in_buf; - } - - gz->member_eof = 1; - break; - } - else { - syslog(LOG_DEBUG, "IOERROR: gzuc_read: returning %i (%s)", r, gz->strm.msg); - return r; - } - } while (gz->strm.avail_out); // keep going while we haven't filled the buffer - - gz->bytes_read += uncompressed; - return uncompressed; -} - -EXPORTED int gzuc_skip(struct gzuncat *gz, size_t len) -{ - if (gzuc_member_eof(gz)) return -1; - - while (len) { - unsigned char discard[16 * 1024]; - - size_t want = len; - if (want > sizeof(discard)) want = sizeof(discard); - - ssize_t got = gzuc_read(gz, discard, want); - if (got == 0) return -1; - if (got < 0) return got; - - len -= got; - } - - return 0; -} - -/* n.b. not a conventional seek function - only seeks to absolute position - * it's also pretty expensive if the position is earlier than the - * current position, so try to keep your reads in order - */ -EXPORTED int gzuc_seekto(struct gzuncat *gz, size_t pos) -{ - if (gz->current_offset < 0) return -1; - - gz->member_eof = 0; - gz->file_eof = 0; - - if (pos == gz->bytes_read) return 0; - - if (pos < gz->bytes_read) { - off_t p = lseek(gz->fd, gz->current_offset, SEEK_SET); - if (p < 0) return -1; - - inflateEnd(&gz->strm); - int r = _inflate_init(&gz->strm, gz->in_buf); - if (r) return r; - - gz->bytes_read = 0; - } - - return gzuc_skip(gz, pos - gz->bytes_read); -} - -EXPORTED off_t gzuc_member_offset(struct gzuncat *gz) -{ - return gz->current_offset; -} - -EXPORTED size_t gzuc_member_bytes_read(struct gzuncat *gz) -{ - return gz->bytes_read; -} diff --git a/lib/gzuncat.h b/lib/gzuncat.h deleted file mode 100644 index dc6371abc8f..00000000000 --- a/lib/gzuncat.h +++ /dev/null @@ -1,68 +0,0 @@ -/* gzuncat.h - read individual members from concatenated gzip files - * - * Copyright (c) 2015 Carnegie Mellon University. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * 3. The name "Carnegie Mellon University" must not be used to - * endorse or promote products derived from this software without - * prior written permission. For permission or any legal - * details, please contact - * Carnegie Mellon University - * Center for Technology Transfer and Enterprise Creation - * 4615 Forbes Avenue - * Suite 302 - * Pittsburgh, PA 15213 - * (412) 268-7393, fax: (412) 268-7395 - * innovation@andrew.cmu.edu - * - * 4. Redistributions of any form whatsoever must retain the following - * acknowledgment: - * "This product includes software developed by Computing Services - * at Carnegie Mellon University (http://www.cmu.edu/computing/)." - * - * CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO - * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY - * AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE - * FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN - * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING - * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ -#ifndef GZUNCAT_H -#define GZUNCAT_H - -#include - -#include - -#include - -struct gzuncat; - -struct gzuncat *gzuc_new(int fd); -void gzuc_free(struct gzuncat **gzp); - -int gzuc_set_bufsize(struct gzuncat *gz, size_t size); -int gzuc_member_start_from(struct gzuncat *gz, off_t offset); -int gzuc_member_start(struct gzuncat *gz); -int gzuc_member_end(struct gzuncat *gz, off_t *offset); -int gzuc_member_eof(struct gzuncat *gz); -int gzuc_eof(struct gzuncat *gz); -ssize_t gzuc_read(struct gzuncat *gz, void *buf, size_t count); -int gzuc_skip(struct gzuncat *gz, size_t len); -int gzuc_seekto(struct gzuncat *gz, size_t pos); -off_t gzuc_member_offset(struct gzuncat *gz); -size_t gzuc_member_bytes_read(struct gzuncat *gz); - -#endif