cyrusdb: lock conversion section, and add cyrusdb_autconvert config

Author: brong

imap/global.c

@@ -409,6 +409,8 @@ EXPORTED int cyrus_init(const char *alt_config, const char *ident, unsigned flag
                                   config_getswitch(IMAPOPT_SKIPLIST_ALWAYS_CHECKPOINT));
         libcyrus_config_setswitch(CYRUSOPT_ACL_ADMIN_IMPLIES_WRITE,
                                   config_getswitch(IMAPOPT_ACL_ADMIN_IMPLIES_WRITE));
+        libcyrus_config_setswitch(CYRUSOPT_CYRUSDB_AUTOCONVERT,
+                                  config_getswitch(IMAPOPT_CYRUSDB_AUTOCONVERT));
 
         /* Not until all configuration parameters are set! */
         libcyrus_init();

lib/cyrusdb.c

@@ -57,6 +57,7 @@
 #include "assert.h"
 #include "bsearch.h"
 #include "cyrusdb.h"
+#include "cyr_lock.h"
 #include "util.h"
 #include "libcyr_cfg.h"
 #include "xmalloc.h"
@@ -108,30 +109,10 @@ static struct cyrusdb_backend *cyrusdb_fromname(const char *name)
     fatal(errbuf, EX_CONFIG);
 }
 
-static int _myopen(const char *backend, const char *fname,
-                 int flags, struct db **ret, struct txn **tid)
+static int _detect_or_convert(struct db *db, const char *backend,
+                              const char *fname, int flags)
 {
-    const char *realname;
-    struct db *db = xzmalloc(sizeof(struct db));
-    int r;
-
-    if (!backend) backend = DEFAULT_BACKEND; /* not used yet, later */
-    db->backend = cyrusdb_fromname(backend);
-
-    /* Check if shared lock is requested */
-    if (flags & CYRUSDB_SHARED) {
-        assert(tid && *tid == NULL);
-        if (flags & CYRUSDB_CONVERT) {
-            xsyslog(LOG_ERR,
-                    "DBERROR: CONVERT and SHARED are mutually exclusive,"
-                        " won't open db",
-                    "fname=<%s> backend=<%s>",
-                    fname, backend);
-            r = CYRUSDB_INTERNAL;
-            goto done;
-        }
-    }
-
+    int r = 0;
     /* This whole thing is a fricking critical section.  We don't have the API
      * in place for a safe rename of a locked database, so the choices are
      * basically:
@@ -142,29 +123,43 @@ static int _myopen(const char *backend, const char *fname,
      *    We do that.
      */
 
-    /* check if it opens normally.  Horray */
-    r = db->backend->open(fname, flags, &db->engine, tid);
-    if (r == CYRUSDB_NOTFOUND) goto done; /* no open flags */
-    if (!r) goto done;
+    // make sure we have a lock file fd
+    if (cyrusdb_convertlock_fd < 0) {
+        struct buf namebuf = BUF_INITIALIZER;
+        const char *config_dir = libcyrus_config_getstring(CYRUSOPT_CONFIG_DIR);
+        buf_printf(&namebuf, "%s/lock/cyrusdb_convert.lock", config_dir);
+        const char *fname = buf_cstring(&namebuf);
+        cyrusdb_convertlock_fd = open(fname, O_CREAT | O_TRUNC | O_RDWR, 0666);
+        if (cyrusdb_convertlock_fd < 0) {
+            if (!cyrus_mkdir(fname, 0755)) {
+                cyrusdb_convertlock_fd = open(fname, O_CREAT | O_TRUNC | O_RDWR, 0666);
+            }
+        }
+        buf_free(&namebuf);
+        if (cyrusdb_convertlock_fd < 0)
+            return CYRUSDB_IOERROR;
+    }
+
+    // lock the lock file
+    if (lock_setlock(cyrusdb_convertlock_fd, 1, 0, "cyrusdb_convert.lock"))
+        return CYRUSDB_IOERROR;
 
     /* magic time - we need to work out if the file was created by a different
      * backend and convert if possible */
-
-    realname = cyrusdb_detect(fname);
+    const char *realname = cyrusdb_detect(fname);
     if (!realname) {
         xsyslog(LOG_ERR, "DBERROR: failed to detect DB type",
-                         "fname=<%s> backend=<%s> r=<%d>",
-                         fname, backend, r);
-        /* r is still set */
-        goto done;
+                         "fname=<%s> backend=<%s>",
+                         fname, backend);
+        r = CYRUSDB_IOERROR;
     }
 
     /* different type */
-    if (strcmp(realname, backend)) {
-        if (flags & CYRUSDB_CONVERT) {
+    if (!r && strcmp(realname, backend)) {
+        if (flags & CYRUSDB_CONVERT || libcyrus_config_getswitch(CYRUSOPT_CYRUSDB_AUTOCONVERT)) {
             r = cyrusdb_convert(fname, fname, realname, backend);
             if (r) {
-                xsyslog(LOG_ERR, "DBERROR: failed to convert, maybe someone beat us",
+                xsyslog(LOG_ERR, "DBERROR: failed to convert",
                                  "fname=<%s> from=<%s> to=<%s>",
                                  fname, realname, backend);
             }
@@ -180,6 +175,42 @@ static int _myopen(const char *backend, const char *fname,
         }
     }
 
+    lock_unlock(cyrusdb_convertlock_fd, "cyrusdb_convert.lock");
+
+    return r;
+}
+
+static int _myopen(const char *backend, const char *fname,
+                 int flags, struct db **ret, struct txn **tid)
+{
+    struct db *db = xzmalloc(sizeof(struct db));
+    int r = 0;
+
+    if (!backend) backend = DEFAULT_BACKEND; /* not used yet, later */
+    db->backend = cyrusdb_fromname(backend);
+
+    /* Check if shared lock is requested */
+    if (flags & CYRUSDB_SHARED) {
+        assert(tid && *tid == NULL);
+        if (flags & CYRUSDB_CONVERT) {
+            xsyslog(LOG_ERR,
+                    "DBERROR: CONVERT and SHARED are mutually exclusive,"
+                        " won't open db",
+                    "fname=<%s> backend=<%s>",
+                    fname, backend);
+            r = CYRUSDB_INTERNAL;
+            goto done;
+        }
+    }
+
+    /* check if it opens normally.  Horray */
+    r = db->backend->open(fname, flags, &db->engine, tid);
+    if (r == CYRUSDB_NOTFOUND) goto done; /* no open flags */
+    if (!r) goto done;
+
+    r = _detect_or_convert(db, backend, fname, flags);
+    if (r) goto done;
+
     r = db->backend->open(fname, flags, &db->engine, tid);
 
 #ifdef DEBUGDB
@@ -549,26 +580,6 @@ EXPORTED int cyrusdb_convert(const char *fromfname, const char *tofname,
     struct txn *totid = NULL;
     int r;
 
-    // make sure we have a lock file fd
-    if (cyrusdb_convertlock_fd < 0) {
-        struct buf namebuf = BUF_INITIALIZER;
-        buf_printf(&namebuf, "%s/lock/cyrusdb_convert.lock", config_dir);
-        const char *fname = buf_cstring(&namebuf);
-        cyrusdb_convertlock_fd = open(fname, O_CREAT | O_TRUNC | O_RDWR, 0666);
-        if (cyrusdb_convertlock_fd < 0) {
-            if (!cyrus_mkdir(fname, 0755))
-                cyrusdb_convertlock_fd = open(fname, O_CREAT | O_TRUNC | O_RDWR, 0666);
-            }
-        }
-        buf_free(&namebuf);
-    }
-    if (cyrusdb_convertlock_fd < 0)
-        return CYRUSDB_IOERROR;
-
-    // lock the lock file
-    if (lock_setlock(cyrusdb_convertlock_fd, LOCK_EXCLUSIVE, 0, "cyrusdb_convert.lock"))
-        return CYRUSDB_IOERROR;
-
     /* open source database */
     r = cyrusdb_open(frombackend, fromfname, 0, &fromdb);
     if (r) goto err;
@@ -610,8 +621,6 @@ EXPORTED int cyrusdb_convert(const char *fromfname, const char *tofname,
 
     free(newfname);
 
-    lock_unlock(cyrusdb_convertlock_fd, "cyrusdb_convert.lock");
-
     return 0;
 
 err:
@@ -623,8 +632,6 @@ EXPORTED int cyrusdb_convert(const char *fromfname, const char *tofname,
     xunlink(tofname);
     free(newfname);
 
-    lock_unlock(cyrusdb_convertlock_fd, "cyrusdb_convert.lock");
-
     return r;
 }
 

lib/imapoptions

@@ -762,6 +762,10 @@ Blank lines and lines beginning with ``#'' are ignored.
 /* only show the domain for users in other domains than your own (for
    backwards compatibility if you're already sharing. */
 
+{ "cyrusdb_autoconvert", 0, SWITCH, "UNRELEASED" }
+/* automatically convert any existing db to the configured engine for that
+   database if possible. */
+
 { "cyrus_group", NULL, STRING, "3.1.7" }
 /* The name of the group Cyrus services will run as.  If not configured, the
    primary group of cyrus_user will be used. Can be further overridden by

lib/libcyr_cfg.c

@@ -161,6 +161,10 @@ static struct cyrusopt_s cyrus_options[] = {
       CFGVAL(long, 0),
       CYRUS_OPT_SWITCH },
 
+    { CYRUSOPT_CYRUSDB_AUTOCONVERT,
+      CFGVAL(long, 0),
+      CYRUS_OPT_SWITCH },
+
     { CYRUSOPT_LAST, { NULL }, CYRUS_OPT_NOTOPT }
 };
 

lib/libcyr_cfg.h

@@ -107,6 +107,8 @@ enum cyrus_opt {
     CYRUSOPT_SKIPLIST_ALWAYS_CHECKPOINT,
     /* ACL override */
     CYRUSOPT_ACL_ADMIN_IMPLIES_WRITE,
+    /* cyrusdb autoconvert */
+    CYRUSOPT_CYRUSDB_AUTOCONVERT,
 
     CYRUSOPT_LAST