Twom database format and first implementation

Makefile.am

@@ -805,6 +805,7 @@ include_HEADERS = \
     lib/stristr.h \
     lib/times.h \
     lib/tok.h \
+    lib/twom.h \
     lib/wildmat.h \
     lib/xmalloc.h \
     lib/xsha1.h \
@@ -1473,6 +1474,7 @@ lib_libcyrus_la_SOURCES = \
     lib/cyrusdb_quotalegacy.c \
     lib/cyrusdb_skiplist.c \
     lib/cyrusdb_twoskip.c \
+    lib/cyrusdb_twom.c \
     lib/glob.c \
     lib/htmlchar.c \
     lib/htmlchar.h \
@@ -1495,6 +1497,7 @@ lib_libcyrus_la_SOURCES = \
     lib/signals.c \
     lib/stristr.c \
     lib/times.c \
+    lib/twom.c \
     lib/wildmat.c
 if USE_CYRUSDB_SQL
 lib_libcyrus_la_SOURCES += lib/cyrusdb_sql.c

bench/cyrdbbench.c

@@ -368,12 +368,12 @@ static size_t do_write(int txnmode, int insmode)
     return bytes;
 }
 
-static int parse_options(int argc, char **argv, const struct option *options)
+static int parse_options(int argc, char **argv, const struct option *options __attribute__((unused)))
 {
     int option;
     int option_index;
 
-    while ((option = getopt_long(argc, argv, "d:b:t:h?",
+    while ((option = getopt_long(argc, argv, "d:b:n:t:h?",
                                  long_options, &option_index)) != -1) {
         switch (option) {
             case 'b':
@@ -480,11 +480,12 @@ int main(int argc, char *argv[])
         goto done;
     }
 
-    if (strncmp(BACKEND, "twoskip", strlen("twoskip")) == 0) {
+    if (strncmp(BACKEND, "twoskip", strlen("twoskip")) == 0 ||
+        strncmp(BACKEND, "twom", strlen("twom")) == 0) {
         fprintf(stderr, "Running benchmarks for `%s` backend\n", BACKEND);
     } else {
         fprintf(stderr, "%s is not a valid CyrusDB backend. ", BACKEND);
-        fprintf(stderr, "Only `twoskip` is supported.\n");
+        fprintf(stderr, "Choose between `twom` or `zeroskip`.\n");
         ret = EXIT_FAILURE;
         goto done;
     }

changes/next/twom

@@ -0,0 +1,30 @@
+Description:
+
+twom: a new cyrusdb backend
+
+
+Config changes:
+
+A new backend called 'twom' is added to most of the database engine options
+in imapd.conf.  Twom is a database format similar to twoskip, with some
+performance improvements.
+
+There's also a new boolean option 'cyrusdb_autoconvert', which - if set to
+true, will cause Cyrus to convert the database on first access if the
+existing file can be detected (by looking at the first few bytes for file
+magic).  This works for skiplist, twoskip and twom source formats.  The
+conversions are performed under a global lock, so this can slow the server
+down somewhat while the conversions are happening.
+
+
+Upgrade instructions:
+
+No changes are required if you don't want to switch to twom.  No defaults have
+been changed at this time.  If you do want to change, you can optionally set
+cyrusdb_autoconvert (otherwise watch the logs for files being opened with the
+wrong format and convert them manually), and also update the DB engine for
+individual database types.
+
+GitHub issue:
+
+No issue number.

cunit/aaa-db.testc

@@ -19,7 +19,7 @@ struct binary_result
     size_t datalen;
 };
 
-static const char *backend = CUNIT_PARAM("skiplist,flat,twoskip");
+static const char *backend = CUNIT_PARAM("skiplist,flat,twom,twoskip");
 static char *filename;
 static char *filename2;
 
@@ -82,6 +82,15 @@ static char *make_basedir(const char * const *reldirs)
     CU_ASSERT_EQUAL(r, CYRUSDB_OK); \
     CU_ASSERT_PTR_NOT_NULL(txn);
 
+#define CANDELETE(key, keylen) \
+    r = cyrusdb_delete(db, key, keylen, &txn, 1); \
+    CU_ASSERT_EQUAL(r, CYRUSDB_OK); \
+    CU_ASSERT_PTR_NOT_NULL(txn);
+
+#define ISCONSISTENT() \
+    r = cyrusdb_consistent(db); \
+    CU_ASSERT_EQUAL(r, CYRUSDB_OK);
+
 #define BADDATA         ((const char *)0xdeadbeef)
 #define BADLEN          ((int)0xcafebabe)
 
@@ -150,6 +159,18 @@ static char *make_basedir(const char * const *reldirs)
     CU_ASSERT_NOT_EQUAL(_datalen, BADLEN); \
 }
 
+#define CANNOTFETCH_NOTXN(key, keylen, experror) \
+{ \
+    const char *_data = BADDATA; \
+    size_t _datalen = BADLEN; \
+    r = cyrusdb_fetch(db, key, keylen, &_data, &_datalen, NULL); \
+    CU_ASSERT_EQUAL(r, experror); \
+    CU_ASSERT_PTR_NULL(_data); \
+    CU_ASSERT_PTR_NOT_EQUAL(_data, BADDATA); \
+    CU_ASSERT_EQUAL(_datalen, 0); \
+    CU_ASSERT_NOT_EQUAL(_datalen, BADLEN); \
+}
+
 #define CANNOTFETCHNEXT(key, keylen, experror) \
 { \
     r = cyrusdb_fetchnext(db, key, keylen, NULL, 0, NULL, 0, &txn); \
@@ -311,6 +332,117 @@ static void test_multiopen(void)
     CU_ASSERT_EQUAL(fexists(filename), 0);
 }
 
+static void test_read_and_delete(void)
+{
+    struct db *db = NULL;
+    struct txn *txn = NULL;
+    int r;
+    /* data courtesy hipsteripsum.me */
+    static const char KEY1[] = "mustache";
+    static const char DATA1[] = "blog lomo";
+    static const char KEY2[] = "cred";
+    static const char DATA2[] = "beard ethical";
+    static const char KEY3[] = "leggings";
+    static const char DATA3[] = "tumblr salvia";
+    static const char KEY3CHILD[] = "leggings.biodiesel";
+    static const char KEY4[] = "occupy";
+    static const char DATA4[] = "etsy tote bag";
+
+    CU_ASSERT_EQUAL(fexists(filename), -ENOENT);
+
+    /* open() with _CREATE succeeds and creates the db */
+    r = cyrusdb_open(backend, filename, CYRUSDB_CREATE, &db);
+    CU_ASSERT_EQUAL(r, CYRUSDB_OK);
+    CU_ASSERT_PTR_NOT_NULL(db);
+    CU_ASSERT_EQUAL(fexists(filename), 0);
+
+    /* 1st txn starts */
+    CANSTORE(KEY1, strlen(KEY1), DATA1, strlen(DATA1));
+    CANSTORE(KEY2, strlen(KEY2), DATA2, strlen(DATA2));
+    CANSTORE(KEY3, strlen(KEY3), DATA3, strlen(DATA3));
+    CANSTORE(KEY4, strlen(KEY4), DATA4, strlen(DATA4));
+    CANCOMMIT();
+    ISCONSISTENT();
+    /* 1st txn ends */
+
+    /* 2nd txn starts - try to fetch the immediately preceding value */
+    CANNOTFETCH(KEY3CHILD, strlen(KEY3CHILD), CYRUSDB_NOTFOUND);
+    CANFETCH(KEY3, strlen(KEY3), DATA3, strlen(DATA3));
+    CANDELETE(KEY3, strlen(KEY3));
+    CANFETCH(KEY1, strlen(KEY1), DATA1, strlen(DATA1));
+    CANDELETE(KEY1, strlen(KEY1));
+    CANCOMMIT();
+    ISCONSISTENT();
+
+    /* what is left? */
+    CANNOTFETCH_NOTXN(KEY1, strlen(KEY1), CYRUSDB_NOTFOUND);
+    CANFETCH_NOTXN(KEY2, strlen(KEY2), DATA2, strlen(DATA2));
+    CANNOTFETCH_NOTXN(KEY3, strlen(KEY3), CYRUSDB_NOTFOUND);
+    CANFETCH_NOTXN(KEY4, strlen(KEY4), DATA4, strlen(DATA4));
+    ISCONSISTENT();
+
+    /* closing succeeds and leaves the file in place */
+    r = cyrusdb_close(db);
+    CU_ASSERT_EQUAL(r, CYRUSDB_OK);
+    CU_ASSERT_EQUAL(fexists(filename), 0);
+}
+
+static void test_replace_before_delete(void)
+{
+    struct db *db = NULL;
+    struct txn *txn = NULL;
+    int r;
+    /* data courtesy hipsteripsum.me */
+    static const char KEY1[] = "alphabet";
+    static const char DATA1[] = "blog lomo";
+    static const char KEY2[] = "blanket";
+    static const char DATA2[] = "beard ethical";
+    static const char KEY3[] = "cobra";
+    static const char DATA3[] = "prius toke";
+    static const char KEY4[] = "dynamo";
+    static const char DATA4[] = "etsy tote bag";
+
+    CU_ASSERT_EQUAL(fexists(filename), -ENOENT);
+
+    /* open() with _CREATE succeeds and creates the db */
+    r = cyrusdb_open(backend, filename, CYRUSDB_CREATE, &db);
+    CU_ASSERT_EQUAL(r, CYRUSDB_OK);
+    CU_ASSERT_PTR_NOT_NULL(db);
+    CU_ASSERT_EQUAL(fexists(filename), 0);
+
+    /* 1st txn */
+    CANSTORE(KEY1, strlen(KEY1), DATA1, strlen(DATA1));
+    CANSTORE(KEY3, strlen(KEY3), DATA3, strlen(DATA3));
+    CANSTORE(KEY4, strlen(KEY4), DATA4, strlen(DATA4));
+    CANCOMMIT();
+    ISCONSISTENT();
+
+    /* 2nd txn */
+    CANDELETE(KEY3, strlen(KEY3));
+    CANCOMMIT();
+    ISCONSISTENT();
+
+    /* 3rd txn */
+    CANSTORE(KEY2, strlen(KEY2), DATA3, strlen(DATA3));
+    CANCOMMIT();
+    ISCONSISTENT();
+
+    /* 4th txn (new value) */
+    CANSTORE(KEY2, strlen(KEY2), DATA2, strlen(DATA2));
+    CANCOMMIT();
+    ISCONSISTENT();
+
+    CANFETCH_NOTXN(KEY1, strlen(KEY1), DATA1, strlen(DATA1));
+    CANFETCH_NOTXN(KEY2, strlen(KEY2), DATA2, strlen(DATA2));
+    CANNOTFETCH_NOTXN(KEY3, strlen(KEY3), CYRUSDB_NOTFOUND);
+    CANFETCH_NOTXN(KEY4, strlen(KEY4), DATA4, strlen(DATA4));
+
+    /* closing succeeds and leaves the file in place */
+    r = cyrusdb_close(db);
+    CU_ASSERT_EQUAL(r, CYRUSDB_OK);
+    CU_ASSERT_EQUAL(fexists(filename), 0);
+}
+
 static void test_opentwo(void)
 {
     struct db *db1 = NULL;