Support MySQL backend

Bug: T242055

.travis.yml

@@ -14,15 +14,30 @@ notifications:
 addons:
   apt:
     packages:
-    - openjdk-8-jre-headless
+      - openjdk-8-jre-headless
+
+services:
+  - mysql
 
 env:
   - CASSANDRA_VERSION=3.11.4 TEST_TARGET=sqlite TEST_MODE=fs
   - CASSANDRA_VERSION=3.11.4 TEST_TARGET=cassandra TEST_MODE=fs
+  - CASSANDRA_VERSION=3.11.4 TEST_TARGET=mysql TEST_MODE=fs
   - CASSANDRA_VERSION=3.11.4 TEST_TARGET=sqlite TEST_MODE=fefs
   - CASSANDRA_VERSION=3.11.4 TEST_TARGET=cassandra TEST_MODE=fefs
+  - CASSANDRA_VERSION=3.11.4 TEST_TARGET=mysql TEST_MODE=fefs
   - CASSANDRA_VERSION=3.11.4 TEST_TARGET=sqlite TEST_MODE=febe
   - CASSANDRA_VERSION=3.11.4 TEST_TARGET=cassandra TEST_MODE=febe
+  - CASSANDRA_VERSION=3.11.4 TEST_TARGET=mysql TEST_MODE=febe
+
+matrix:
+  allow_failures:
+    - node_js: "6"
+      env: CASSANDRA_VERSION=3.11.4 TEST_TARGET=mysql TEST_MODE=fs
+    - node_js: "6"
+      env: CASSANDRA_VERSION=3.11.4 TEST_TARGET=mysql TEST_MODE=fefs
+    - node_js: "6"
+      env: CASSANDRA_VERSION=3.11.4 TEST_TARGET=mysql TEST_MODE=febe
 
 before_install:
   - wget https://archive.apache.org/dist/cassandra/${CASSANDRA_VERSION}/apache-cassandra-${CASSANDRA_VERSION}-bin.tar.gz -P ../
@@ -32,6 +47,11 @@ before_install:
   - echo "-Xms256m" >> ../apache-cassandra-${CASSANDRA_VERSION}/conf/jvm.options
   - echo "-Xmx512m" >> ../apache-cassandra-${CASSANDRA_VERSION}/conf/jvm.options
   - sed -i -e 's/^-XX:+UseNUMA/#-XX:+UseNUMA/' ../apache-cassandra-${CASSANDRA_VERSION}/conf/jvm.options
-  - if [[ ${TEST_TARGET} == 'cassandra' ]]; then bash -x ../apache-cassandra-${CASSANDRA_VERSION}/bin/cassandra; else /bin/true; fi
+  - if [[ ${TEST_TARGET} == 'cassandra' ]]; then bash -x ../apache-cassandra-${CASSANDRA_VERSION}/bin/cassandra;
+    elif [[ ${TEST_TARGET} == 'mysql' ]]; then
+      mysql -uroot -e "CREATE DATABASE \`test_db\`;";
+      mysql -uroot -e "CREATE USER 'mysql'@\`%\` IDENTIFIED BY 'mysql';";
+      mysql -uroot -e "GRANT ALL PRIVILEGES ON \`test_db\`.* TO 'mysql'@\`%\`;";
+    else /bin/true; fi
 
 script: npm run lint && npm run coverage -- ${TEST_TARGET} ${TEST_MODE} && (npm run-script coveralls || exit 0)

config.backend.test.yaml

@@ -28,14 +28,15 @@ default_project: &default_project
                               backend: '{env(RB_TEST_BACKEND, sqlite)}'
                               hosts: [localhost]
                               keyspace: system
-                              username: cassandra
-                              password: cassandra
+                              username: '{env(RB_TEST_BACKEND_USERNAME, cassandra)}'
+                              password: '{env(RB_TEST_BACKEND_PASSWORD, cassandra)}'
                               defaultConsistency: one # or 'localQuorum' for production
                               storage_groups:
                                 - name: test.group.local
                                   domains: /./
-                              # ignored in cassandra, but useful in SQLite testing
-                              dbname: '{env(RB_SQLITE_FILE, test.db.sqlite3)}'
+                              host: localhost # ignored in cassandra, but useful in MySQL testing
+                              database: test_db # ignored in cassandra, but useful in MySQL testing
+                              dbname: '{env(RB_SQLITE_FILE, test.db.sqlite3)}' # ignored in cassandra, but useful in SQLite testing
                     /action:
                       x-modules:
                         - path: sys/action.js

config.fullstack.test.yaml

@@ -71,14 +71,15 @@ default_project: &default_project
                               backend: '{env(RB_TEST_BACKEND, sqlite)}'
                               hosts: [localhost]
                               keyspace: system
-                              username: cassandra
-                              password: cassandra
+                              username: '{env(RB_TEST_BACKEND_USERNAME, cassandra)}'
+                              password: '{env(RB_TEST_BACKEND_PASSWORD, cassandra)}'
                               defaultConsistency: one # or 'localQuorum' for production
                               storage_groups:
                                 - name: test.group.local
                                   domains: /./
-                              # ignored in cassandra, but useful in SQLite testing
-                              dbname: '{env(RB_SQLITE_FILE, test.db.sqlite3)}'
+                              host: localhost # ignored in cassandra, but useful in MySQL testing
+                              database: test_db # ignored in cassandra, but useful in MySQL testing
+                              dbname: '{env(RB_SQLITE_FILE, test.db.sqlite3)}' # ignored in cassandra, but useful in SQLite testing
 
 wikimedia_project: &wikimedia_project
   x-modules:

package.json

@@ -25,6 +25,7 @@
     "tables",
     "queues",
     "cassandra",
+    "mysql",
     "kafka"
   ],
   "author": "Wikimedia Service Team <[email protected]>",
@@ -43,6 +44,7 @@
     "jsonwebtoken": "^8.5.1",
     "mediawiki-title": "^0.7.2",
     "restbase-mod-table-cassandra": "^1.2.1",
+    "restbase-mod-table-mysql": "^0.1.1",
     "semver": "latest",
     "service-runner": "^2.7.3",
     "uuid": "^3.3.3"

sys/table.js

@@ -9,8 +9,11 @@ module.exports = (options) => {
     options.conf.backend = options.conf.backend || 'cassandra';
     options.log = options.logger.log.bind(options.logger);
 
-    if (options.conf.backend !== 'cassandra' &&
-            options.conf.backend !== 'sqlite') {
+    if (
+        options.conf.backend !== 'cassandra' &&
+        options.conf.backend !== 'mysql' &&
+        options.conf.backend !== 'sqlite'
+    ) {
         throw new Error(`Unsupported backend version specified: ${options.conf.backend}`);
     }
 

test/utils/cleandb.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+rb_test_backend=${RB_TEST_BACKEND:-$2}
+
 dropKeyspaces ( ) {
   if [ "$#" -eq 1 ]
   then
@@ -13,4 +15,26 @@ dropKeyspaces ( ) {
   fi
 }
 
-dropKeyspaces "local_group_test"
+dropTables ( ) {
+  if [ "$#" -eq 1 ]
+  then
+    DATABASE=$1
+    echo "looking for database named '*$DATABASE*'..."
+    echo 'begin;' | mysql -BD${DATABASE}
+    for TABLE in `echo 'show tables;' | mysql -BD${DATABASE}`
+    do
+      echo dropping table $TABLE
+      echo "drop table if exists $TABLE;" | mysql -BD${DATABASE}
+    done
+    echo 'commit;' | mysql -BD${DATABASE}
+  fi
+}
+
+if [ "$rb_test_backend" = "cassandra" ]; then
+  dropKeyspaces "local_group_test"
+elif [ "$rb_test_backend" = "mysql" ]; then
+  dropTables "test_db"
+else
+    echo "Invalid TEST_TARGET $rb_test_backend. Must me 'sqlite', 'cassandra' or 'mysql' if specified"
+    exit 1
+fi

test/utils/run_tests.sh

@@ -33,9 +33,24 @@ elif [ "$test_target" = "cassandra" ]; then
       echo "Cassandra is ready."
     fi
     export RB_TEST_BACKEND=cassandra
+    export RB_TEST_BACKEND_USERNAME=cassandra
+    export RB_TEST_BACKEND_PASSWORD=cassandra
+    sh ./test/utils/cleandb.sh local_group_test
+elif [ "$test_target" = "mysql" ]; then
+    echo "Running with MySQL backend"
+    if [ `nc -z localhost 3306 < /dev/null; echo $?` != 0 ]; then
+      echo "Waiting for MySQL to start..."
+      while [ `nc -z localhost 3306; echo $?` != 0 ]; do
+        sleep 1
+      done
+      echo "MySQL is ready."
+    fi
+    export RB_TEST_BACKEND=mysql
+    export RB_TEST_BACKEND_USERNAME=mysql
+    export RB_TEST_BACKEND_PASSWORD=mysql
     sh ./test/utils/cleandb.sh local_group_test
 else
-    echo "Invalid TEST_TARGET $test_target. Must me 'sqlite' or 'cassandra' if specified"
+    echo "Invalid TEST_TARGET $test_target. Must me 'sqlite', 'cassandra' or 'mysql' if specified"
     exit 1
 fi