GPII-2966 Initial commit of WIP migration utility

documentation/ChangeLogSchemas.md

@@ -0,0 +1,42 @@
+# GPII Schemas Change Log
+
+This change log describes all changes made to the CouchDB document based data schemas for the
+GPII. If applicable a ticket documenting the work done for each change is included. Additionally,
+a git commit hash is listed, which can be used to checkout a version of the code at that schema
+for reference.
+
+## 0.3 2019-07-XX GPII-2966
+
+- Ticket: [GPII-2966](https://issues.gpii.net/browse/GPII-2966)
+- Git Commit: pending
+
+This work increments the `schemaVersion` for all GPII documents to version `0.3`. For documents
+of type `prefsSafe` it removes deprecated field `password` from the document.
+
+This ticket also introduces a new document type `gpiiCloudSafeCredential`, however given it has
+no previous occurances in our datastore and is not required for previously existing data, no
+migrations are included for it.
+
+## 0.2 2019-06-XX GPII-3717
+
+- Ticket: [GPII-3717](https://issues.gpii.net/browse/GPII-3717)
+- Git Commit: pending
+
+This work increments the `schemaVersion` for all GPII documents to version `0.2`.  For documents
+of type `clientCredential` it adds four new fields with the following defaults:
+
+```json
+{
+"allowedIPBlocks": null,
+"allowedPrefsToWrite": null,
+"isCreateGpiiKeyAllowed": false,
+"isCreatePrefsSafeAllowed": false
+}
+```
+
+## 0.1 Pre June 2019
+
+- Git Commit: 9bd021f95b3fe64a6a9d1fdcd18b8e8044007187
+
+Previously throughout the project, and prior to production usage and datastores, the `schemaVersion`
+field was kept at 0.1 for all changes and modifications.

documentation/Migrations.md

@@ -0,0 +1,5 @@
+# GPII Migrations Framework
+
+This document describes our micro framework for performing migrations on existing documents
+in our CouchDB database. The need to perform a migration could result from something as simple
+as removing a document field, to a more advanced restructing of hierarchical data in a record.

gpii/node_modules/gpii-migrations/index.js

@@ -0,0 +1,8 @@
+"use strict";
+
+var fluid = require("infusion");
+
+fluid.module.register("gpii-migrations", __dirname, require);
+
+require("./src/migrations.js");
+require("./src/migrationsUtils.js");

gpii/node_modules/gpii-migrations/src/migrations.js

@@ -0,0 +1,317 @@
+/*!
+Copyright 2019 Raising The Floor - US
+
+Licensed under the New BSD license. You may not use this file except in
+compliance with this License.
+
+You may obtain a copy of the License at
+https://github.com/GPII/universal/blob/master/LICENSE.txt
+*/
+
+"use strict";
+
+var fluid = require("infusion"),
+    request = require("request");
+
+var gpii = fluid.registerNamespace("gpii");
+fluid.registerNamespace("gpii.migrations");
+
+fluid.setLogging(fluid.logLevel.INFO);
+
+/**
+ * `gpii.migrations.mangoQuery`
+ *
+ * Using a CouchDB URI and full Mango style query, connects to the CouchDB
+ * instance, performs the query, and returns the results in a promise.
+ *
+ * @param {Object} options - Options block
+ * @param {String} options.couchDbUri - The CouchDB URI including, the specific
+ * database to use. ex `http://localhost:5984/gpii`.
+ * @param {Object} options.mangoQuery - Full mango JSON query. This can use
+ * any of the options allowed by CouchDB. Mango documentation can be found
+ * `https://docs.couchdb.org/en/stable/api/database/find.html`
+ * @return {fluid.promise} Promise resolving with the full return payload from
+ * CouchDB.
+ */
+gpii.migrations.mangoQuery = function (options) {
+    var togo = fluid.promise();
+    request({
+        method: "POST",
+        uri: options.couchDbUri + "/_find",
+        json: true,
+        body: options.mangoQuery
+    }, function (err, res, body) {
+        if (err) {
+            togo.reject({
+                isError: true,
+                message: err
+            });
+        }
+        else {
+            togo.resolve(body);
+        }
+    });
+    return togo;
+};
+
+/**
+ * `gpii.migrations.nextDocument`
+ *
+ * This function takes the results from a CouchDB Mango query
+ * and selects the next document for the migration to process,
+ * returning that in a promise.
+ *
+ * If the data contains no more records to migrate, a `finishedMigration`
+ * event is given to be fired.
+ *
+ * @param {Object} data - Results from a CouchDB Mango query
+ * @param {Event} finishedMigration - Event that can be fired if no
+ * further documents need to be migrated based on the query results.
+ * @return {fluid.promise} Promise resolving with the next document to
+ * migrate.
+ */
+gpii.migrations.nextDocument = function (data, finishedMigration) {
+    var prom = fluid.promise();
+    fluid.log("Remaining number to migrate: ", data.docs.length);
+    if (data.docs.length === 0) {
+        finishedMigration.fire("No more documents to migrate.");
+        prom.reject({
+            message: "No more documents to migrate."
+        });
+    }
+    else {
+        prom.resolve(data.docs[0]);
+    }
+    return prom;
+};
+
+/**
+ * `gpii.migrations.nextBulkDocument`
+ *
+ * This function takes the results from a CouchDB Mango query
+ * and transforms each document using the supplied `processDoc` function.
+ * This array of transformed documents is used to resolve the returned
+ * promise. This array of documents is destined for a CouchDB bulk update.
+ *
+ * If the data contains no more records to migrate, a `finishedMigration`
+ * event is given to be fired.
+ *
+ * @param {Object} data - Results from a CouchDB Mango query
+ * @param {Event} finishedMigration - Event that can be fired if no
+ * further documents need to be migrated based on the query results.
+ * @param {Function} processDoc - Function used to transform the document
+ * for this migration.
+ * @return {fluid.promise} Promise resolving with the next document to
+ * migrate.
+ */
+gpii.migrations.nextBulkDocument = function (data, finishedMigration, processDoc) {
+    var prom = fluid.promise();
+    fluid.log("BULK: Remaining number to migrate: ", data.docs.length);
+    if (data.docs.length === 0) {
+        finishedMigration.fire("No more documents to migrate.");
+        prom.reject({
+            message: "No more documents to migrate."
+        });
+    }
+    else {
+        var togo = [];
+        fluid.each(data.docs, function (doc) {
+            togo.push(processDoc(doc));
+        });
+        prom.resolve(togo);
+    }
+    return prom;
+};
+
+/**
+ * `gpii.migrations.updateDoc`
+ *
+ * This function takes a document that has been processed by the
+ * migration function `processDocument` and saves the updates back to CouchDB.
+ *
+ * @param {Object} options - Options block
+ * @param {String} options.couchDbUri - The CouchDB URI including, the specific
+ * database to use. ex `http://localhost:5984/gpii`.
+ * @param {Object} doc - The fully migrated CouchDB document (with original `_id`
+ * and `_rev` fields) to be saved back to CouchDB.
+ * @return {fluid.promise} Promise resolving with the return message from CouchDB.
+ */
+gpii.migrations.updateDoc = function (options, doc) {
+    var togo = fluid.promise();
+    request({
+        method: "PUT",
+        uri: options.couchDbUri + "/" + doc._id,
+        json: true,
+        body: doc
+    }, function (err, res, body) {
+        if (err) {
+            togo.reject({
+                isError: true,
+                message: err
+            });
+        }
+        else {
+            togo.resolve(body);
+        }
+    });
+    return togo;
+};
+
+/**
+ * `gpii.migrations.updateBulkDocs`
+ *
+ * This function takes an array of documents that have been processed by the
+ * migration function `processDocument` and saves the updates back to CouchDB.
+ *
+ * @param {Object} options - Options block
+ * @param {String} options.couchDbUri - The CouchDB URI including, the specific
+ * database to use. ex `http://localhost:5984/gpii`.
+ * @param {Object} docs - The fully migrated CouchDB document (with original `_id`
+ * and `_rev` fields) to be saved back to CouchDB.
+ * @return {fluid.promise} Promise resolving with the return message from CouchDB.
+ */
+gpii.migrations.updateBulkDocs = function (options, docs) {
+    var togo = fluid.promise();
+    request({
+        method: "POST",
+        uri: options.couchDbUri + "/_bulk_docs",
+        json: true,
+        body: {
+            "docs": docs
+        }
+    }, function (err, res, body) {
+        if (err) {
+            togo.reject({
+                isError: true,
+                message: err
+            });
+        }
+        else {
+            togo.resolve(body);
+        }
+    });
+    return togo;
+};
+
+/**
+ * `gpii.migrations.couchDBmigration`
+ *
+ * An infusion grade used as a basis for CouchDB Migration operations, which
+ * each instance acting as a single migration. In it's default configuration
+ * instances of this grade will perform a migration by:
+ *
+ * 1. Running a CouchDB Mango Query to select the next document, and then
+ * 2. migrating that document and saving it back to the database.
+ *
+ * The Mango query is specified in the component options under `mangoQuery`. This
+ * is a full query as described here `https://docs.couchdb.org/en/stable/api/database/find.html`
+ * and any of the options can be used.
+ *
+ * The URI to CouchDB, optionally including username/password in the URI is
+ * configured in the options under `couchDbUri`.
+ *
+ * Inheritors of this grade need to supply one invoker `processDocument`. This is a
+ * method that takes the next document and performs the necessary changes on it, such
+ * as removing or updating a field.
+ *
+ * Ideally migrations should be idemponent, such that they can be run repeatedly, across
+ * clusters, and be restarted in the case of failure or termination of their process. It's
+ * very possible however, that some future migration involving substantial restructuring
+ * of documents and related documents may not be idempotent. In that case, implementors
+ * should mark the `idemponent` option as `false`. This can be used to exclude the migration
+ * from any batch jobs that regularly rerun all available migrations.
+ *
+ * The listener chain `continueMigration` contains the steps to repeatedly follow until the
+ * migration is finished. Future configurations of the this chain could potentially
+ * contain functionality to process operations in batch, running `processDocument` and
+ * `updateDoc` on multiple documents using `fluid.sequence` before fetching the next batch
+ * of documents to migrate. Other more complex migration strategies are also possible.
+ *
+ * In the case of failure, the `continueMigration` chain will simply reject and the migration
+ * will stop.  Unless your migration is not idempotent, it should be possible to simply
+ * inspect the error and restart the migration. Future configurations could be created to
+ * simply log any errors, and continue migrating, skipping any documents that have failed
+ * on that run (should they show up again in the Mango query results).
+ *
+ * Another chain `continueBulkMigration` can be used in place of `continueMigration`.  Calling
+ * this chain will have the same end result, but documents will be sent using CouchDB's
+ * bulk docs endpoint, rather than processing each document separately. For large migrations
+ * this will result in significant speed up, however, you may find value in `continueMigration`
+ * as it will stop on the first occurance of any issues. Bulk document updates in Couch are
+ * not transactional, so in a large update request, only a few may fail while the rest succeed.
+ *
+ * In theory it should be possible to run migrations in parallel.  However, if two parallel
+ * jobs try to migrate the same document, the job with the older `_rev` ID will fail, causing
+ * it to stop.  If this functionality is required in the future, an option to explicitly
+ * ignore `_rev` update errors from CouchDB should be added.
+ *
+ */
+fluid.defaults("gpii.migrations.couchDBmigration", {
+    gradeNames: "fluid.component",
+    idempotent: true,
+    couchDbUri: "http://localhost:5984/gpii",
+    mangoQuery: {},
+    events: {
+        continueMigration: null,
+        continueBulkMigration: null,
+        finishedMigration: null
+    },
+    listeners: {
+        "continueMigration": [
+            {
+                listener: "gpii.migrations.mangoQuery",
+                args: ["{that}.options"]
+            },
+            {
+                listener: "gpii.migrations.nextDocument",
+                args: ["{arguments}.0", "{that}.events.finishedMigration"],
+                namespace: "nextDocument"
+            },
+            {
+                listener: "{that}.processDocument",
+                args: ["{arguments}.0"],
+                namespace: "processDocument"
+            },
+            {
+                listener: "gpii.migrations.updateDoc",
+                args: ["{that}.options", "{arguments}.0"],
+                namespace: "updateDoc"
+            },
+            {
+                listener: "fluid.promise.fireTransformEvent",
+                args: ["{that}.events.continueMigration"],
+                namespace: "startAnotherMigrationBatch"
+            }
+        ],
+        "continueBulkMigration": [
+            {
+                listener: "gpii.migrations.mangoQuery",
+                args: ["{that}.options"]
+            },
+            {
+                listener: "gpii.migrations.nextBulkDocument",
+                args: ["{arguments}.0", "{that}.events.finishedMigration", "{that}.processDocument"],
+                namespace: "nextBulkDocument"
+            },
+            {
+                listener: "gpii.migrations.updateBulkDocs",
+                args: ["{that}.options", "{arguments}.0"],
+                namespace: "updateDoc"
+            },
+            {
+                listener: "fluid.promise.fireTransformEvent",
+                args: ["{that}.events.continueBulkMigration"],
+                namespace: "startAnotherMigrationBatch"
+            }
+        ],
+        "finishedMigration": {
+            funcName: "fluid.log",
+            args: ["Finished migration: ", "{arguments}.0"]
+        }
+    },
+    invokers: {
+        processDocument: {
+            funcName: "fluid.notImplemented"
+        }
+    }
+});