diff --git a/documentation/ChangeLogSchemas.md b/documentation/ChangeLogSchemas.md new file mode 100644 index 000000000..03ce262a4 --- /dev/null +++ b/documentation/ChangeLogSchemas.md @@ -0,0 +1,42 @@ +# GPII Schemas Change Log + +This change log describes all changes made to the CouchDB document based data schemas for the +GPII. If applicable a ticket documenting the work done for each change is included. Additionally, +a git commit hash is listed, which can be used to checkout a version of the code at that schema +for reference. + +## 0.3 2019-07-XX GPII-2966 + +- Ticket: [GPII-2966](https://issues.gpii.net/browse/GPII-2966) +- Git Commit: pending + +This work increments the `schemaVersion` for all GPII documents to version `0.3`. For documents +of type `prefsSafe` it removes deprecated field `password` from the document. + +This ticket also introduces a new document type `gpiiCloudSafeCredential`, however given it has +no previous occurances in our datastore and is not required for previously existing data, no +migrations are included for it. + +## 0.2 2019-06-XX GPII-3717 + +- Ticket: [GPII-3717](https://issues.gpii.net/browse/GPII-3717) +- Git Commit: pending + +This work increments the `schemaVersion` for all GPII documents to version `0.2`. For documents +of type `clientCredential` it adds four new fields with the following defaults: + +```json +{ +"allowedIPBlocks": null, +"allowedPrefsToWrite": null, +"isCreateGpiiKeyAllowed": false, +"isCreatePrefsSafeAllowed": false +} +``` + +## 0.1 Pre June 2019 + +- Git Commit: 9bd021f95b3fe64a6a9d1fdcd18b8e8044007187 + +Previously throughout the project, and prior to production usage and datastores, the `schemaVersion` +field was kept at 0.1 for all changes and modifications. diff --git a/documentation/Migrations.md b/documentation/Migrations.md new file mode 100644 index 000000000..5ed9bba2e --- /dev/null +++ b/documentation/Migrations.md @@ -0,0 +1,5 @@ +# GPII Migrations Framework + +This document describes our micro framework for performing migrations on existing documents +in our CouchDB database. The need to perform a migration could result from something as simple +as removing a document field, to a more advanced restructing of hierarchical data in a record. diff --git a/gpii/node_modules/gpii-migrations/README.md b/gpii/node_modules/gpii-migrations/README.md new file mode 100644 index 000000000..e69de29bb diff --git a/gpii/node_modules/gpii-migrations/index.js b/gpii/node_modules/gpii-migrations/index.js new file mode 100644 index 000000000..899b3cbbd --- /dev/null +++ b/gpii/node_modules/gpii-migrations/index.js @@ -0,0 +1,8 @@ +"use strict"; + +var fluid = require("infusion"); + +fluid.module.register("gpii-migrations", __dirname, require); + +require("./src/migrations.js"); +require("./src/migrationsUtils.js"); diff --git a/gpii/node_modules/gpii-migrations/package.json b/gpii/node_modules/gpii-migrations/package.json new file mode 100644 index 000000000..9124888ed --- /dev/null +++ b/gpii/node_modules/gpii-migrations/package.json @@ -0,0 +1,13 @@ +{ + "name": "gpii-migrations", + "description": "Small framework and utilities for migrating CouchDB Document formats", + "version": "0.1.0", + "author": "GPII", + "bugs": "http://issues.gpii.net/browse/GPII", + "homepage": "http://gpii.net/", + "dependencies": {}, + "license" : "BSD-3-Clause", + "repository": "git://github.com/GPII/universal.git", + "main": "./index.js", + "engines": { "node" : ">=4.2.1" } +} diff --git a/gpii/node_modules/gpii-migrations/src/migrations.js b/gpii/node_modules/gpii-migrations/src/migrations.js new file mode 100644 index 000000000..e8b1e76a0 --- /dev/null +++ b/gpii/node_modules/gpii-migrations/src/migrations.js @@ -0,0 +1,317 @@ +/*! +Copyright 2019 Raising The Floor - US + +Licensed under the New BSD license. You may not use this file except in +compliance with this License. + +You may obtain a copy of the License at +https://github.com/GPII/universal/blob/master/LICENSE.txt +*/ + +"use strict"; + +var fluid = require("infusion"), + request = require("request"); + +var gpii = fluid.registerNamespace("gpii"); +fluid.registerNamespace("gpii.migrations"); + +fluid.setLogging(fluid.logLevel.INFO); + +/** + * `gpii.migrations.mangoQuery` + * + * Using a CouchDB URI and full Mango style query, connects to the CouchDB + * instance, performs the query, and returns the results in a promise. + * + * @param {Object} options - Options block + * @param {String} options.couchDbUri - The CouchDB URI including, the specific + * database to use. ex `http://localhost:5984/gpii`. + * @param {Object} options.mangoQuery - Full mango JSON query. This can use + * any of the options allowed by CouchDB. Mango documentation can be found + * `https://docs.couchdb.org/en/stable/api/database/find.html` + * @return {fluid.promise} Promise resolving with the full return payload from + * CouchDB. + */ +gpii.migrations.mangoQuery = function (options) { + var togo = fluid.promise(); + request({ + method: "POST", + uri: options.couchDbUri + "/_find", + json: true, + body: options.mangoQuery + }, function (err, res, body) { + if (err) { + togo.reject({ + isError: true, + message: err + }); + } + else { + togo.resolve(body); + } + }); + return togo; +}; + +/** + * `gpii.migrations.nextDocument` + * + * This function takes the results from a CouchDB Mango query + * and selects the next document for the migration to process, + * returning that in a promise. + * + * If the data contains no more records to migrate, a `finishedMigration` + * event is given to be fired. + * + * @param {Object} data - Results from a CouchDB Mango query + * @param {Event} finishedMigration - Event that can be fired if no + * further documents need to be migrated based on the query results. + * @return {fluid.promise} Promise resolving with the next document to + * migrate. + */ +gpii.migrations.nextDocument = function (data, finishedMigration) { + var prom = fluid.promise(); + fluid.log("Remaining number to migrate: ", data.docs.length); + if (data.docs.length === 0) { + finishedMigration.fire("No more documents to migrate."); + prom.reject({ + message: "No more documents to migrate." + }); + } + else { + prom.resolve(data.docs[0]); + } + return prom; +}; + +/** + * `gpii.migrations.nextBulkDocument` + * + * This function takes the results from a CouchDB Mango query + * and transforms each document using the supplied `processDoc` function. + * This array of transformed documents is used to resolve the returned + * promise. This array of documents is destined for a CouchDB bulk update. + * + * If the data contains no more records to migrate, a `finishedMigration` + * event is given to be fired. + * + * @param {Object} data - Results from a CouchDB Mango query + * @param {Event} finishedMigration - Event that can be fired if no + * further documents need to be migrated based on the query results. + * @param {Function} processDoc - Function used to transform the document + * for this migration. + * @return {fluid.promise} Promise resolving with the next document to + * migrate. + */ +gpii.migrations.nextBulkDocument = function (data, finishedMigration, processDoc) { + var prom = fluid.promise(); + fluid.log("BULK: Remaining number to migrate: ", data.docs.length); + if (data.docs.length === 0) { + finishedMigration.fire("No more documents to migrate."); + prom.reject({ + message: "No more documents to migrate." + }); + } + else { + var togo = []; + fluid.each(data.docs, function (doc) { + togo.push(processDoc(doc)); + }); + prom.resolve(togo); + } + return prom; +}; + +/** + * `gpii.migrations.updateDoc` + * + * This function takes a document that has been processed by the + * migration function `processDocument` and saves the updates back to CouchDB. + * + * @param {Object} options - Options block + * @param {String} options.couchDbUri - The CouchDB URI including, the specific + * database to use. ex `http://localhost:5984/gpii`. + * @param {Object} doc - The fully migrated CouchDB document (with original `_id` + * and `_rev` fields) to be saved back to CouchDB. + * @return {fluid.promise} Promise resolving with the return message from CouchDB. + */ +gpii.migrations.updateDoc = function (options, doc) { + var togo = fluid.promise(); + request({ + method: "PUT", + uri: options.couchDbUri + "/" + doc._id, + json: true, + body: doc + }, function (err, res, body) { + if (err) { + togo.reject({ + isError: true, + message: err + }); + } + else { + togo.resolve(body); + } + }); + return togo; +}; + +/** + * `gpii.migrations.updateBulkDocs` + * + * This function takes an array of documents that have been processed by the + * migration function `processDocument` and saves the updates back to CouchDB. + * + * @param {Object} options - Options block + * @param {String} options.couchDbUri - The CouchDB URI including, the specific + * database to use. ex `http://localhost:5984/gpii`. + * @param {Object} docs - The fully migrated CouchDB document (with original `_id` + * and `_rev` fields) to be saved back to CouchDB. + * @return {fluid.promise} Promise resolving with the return message from CouchDB. + */ +gpii.migrations.updateBulkDocs = function (options, docs) { + var togo = fluid.promise(); + request({ + method: "POST", + uri: options.couchDbUri + "/_bulk_docs", + json: true, + body: { + "docs": docs + } + }, function (err, res, body) { + if (err) { + togo.reject({ + isError: true, + message: err + }); + } + else { + togo.resolve(body); + } + }); + return togo; +}; + +/** + * `gpii.migrations.couchDBmigration` + * + * An infusion grade used as a basis for CouchDB Migration operations, which + * each instance acting as a single migration. In it's default configuration + * instances of this grade will perform a migration by: + * + * 1. Running a CouchDB Mango Query to select the next document, and then + * 2. migrating that document and saving it back to the database. + * + * The Mango query is specified in the component options under `mangoQuery`. This + * is a full query as described here `https://docs.couchdb.org/en/stable/api/database/find.html` + * and any of the options can be used. + * + * The URI to CouchDB, optionally including username/password in the URI is + * configured in the options under `couchDbUri`. + * + * Inheritors of this grade need to supply one invoker `processDocument`. This is a + * method that takes the next document and performs the necessary changes on it, such + * as removing or updating a field. + * + * Ideally migrations should be idemponent, such that they can be run repeatedly, across + * clusters, and be restarted in the case of failure or termination of their process. It's + * very possible however, that some future migration involving substantial restructuring + * of documents and related documents may not be idempotent. In that case, implementors + * should mark the `idemponent` option as `false`. This can be used to exclude the migration + * from any batch jobs that regularly rerun all available migrations. + * + * The listener chain `continueMigration` contains the steps to repeatedly follow until the + * migration is finished. Future configurations of the this chain could potentially + * contain functionality to process operations in batch, running `processDocument` and + * `updateDoc` on multiple documents using `fluid.sequence` before fetching the next batch + * of documents to migrate. Other more complex migration strategies are also possible. + * + * In the case of failure, the `continueMigration` chain will simply reject and the migration + * will stop. Unless your migration is not idempotent, it should be possible to simply + * inspect the error and restart the migration. Future configurations could be created to + * simply log any errors, and continue migrating, skipping any documents that have failed + * on that run (should they show up again in the Mango query results). + * + * Another chain `continueBulkMigration` can be used in place of `continueMigration`. Calling + * this chain will have the same end result, but documents will be sent using CouchDB's + * bulk docs endpoint, rather than processing each document separately. For large migrations + * this will result in significant speed up, however, you may find value in `continueMigration` + * as it will stop on the first occurance of any issues. Bulk document updates in Couch are + * not transactional, so in a large update request, only a few may fail while the rest succeed. + * + * In theory it should be possible to run migrations in parallel. However, if two parallel + * jobs try to migrate the same document, the job with the older `_rev` ID will fail, causing + * it to stop. If this functionality is required in the future, an option to explicitly + * ignore `_rev` update errors from CouchDB should be added. + * + */ +fluid.defaults("gpii.migrations.couchDBmigration", { + gradeNames: "fluid.component", + idempotent: true, + couchDbUri: "http://localhost:5984/gpii", + mangoQuery: {}, + events: { + continueMigration: null, + continueBulkMigration: null, + finishedMigration: null + }, + listeners: { + "continueMigration": [ + { + listener: "gpii.migrations.mangoQuery", + args: ["{that}.options"] + }, + { + listener: "gpii.migrations.nextDocument", + args: ["{arguments}.0", "{that}.events.finishedMigration"], + namespace: "nextDocument" + }, + { + listener: "{that}.processDocument", + args: ["{arguments}.0"], + namespace: "processDocument" + }, + { + listener: "gpii.migrations.updateDoc", + args: ["{that}.options", "{arguments}.0"], + namespace: "updateDoc" + }, + { + listener: "fluid.promise.fireTransformEvent", + args: ["{that}.events.continueMigration"], + namespace: "startAnotherMigrationBatch" + } + ], + "continueBulkMigration": [ + { + listener: "gpii.migrations.mangoQuery", + args: ["{that}.options"] + }, + { + listener: "gpii.migrations.nextBulkDocument", + args: ["{arguments}.0", "{that}.events.finishedMigration", "{that}.processDocument"], + namespace: "nextBulkDocument" + }, + { + listener: "gpii.migrations.updateBulkDocs", + args: ["{that}.options", "{arguments}.0"], + namespace: "updateDoc" + }, + { + listener: "fluid.promise.fireTransformEvent", + args: ["{that}.events.continueBulkMigration"], + namespace: "startAnotherMigrationBatch" + } + ], + "finishedMigration": { + funcName: "fluid.log", + args: ["Finished migration: ", "{arguments}.0"] + } + }, + invokers: { + processDocument: { + funcName: "fluid.notImplemented" + } + } +}); diff --git a/gpii/node_modules/gpii-migrations/src/migrationsUtils.js b/gpii/node_modules/gpii-migrations/src/migrationsUtils.js new file mode 100644 index 000000000..1273143e9 --- /dev/null +++ b/gpii/node_modules/gpii-migrations/src/migrationsUtils.js @@ -0,0 +1,211 @@ +/*! +Copyright 2018 Raising The Floor - US + +Licensed under the New BSD license. You may not use this file except in +compliance with this License. + +You may obtain a copy of the License at +https://github.com/GPII/universal/blob/master/LICENSE.txt +*/ + +"use strict"; + +var fluid = require("infusion"); + +var gpii = fluid.registerNamespace("gpii"); +fluid.registerNamespace("gpii.migrations.tests"); +fluid.registerNamespace("gpii.migrations.utils.tests"); + +var jqUnit = require("node-jqunit"); + +fluid.defaults("gpii.migrations.tests.simpleDocumentMigration", { + gradeNames: ["gpii.migrations.couchDBmigration"], + testFixtures: { + beforeMigration: "%gpii-universal/gpii/node_modules/gpii-migrations/test/data/simpleMigrationDocumentsBeforeFixture.json", + afterMigration: "@expand:fluid.module.resolvePath(%gpii-universal/gpii/node_modules/gpii-migrations/test/data/simpleMigrationDocumentsAfterFixture.json)" + }, + mangoQuery: { + "selector": { + "schemaVersion": { + "$eq": "0.1" + }, + "type": "prefsSafe" + } + }, + invokers: { + "processDocument": { + funcName: "gpii.migrations.tests.simpleDocumentMigration.processDocument", + args: ["{arguments}.0"] + } + } +}); + +gpii.migrations.tests.simpleDocumentMigration.processDocument = function (doc) { + doc.schemaVersion = "0.2"; + delete doc.password; + return doc; +}; + +fluid.defaults("gpii.tests.migrations.caseHolder", { + gradeNames: ["fluid.test.testCaseHolder"], + events: { + createPreferencesService: null + }, + components: { + simpleDocumentMigration: { + type: "gpii.migrations.tests.simpleDocumentMigration", + createOnEvent: "createPreferencesService", + options: { + couchDbUri: "http://localhost:25984/gpii", + bulk: false + } + }, + simpleBulkDocumentMigration: { + type: "gpii.migrations.tests.simpleDocumentMigration", + createOnEvent: "createPreferencesService", + options: { + couchDbUri: "http://localhost:25984/gpii", + bulk: true + } + }, + preferencesService: { + type: "gpii.preferencesServer.preferencesService", + createOnEvent: "createPreferencesService", + options: { + gradeNames: ["gpii.tests.dbOperation.dbDataStore.base"], + components: { + dataStore: { + type: "gpii.dbOperation.dbDataStore", + options: { + dataSourceConfig: { + baseUrl: "http://localhost", + port: 25984, + dbName: "gpii" + } + } + } + } + } + } + } +}); + +fluid.defaults("gpii.tests.migrations.createPreferencesService", { + gradeNames: ["fluid.test.sequenceElement"], + sequence: [{ + func: "{caseHolder}.events.createPreferencesService.fire" + }] +}); + +fluid.defaults("gpii.tests.migrations.sequenceGrade", { + gradeNames: ["fluid.test.sequence"], + sequenceElements: { + startCouch: { + gradeNames: "gpii.test.startCouchSequence", + priority: "before:sequence" + }, + createPreferencesService: { + gradeNames: "gpii.tests.migrations.createPreferencesService", + priority: "after:startCouch" + }, + stopCouch: { + gradeNames: "gpii.test.stopCouchSequence", + priority: "after:sequence" + } + } +}); + +fluid.defaults("gpii.tests.migrations.environment", { + gradeNames: ["gpii.test.couchdb.environment.base"], + databases: { + gpii: { + data: [ + "%gpii-migrations/test/data/simpleMigrationDocumentsBeforeFixture.json", + "%gpii-universal/testData/dbData/views.json" + ] + } + }, + components: { + caseHolder: { + type: "gpii.tests.migrations.caseHolder" + } + } +}); + +gpii.migrations.utils.tests.compareFixtures = function (expected, couchPayload, value) { + jqUnit.assertEquals("Each object should be the same size", expected.length, value.length); + fluid.each(expected, function (item) { + var toCompare = fluid.find(value, function (el) { + return el._id === item._id ? el : undefined; + }); + delete toCompare._rev; + jqUnit.assertDeepEq("Checking item", item, toCompare); + }); +}; + +gpii.migrations.utils.createBeforeAfterTests = function (that) { + var migrationTypeName = that.typeName + ".beforeAfterTest"; + fluid.defaults(migrationTypeName, { + gradeNames: ["gpii.tests.migrations.environment"], + databases: { + gpii: { + data: [ + that.options.testFixtures.beforeMigration, + "%gpii-universal/testData/dbData/views.json" + ] + } + }, + components: { + caseHolder: { + type: "gpii.tests.migrations.caseHolder", + options: { + components: { + migrationToTest: { + type: that.typeName, + createOnEvent: "createPreferencesService", + options: { + couchDbUri: "http://localhost:25984/gpii", + bulk: false + } + } + }, + modules: [{ + name: "Migration tests for : " + that.typeName, + tests: [ + { + name: "Running before/after migration test.", + sequenceGrade: "gpii.tests.migrations.sequenceGrade", + sequence: [{ + task: "kettle.JSON.readFileSync", + args: ["{migrationToTest}.options.testFixtures.afterMigration", "After fixtures"], + resolve: "fluid.set", + resolveArgs: ["{that}", ["afterFixture"], "{arguments}.0"] + }, { + task: "fluid.promise.fireTransformEvent", + args: "{migrationToTest}.events.continueMigration", + reject: "jqUnit.assertDeepEq", + rejectArgs: ["There should be no more to migrate.", { + "message": "No more documents to migrate." + },"{arguments}.0"] + }, { + task: "gpii.migrations.mangoQuery", + args: [{ + mangoQuery: { + "selector": { + "type": { "$exists": true } + } + }, + couchDbUri: "http://localhost:25984/gpii" + }], + resolve: "gpii.migrations.utils.tests.compareFixtures", + resolveArgs: ["{that}.afterFixture", "{arguments}.0", "{arguments}.0.docs"] + }] + } + ] + }] + } + } + } + }); + return migrationTypeName; +}; diff --git a/gpii/node_modules/gpii-migrations/test/data/simpleMigrationDocumentsAfterFixture.json b/gpii/node_modules/gpii-migrations/test/data/simpleMigrationDocumentsAfterFixture.json new file mode 100644 index 000000000..151596a83 --- /dev/null +++ b/gpii/node_modules/gpii-migrations/test/data/simpleMigrationDocumentsAfterFixture.json @@ -0,0 +1,79 @@ +[ + { + "_id": "prefsSafe-1", + "type": "prefsSafe", + "schemaVersion": "0.2", + "prefsSafeType": "user", + "name": null, + "email": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "prefsSafe-2", + "type": "prefsSafe", + "schemaVersion": "0.2", + "prefsSafeType": "user", + "name": null, + "email": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "prefsSafe-3", + "type": "prefsSafe", + "schemaVersion": "0.2", + "prefsSafeType": "user", + "name": null, + "email": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "prefsSafe-4", + "type": "prefsSafe", + "schemaVersion": "0.2", + "prefsSafeType": "user", + "name": null, + "email": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "prefsSafe-5", + "type": "prefsSafe", + "schemaVersion": "0.2", + "prefsSafeType": "user", + "name": null, + "email": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "alice_gpii_key", + "type": "gpiiKey", + "schemaVersion": "0.1", + "prefsSafeId": "prefsSafe-1", + "prefsSetId": "gpii-default", + "revoked": false, + "revokedReason": null, + "timestampCreated": "2017-11-21T18:11:22.101Z", + "timestampUpdated": null + }, + { + "_id": "bob_gpii_key", + "type": "gpiiKey", + "schemaVersion": "0.1", + "prefsSafeId": "prefsSafe-2", + "prefsSetId": "gpii-default", + "revoked": false, + "revokedReason": null, + "timestampCreated": "2017-11-21T18:11:22.101Z", + "timestampUpdated": null + } +] diff --git a/gpii/node_modules/gpii-migrations/test/data/simpleMigrationDocumentsBeforeFixture.json b/gpii/node_modules/gpii-migrations/test/data/simpleMigrationDocumentsBeforeFixture.json new file mode 100644 index 000000000..f8f9fba7b --- /dev/null +++ b/gpii/node_modules/gpii-migrations/test/data/simpleMigrationDocumentsBeforeFixture.json @@ -0,0 +1,84 @@ +[ + { + "_id": "prefsSafe-1", + "type": "prefsSafe", + "schemaVersion": "0.1", + "prefsSafeType": "user", + "name": null, + "email": null, + "password": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "prefsSafe-2", + "type": "prefsSafe", + "schemaVersion": "0.1", + "prefsSafeType": "user", + "name": null, + "email": null, + "password": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "prefsSafe-3", + "type": "prefsSafe", + "schemaVersion": "0.1", + "prefsSafeType": "user", + "name": null, + "email": null, + "password": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "prefsSafe-4", + "type": "prefsSafe", + "schemaVersion": "0.1", + "prefsSafeType": "user", + "name": null, + "email": null, + "password": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "prefsSafe-5", + "type": "prefsSafe", + "schemaVersion": "0.1", + "prefsSafeType": "user", + "name": null, + "email": null, + "password": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "alice_gpii_key", + "type": "gpiiKey", + "schemaVersion": "0.1", + "prefsSafeId": "prefsSafe-1", + "prefsSetId": "gpii-default", + "revoked": false, + "revokedReason": null, + "timestampCreated": "2017-11-21T18:11:22.101Z", + "timestampUpdated": null + }, + { + "_id": "bob_gpii_key", + "type": "gpiiKey", + "schemaVersion": "0.1", + "prefsSafeId": "prefsSafe-2", + "prefsSetId": "gpii-default", + "revoked": false, + "revokedReason": null, + "timestampCreated": "2017-11-21T18:11:22.101Z", + "timestampUpdated": null + } +] diff --git a/gpii/node_modules/gpii-migrations/test/migrationsTests.js b/gpii/node_modules/gpii-migrations/test/migrationsTests.js new file mode 100644 index 000000000..d427ee92e --- /dev/null +++ b/gpii/node_modules/gpii-migrations/test/migrationsTests.js @@ -0,0 +1,194 @@ +/** + GPII Migrations Tests + + Copyright 2016-2017 OCAD University + + Licensed under the New BSD license. You may not use this file except in + compliance with this License. + + You may obtain a copy of the License at + https://github.com/GPII/universal/blob/master/LICENSE.txt + */ + +"use strict"; + +var fluid = require("infusion"), + gpii = fluid.registerNamespace("gpii"); + + +fluid.require("%gpii-universal"); +gpii.loadTestingSupport(); + +require("gpii-migrations"); + +/** + * Test sequence: + * Run mangoQuery for original number of items, verify correct. + * Run mangoQuery for total number of documents, verify correct. + * Run the migration. + * Re-run both mangoQueries and verify number of items. + * Check one of the migrated safes and verify it's structure + */ +fluid.defaults("gpii.tests.migrations.simpleDocumentMigrationTest", { + gradeNames: ["gpii.tests.migrations.environment"], + components: { + caseHolder: { + options: { + modules: [{ + name: "Simple Document Migration: Single Document at a Time", + tests: [ + { + name: "Number of items to migrate", + sequenceGrade: "gpii.tests.migrations.sequenceGrade", + sequence: [{ + task: "gpii.migrations.mangoQuery", + args: [{ + mangoQuery: { + "selector": { + "schemaVersion": { + "$eq": "0.1" + }, + "type": "prefsSafe" + } + }, + couchDbUri: "http://localhost:25984/gpii" + }], + resolve: "jqUnit.assertEquals", + resolveArgs: ["5 items to migrate", 5, "{arguments}.0.docs.length"] + }, { + task: "gpii.migrations.mangoQuery", + args: [{ + mangoQuery: { + "selector": { + "type": { "$exists": true } + } + }, + couchDbUri: "http://localhost:25984/gpii" + }], + resolve: "jqUnit.assertEquals", + resolveArgs: ["7 docs total", 7, "{arguments}.0.docs.length"] + }, + { + task: "fluid.promise.fireTransformEvent", + args: "{simpleDocumentMigration}.events.continueMigration", + reject: "jqUnit.assertDeepEq", + rejectArgs: ["There should be no more to migrate.", { + "message": "No more documents to migrate." + },"{arguments}.0"] + }, + { + task: "gpii.migrations.mangoQuery", + args: [{ + mangoQuery: { + "selector": { + "schemaVersion": { + "$eq": "0.1" + }, + "type": "prefsSafe" + } + }, + couchDbUri: "http://localhost:25984/gpii" + }], + resolve: "jqUnit.assertEquals", + resolveArgs: ["There should be zero left", 0, "{arguments}.0.docs.length"] + }, + { + task: "gpii.migrations.mangoQuery", + args: [{ + mangoQuery: { + "selector": { + "type": { "$exists": true } + } + }, + couchDbUri: "http://localhost:25984/gpii" + }], + resolve: "jqUnit.assertEquals", + resolveArgs: ["7 docs total", 7, "{arguments}.0.docs.length"] + }] + } + ] + }, { + name: "Simple Document Migration: Bulk Migration", + tests: [ + { + name: "Number of items to migrate", + sequenceGrade: "gpii.tests.migrations.sequenceGrade", + sequence: [{ + task: "gpii.migrations.mangoQuery", + args: [{ + mangoQuery: { + "selector": { + "schemaVersion": { + "$eq": "0.1" + }, + "type": "prefsSafe" + } + }, + couchDbUri: "http://localhost:25984/gpii" + }], + resolve: "jqUnit.assertEquals", + resolveArgs: ["5 items to migrate", 5, "{arguments}.0.docs.length"] + }, { + task: "gpii.migrations.mangoQuery", + args: [{ + mangoQuery: { + "selector": { + "type": { "$exists": true } + } + }, + couchDbUri: "http://localhost:25984/gpii" + }], + resolve: "jqUnit.assertEquals", + resolveArgs: ["7 docs total", 7, "{arguments}.0.docs.length"] + }, + { + task: "fluid.promise.fireTransformEvent", + args: "{simpleBulkDocumentMigration}.events.continueBulkMigration", + reject: "jqUnit.assertDeepEq", + rejectArgs: ["There should be no more to migrate.", { + "message": "No more documents to migrate." + },"{arguments}.0"] + }, + { + task: "gpii.migrations.mangoQuery", + args: [{ + mangoQuery: { + "selector": { + "schemaVersion": { + "$eq": "0.1" + }, + "type": "prefsSafe" + } + }, + couchDbUri: "http://localhost:25984/gpii" + }], + resolve: "jqUnit.assertEquals", + resolveArgs: ["There should be zero left", 0, "{arguments}.0.docs.length"] + }, + { + task: "gpii.migrations.mangoQuery", + args: [{ + mangoQuery: { + "selector": { + "type": { "$exists": true } + } + }, + couchDbUri: "http://localhost:25984/gpii" + }], + resolve: "jqUnit.assertEquals", + resolveArgs: ["7 docs total", 7, "{arguments}.0.docs.length"] + }] + } + ] + }] + } + } + } +}); + +var beforeAfterTestDefaults = gpii.migrations.utils.createBeforeAfterTests(gpii.migrations.tests.simpleDocumentMigration()); + +fluid.test.runTests([ + "gpii.tests.migrations.simpleDocumentMigrationTest", + beforeAfterTestDefaults +]); diff --git a/scripts/migrations/data/gpii-2966-after-fixture.json b/scripts/migrations/data/gpii-2966-after-fixture.json new file mode 100644 index 000000000..54337cf84 --- /dev/null +++ b/scripts/migrations/data/gpii-2966-after-fixture.json @@ -0,0 +1,80 @@ +[ + { + "_id": "prefsSafe-1", + "type": "prefsSafe", + "schemaVersion": "0.3", + "prefsSafeType": "user", + "name": null, + "email": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "prefsSafe-2", + "type": "prefsSafe", + "schemaVersion": "0.3", + "prefsSafeType": "user", + "name": null, + "email": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "prefsSafe-3", + "type": "prefsSafe", + "schemaVersion": "0.3", + "prefsSafeType": "user", + "name": null, + "email": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "prefsSafe-4", + "type": "prefsSafe", + "schemaVersion": "0.3", + "prefsSafeType": "user", + "name": null, + "email": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "prefsSafe-5", + "type": "prefsSafe", + "schemaVersion": "0.1", + "prefsSafeType": "user", + "name": null, + "email": null, + "password": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "alice_gpii_key", + "type": "gpiiKey", + "schemaVersion": "0.1", + "prefsSafeId": "prefsSafe-1", + "prefsSetId": "gpii-default", + "revoked": false, + "revokedReason": null, + "timestampCreated": "2017-11-21T18:11:22.101Z", + "timestampUpdated": null + }, + { + "_id": "bob_gpii_key", + "type": "gpiiKey", + "schemaVersion": "0.2", + "prefsSafeId": "prefsSafe-2", + "prefsSetId": "gpii-default", + "revoked": false, + "revokedReason": null, + "timestampCreated": "2017-11-21T18:11:22.101Z", + "timestampUpdated": null + } +] diff --git a/scripts/migrations/data/gpii-2966-after2-fixture.json b/scripts/migrations/data/gpii-2966-after2-fixture.json new file mode 100644 index 000000000..50549cd34 --- /dev/null +++ b/scripts/migrations/data/gpii-2966-after2-fixture.json @@ -0,0 +1,82 @@ +[ + { + "_id": "prefsSafe-1", + "type": "prefsSafe", + "schemaVersion": "0.1", + "prefsSafeType": "user", + "name": null, + "email": null, + "password": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "prefsSafe-2", + "type": "prefsSafe", + "schemaVersion": "0.1", + "prefsSafeType": "user", + "name": null, + "email": null, + "password": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "prefsSafe-3", + "type": "prefsSafe", + "schemaVersion": "0.2", + "prefsSafeType": "user", + "name": null, + "email": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "prefsSafe-4", + "type": "prefsSafe", + "schemaVersion": "0.2", + "prefsSafeType": "user", + "name": null, + "email": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "prefsSafe-5", + "type": "prefsSafe", + "schemaVersion": "0.2", + "prefsSafeType": "user", + "name": null, + "email": null, + "password": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "alice_gpii_key", + "type": "gpiiKey", + "schemaVersion": "0.3", + "prefsSafeId": "prefsSafe-1", + "prefsSetId": "gpii-default", + "revoked": false, + "revokedReason": null, + "timestampCreated": "2017-11-21T18:11:22.101Z", + "timestampUpdated": null + }, + { + "_id": "bob_gpii_key", + "type": "gpiiKey", + "schemaVersion": "0.3", + "prefsSafeId": "prefsSafe-2", + "prefsSetId": "gpii-default", + "revoked": false, + "revokedReason": null, + "timestampCreated": "2017-11-21T18:11:22.101Z", + "timestampUpdated": null + } +] diff --git a/scripts/migrations/data/gpii-2966-before-fixture.json b/scripts/migrations/data/gpii-2966-before-fixture.json new file mode 100644 index 000000000..8a504bdef --- /dev/null +++ b/scripts/migrations/data/gpii-2966-before-fixture.json @@ -0,0 +1,84 @@ +[ + { + "_id": "prefsSafe-1", + "type": "prefsSafe", + "schemaVersion": "0.2", + "prefsSafeType": "user", + "name": null, + "email": null, + "password": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "prefsSafe-2", + "type": "prefsSafe", + "schemaVersion": "0.2", + "prefsSafeType": "user", + "name": null, + "email": null, + "password": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "prefsSafe-3", + "type": "prefsSafe", + "schemaVersion": "0.2", + "prefsSafeType": "user", + "name": null, + "email": null, + "password": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "prefsSafe-4", + "type": "prefsSafe", + "schemaVersion": "0.2", + "prefsSafeType": "user", + "name": null, + "email": null, + "password": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "prefsSafe-5", + "type": "prefsSafe", + "schemaVersion": "0.1", + "prefsSafeType": "user", + "name": null, + "email": null, + "password": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "alice_gpii_key", + "type": "gpiiKey", + "schemaVersion": "0.1", + "prefsSafeId": "prefsSafe-1", + "prefsSetId": "gpii-default", + "revoked": false, + "revokedReason": null, + "timestampCreated": "2017-11-21T18:11:22.101Z", + "timestampUpdated": null + }, + { + "_id": "bob_gpii_key", + "type": "gpiiKey", + "schemaVersion": "0.2", + "prefsSafeId": "prefsSafe-2", + "prefsSetId": "gpii-default", + "revoked": false, + "revokedReason": null, + "timestampCreated": "2017-11-21T18:11:22.101Z", + "timestampUpdated": null + } +] diff --git a/scripts/migrations/data/gpii-2966-before2-fixture.json b/scripts/migrations/data/gpii-2966-before2-fixture.json new file mode 100644 index 000000000..eaaa054c2 --- /dev/null +++ b/scripts/migrations/data/gpii-2966-before2-fixture.json @@ -0,0 +1,82 @@ +[ + { + "_id": "prefsSafe-1", + "type": "prefsSafe", + "schemaVersion": "0.1", + "prefsSafeType": "user", + "name": null, + "email": null, + "password": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "prefsSafe-2", + "type": "prefsSafe", + "schemaVersion": "0.1", + "prefsSafeType": "user", + "name": null, + "email": null, + "password": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "prefsSafe-3", + "type": "prefsSafe", + "schemaVersion": "0.2", + "prefsSafeType": "user", + "name": null, + "email": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "prefsSafe-4", + "type": "prefsSafe", + "schemaVersion": "0.2", + "prefsSafeType": "user", + "name": null, + "email": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "prefsSafe-5", + "type": "prefsSafe", + "schemaVersion": "0.2", + "prefsSafeType": "user", + "name": null, + "email": null, + "password": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "alice_gpii_key", + "type": "gpiiKey", + "schemaVersion": "0.2", + "prefsSafeId": "prefsSafe-1", + "prefsSetId": "gpii-default", + "revoked": false, + "revokedReason": null, + "timestampCreated": "2017-11-21T18:11:22.101Z", + "timestampUpdated": null + }, + { + "_id": "bob_gpii_key", + "type": "gpiiKey", + "schemaVersion": "0.2", + "prefsSafeId": "prefsSafe-2", + "prefsSetId": "gpii-default", + "revoked": false, + "revokedReason": null, + "timestampCreated": "2017-11-21T18:11:22.101Z", + "timestampUpdated": null + } +] diff --git a/scripts/migrations/data/gpii-3717-after-fixture.json b/scripts/migrations/data/gpii-3717-after-fixture.json new file mode 100644 index 000000000..bbc1e0382 --- /dev/null +++ b/scripts/migrations/data/gpii-3717-after-fixture.json @@ -0,0 +1,116 @@ +[ + { + "_id": "prefsSafe-1", + "type": "prefsSafe", + "schemaVersion": "0.1", + "prefsSafeType": "user", + "name": null, + "email": null, + "password": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "prefsSafe-2", + "type": "prefsSafe", + "schemaVersion": "0.1", + "prefsSafeType": "user", + "name": null, + "email": null, + "password": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "prefsSafe-3", + "type": "prefsSafe", + "schemaVersion": "0.1", + "prefsSafeType": "user", + "name": null, + "email": null, + "password": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "prefsSafe-4", + "type": "prefsSafe", + "schemaVersion": "0.1", + "prefsSafeType": "user", + "name": null, + "email": null, + "password": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "prefsSafe-5", + "type": "prefsSafe", + "schemaVersion": "0.1", + "prefsSafeType": "user", + "name": null, + "email": null, + "password": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "alice_gpii_key", + "type": "gpiiKey", + "schemaVersion": "0.1", + "prefsSafeId": "prefsSafe-1", + "prefsSetId": "gpii-default", + "revoked": false, + "revokedReason": null, + "timestampCreated": "2017-11-21T18:11:22.101Z", + "timestampUpdated": null + }, + { + "_id": "bob_gpii_key", + "type": "gpiiKey", + "schemaVersion": "0.1", + "prefsSafeId": "prefsSafe-2", + "prefsSetId": "gpii-default", + "revoked": false, + "revokedReason": null, + "timestampCreated": "2017-11-21T18:11:22.101Z", + "timestampUpdated": null + }, + { + "_id": "clientCredential-1", + "type": "clientCredential", + "schemaVersion": "0.2", + "clientId": "gpiiAppInstallationClient-1", + "oauth2ClientId": "pilot-computer", + "oauth2ClientSecret": "pilot-computer-secret", + "revoked": false, + "revokedReason": null, + "timestampCreated": "2017-11-21T18:11:22.101Z", + "timestampRevoked": null, + "allowedIPBlocks": null, + "allowedPrefsToWrite": null, + "isCreateGpiiKeyAllowed": false, + "isCreatePrefsSafeAllowed": false + }, + { + "_id": "clientCredential-2", + "type": "clientCredential", + "schemaVersion": "0.2", + "clientId": "gpiiAppInstallationClient-1", + "oauth2ClientId": "pilot-computer-another", + "oauth2ClientSecret": "pilot-computer-secret-another", + "revoked": true, + "revokedReason": "Security breach", + "timestampCreated": "2018-11-21T18:11:25.101Z", + "timestampRevoked": "2018-12-21T18:11:25.101Z", + "allowedIPBlocks": null, + "allowedPrefsToWrite": null, + "isCreateGpiiKeyAllowed": false, + "isCreatePrefsSafeAllowed": false + } +] diff --git a/scripts/migrations/data/gpii-3717-after2-fixture.json b/scripts/migrations/data/gpii-3717-after2-fixture.json new file mode 100644 index 000000000..7e47b99c3 --- /dev/null +++ b/scripts/migrations/data/gpii-3717-after2-fixture.json @@ -0,0 +1,112 @@ +[ + { + "_id": "prefsSafe-1", + "type": "prefsSafe", + "schemaVersion": "0.2", + "prefsSafeType": "user", + "name": null, + "email": null, + "password": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "prefsSafe-2", + "type": "prefsSafe", + "schemaVersion": "0.2", + "prefsSafeType": "user", + "name": null, + "email": null, + "password": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "prefsSafe-3", + "type": "prefsSafe", + "schemaVersion": "0.2", + "prefsSafeType": "user", + "name": null, + "email": null, + "password": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "prefsSafe-4", + "type": "prefsSafe", + "schemaVersion": "0.2", + "prefsSafeType": "user", + "name": null, + "email": null, + "password": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "prefsSafe-5", + "type": "prefsSafe", + "schemaVersion": "0.2", + "prefsSafeType": "user", + "name": null, + "email": null, + "password": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "alice_gpii_key", + "type": "gpiiKey", + "schemaVersion": "0.2", + "prefsSafeId": "prefsSafe-1", + "prefsSetId": "gpii-default", + "revoked": false, + "revokedReason": null, + "timestampCreated": "2017-11-21T18:11:22.101Z", + "timestampUpdated": null + }, + { + "_id": "bob_gpii_key", + "type": "gpiiKey", + "schemaVersion": "0.2", + "prefsSafeId": "prefsSafe-2", + "prefsSetId": "gpii-default", + "revoked": false, + "revokedReason": null, + "timestampCreated": "2017-11-21T18:11:22.101Z", + "timestampUpdated": null + }, + { + "_id": "clientCredential-1", + "type": "clientCredential", + "schemaVersion": "0.1", + "clientId": "gpiiAppInstallationClient-1", + "oauth2ClientId": "pilot-computer", + "oauth2ClientSecret": "pilot-computer-secret", + "revoked": false, + "revokedReason": null, + "timestampCreated": "2017-11-21T18:11:22.101Z", + "timestampRevoked": null + }, + { + "_id": "clientCredential-2", + "type": "clientCredential", + "schemaVersion": "0.2", + "clientId": "gpiiAppInstallationClient-1", + "oauth2ClientId": "pilot-computer-another", + "oauth2ClientSecret": "pilot-computer-secret-another", + "revoked": true, + "revokedReason": "Security breach", + "timestampCreated": "2018-11-21T18:11:25.101Z", + "timestampRevoked": "2018-12-21T18:11:25.101Z", + "allowedIPBlocks": null, + "allowedPrefsToWrite": null, + "isCreateGpiiKeyAllowed": false, + "isCreatePrefsSafeAllowed": false + } +] diff --git a/scripts/migrations/data/gpii-3717-before-fixture.json b/scripts/migrations/data/gpii-3717-before-fixture.json new file mode 100644 index 000000000..87417489f --- /dev/null +++ b/scripts/migrations/data/gpii-3717-before-fixture.json @@ -0,0 +1,108 @@ +[ + { + "_id": "prefsSafe-1", + "type": "prefsSafe", + "schemaVersion": "0.1", + "prefsSafeType": "user", + "name": null, + "email": null, + "password": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "prefsSafe-2", + "type": "prefsSafe", + "schemaVersion": "0.1", + "prefsSafeType": "user", + "name": null, + "email": null, + "password": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "prefsSafe-3", + "type": "prefsSafe", + "schemaVersion": "0.1", + "prefsSafeType": "user", + "name": null, + "email": null, + "password": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "prefsSafe-4", + "type": "prefsSafe", + "schemaVersion": "0.1", + "prefsSafeType": "user", + "name": null, + "email": null, + "password": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "prefsSafe-5", + "type": "prefsSafe", + "schemaVersion": "0.1", + "prefsSafeType": "user", + "name": null, + "email": null, + "password": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "alice_gpii_key", + "type": "gpiiKey", + "schemaVersion": "0.1", + "prefsSafeId": "prefsSafe-1", + "prefsSetId": "gpii-default", + "revoked": false, + "revokedReason": null, + "timestampCreated": "2017-11-21T18:11:22.101Z", + "timestampUpdated": null + }, + { + "_id": "bob_gpii_key", + "type": "gpiiKey", + "schemaVersion": "0.1", + "prefsSafeId": "prefsSafe-2", + "prefsSetId": "gpii-default", + "revoked": false, + "revokedReason": null, + "timestampCreated": "2017-11-21T18:11:22.101Z", + "timestampUpdated": null + }, + { + "_id": "clientCredential-1", + "type": "clientCredential", + "schemaVersion": "0.1", + "clientId": "gpiiAppInstallationClient-1", + "oauth2ClientId": "pilot-computer", + "oauth2ClientSecret": "pilot-computer-secret", + "revoked": false, + "revokedReason": null, + "timestampCreated": "2017-11-21T18:11:22.101Z", + "timestampRevoked": null + }, + { + "_id": "clientCredential-2", + "type": "clientCredential", + "schemaVersion": "0.1", + "clientId": "gpiiAppInstallationClient-1", + "oauth2ClientId": "pilot-computer-another", + "oauth2ClientSecret": "pilot-computer-secret-another", + "revoked": true, + "revokedReason": "Security breach", + "timestampCreated": "2018-11-21T18:11:25.101Z", + "timestampRevoked": "2018-12-21T18:11:25.101Z" + } +] diff --git a/scripts/migrations/data/gpii-3717-before2-fixture.json b/scripts/migrations/data/gpii-3717-before2-fixture.json new file mode 100644 index 000000000..349957b88 --- /dev/null +++ b/scripts/migrations/data/gpii-3717-before2-fixture.json @@ -0,0 +1,112 @@ +[ + { + "_id": "prefsSafe-1", + "type": "prefsSafe", + "schemaVersion": "0.1", + "prefsSafeType": "user", + "name": null, + "email": null, + "password": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "prefsSafe-2", + "type": "prefsSafe", + "schemaVersion": "0.1", + "prefsSafeType": "user", + "name": null, + "email": null, + "password": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "prefsSafe-3", + "type": "prefsSafe", + "schemaVersion": "0.1", + "prefsSafeType": "user", + "name": null, + "email": null, + "password": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "prefsSafe-4", + "type": "prefsSafe", + "schemaVersion": "0.1", + "prefsSafeType": "user", + "name": null, + "email": null, + "password": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "prefsSafe-5", + "type": "prefsSafe", + "schemaVersion": "0.1", + "prefsSafeType": "user", + "name": null, + "email": null, + "password": null, + "preferences": {}, + "timestampCreated": "2017-12-14T19:55:11.640Z", + "timestampUpdated": null + }, + { + "_id": "alice_gpii_key", + "type": "gpiiKey", + "schemaVersion": "0.1", + "prefsSafeId": "prefsSafe-1", + "prefsSetId": "gpii-default", + "revoked": false, + "revokedReason": null, + "timestampCreated": "2017-11-21T18:11:22.101Z", + "timestampUpdated": null + }, + { + "_id": "bob_gpii_key", + "type": "gpiiKey", + "schemaVersion": "0.1", + "prefsSafeId": "prefsSafe-2", + "prefsSetId": "gpii-default", + "revoked": false, + "revokedReason": null, + "timestampCreated": "2017-11-21T18:11:22.101Z", + "timestampUpdated": null + }, + { + "_id": "clientCredential-1", + "type": "clientCredential", + "schemaVersion": "0.1", + "clientId": "gpiiAppInstallationClient-1", + "oauth2ClientId": "pilot-computer", + "oauth2ClientSecret": "pilot-computer-secret", + "revoked": false, + "revokedReason": null, + "timestampCreated": "2017-11-21T18:11:22.101Z", + "timestampRevoked": null + }, + { + "_id": "clientCredential-2", + "type": "clientCredential", + "schemaVersion": "0.2", + "clientId": "gpiiAppInstallationClient-1", + "oauth2ClientId": "pilot-computer-another", + "oauth2ClientSecret": "pilot-computer-secret-another", + "revoked": true, + "revokedReason": "Security breach", + "timestampCreated": "2018-11-21T18:11:25.101Z", + "timestampRevoked": "2018-12-21T18:11:25.101Z", + "allowedIPBlocks": null, + "allowedPrefsToWrite": null, + "isCreateGpiiKeyAllowed": false, + "isCreatePrefsSafeAllowed": false + } +] diff --git a/scripts/migrations/gpii-2966-migrations.js b/scripts/migrations/gpii-2966-migrations.js new file mode 100644 index 000000000..9612d0c32 --- /dev/null +++ b/scripts/migrations/gpii-2966-migrations.js @@ -0,0 +1,93 @@ +/*! +Copyright 2019 Raising The Floor - US + +Licensed under the New BSD license. You may not use this file except in +compliance with this License. + +You may obtain a copy of the License at +https://github.com/GPII/universal/blob/master/LICENSE.txt +*/ +"use strict"; + +var fluid = require("infusion"); + +var gpii = fluid.registerNamespace("gpii"); +fluid.registerNamespace("gpii.migrations.migrationGPII2966"); + +fluid.require("%gpii-universal"); +gpii.loadTestingSupport(); + +require("../../gpii/node_modules/gpii-migrations/index.js"); + +/** + * GPII-2966 Migration Part 1 + * + * First part of the GPII-2966 which updates `prefsSafe`s at `schemaVersion` 0.2 to + * 0.3 and removes their `password` field. + * + * @typedef {Infusion} gpii.migrations.migrationGPII2966 + */ +fluid.defaults("gpii.migrations.migrationGPII2966", { + gradeNames: ["gpii.migrations.couchDBmigration"], + testFixtures: { + beforeMigration: "%gpii-universal/scripts/migrations/data/gpii-2966-before-fixture.json", + afterMigration: "@expand:fluid.module.resolvePath(%gpii-universal/scripts/migrations/data/gpii-2966-after-fixture.json)" + }, + mangoQuery: { + "selector": { + "schemaVersion": { + "$eq": "0.2" + }, + "type": "prefsSafe" + } + }, + invokers: { + "processDocument": { + funcName: "gpii.migrations.migrationGPII2966.processDocument", + args: ["{arguments}.0"] + } + } +}); + +gpii.migrations.migrationGPII2966.processDocument = function (doc) { + doc.schemaVersion = "0.3"; + delete doc.password; + return doc; +}; + +/** + * GPII-2966 Migration Part 2 + * + * Second part of GPII-2966 migrations which updates all remaining documents to + * `schemaVersion` 0.3. + * + * @typedef {Infusion} gpii.migrations.migrationGPII2966part2 + */ +fluid.defaults("gpii.migrations.migrationGPII2966part2", { + gradeNames: ["gpii.migrations.couchDBmigration"], + testFixtures: { + beforeMigration: "%gpii-universal/scripts/migrations/data/gpii-2966-before2-fixture.json", + afterMigration: "@expand:fluid.module.resolvePath(%gpii-universal/scripts/migrations/data/gpii-2966-after2-fixture.json)" + }, + mangoQuery: { + "selector": { + "schemaVersion": { + "$eq": "0.2" + }, + "type": { + "$ne": "prefsSafe" + } + } + }, + invokers: { + "processDocument": { + funcName: "gpii.migrations.migrationGPII2966part2.processDocument", + args: ["{arguments}.0"] + } + } +}); + +gpii.migrations.migrationGPII2966part2.processDocument = function (doc) { + doc.schemaVersion = "0.3"; + return doc; +}; diff --git a/scripts/migrations/gpii-3717-migrations.js b/scripts/migrations/gpii-3717-migrations.js new file mode 100644 index 000000000..2863b96a4 --- /dev/null +++ b/scripts/migrations/gpii-3717-migrations.js @@ -0,0 +1,97 @@ +/*! +Copyright 2019 Raising The Floor - US + +Licensed under the New BSD license. You may not use this file except in +compliance with this License. + +You may obtain a copy of the License at +https://github.com/GPII/universal/blob/master/LICENSE.txt +*/ +"use strict"; + +var fluid = require("infusion"); + +var gpii = fluid.registerNamespace("gpii"); +fluid.registerNamespace("gpii.migrations.migrationGPII3717"); + +fluid.require("%gpii-universal"); +gpii.loadTestingSupport(); + +require("../../gpii/node_modules/gpii-migrations/index.js"); + +/** + * GPII-3717 Migration Part 1 + * + * First part of the GPII-3717 which updates `clientCredential`s at `schemaVersion` 0.1 to + * 0.2 and adds four new fields `allowedIPBlocks`, `allowedPrefsToWrite`, `isCreateGpiiKeyAllowed`, + * `isCreatePrefsSafeAllowed`. + * + * @typedef {Infusion} gpii.migrations.migrationGPII3717 + */ +fluid.defaults("gpii.migrations.migrationGPII3717", { + gradeNames: ["gpii.migrations.couchDBmigration"], + testFixtures: { + beforeMigration: "%gpii-universal/scripts/migrations/data/gpii-3717-before-fixture.json", + afterMigration: "@expand:fluid.module.resolvePath(%gpii-universal/scripts/migrations/data/gpii-3717-after-fixture.json)" + }, + mangoQuery: { + "selector": { + "schemaVersion": { + "$eq": "0.1" + }, + "type": "clientCredential" + } + }, + invokers: { + "processDocument": { + funcName: "gpii.migrations.migrationGPII3717.processDocument", + args: ["{arguments}.0"] + } + } +}); + +gpii.migrations.migrationGPII3717.processDocument = function (doc) { + doc.schemaVersion = "0.2"; + doc.allowedIPBlocks = null; + doc.allowedPrefsToWrite = null; + doc.isCreateGpiiKeyAllowed = false; + doc.isCreatePrefsSafeAllowed = false; + return doc; +}; + +/** + * GPII-3717 Migration Part 2 + * + * Second part of GPII-3717 migrations which updates all remaining documents to + * `schemaVersion` 0.2. + * + * @typedef {Infusion} gpii.migrations.migrationGPII3717part2 + */ +fluid.defaults("gpii.migrations.migrationGPII3717part2", { + gradeNames: ["gpii.migrations.couchDBmigration"], + testFixtures: { + beforeMigration: "%gpii-universal/scripts/migrations/data/gpii-3717-before2-fixture.json", + afterMigration: "@expand:fluid.module.resolvePath(%gpii-universal/scripts/migrations/data/gpii-3717-after2-fixture.json)" + }, + mangoQuery: { + "selector": { + "schemaVersion": { + "$eq": "0.1" + }, + "type": { + "$ne": "clientCredential" + } + } + }, + invokers: { + "processDocument": { + funcName: "gpii.migrations.migrationGPII3717part2.processDocument", + args: ["{arguments}.0"] + } + } +}); + +gpii.migrations.migrationGPII3717part2.processDocument = function (doc) { + doc.schemaVersion = "0.2"; + return doc; +}; diff --git a/scripts/runAllDocumentMigrations.js b/scripts/runAllDocumentMigrations.js new file mode 100644 index 000000000..a613f33a6 --- /dev/null +++ b/scripts/runAllDocumentMigrations.js @@ -0,0 +1,29 @@ +/*! +Copyright 2019 Raising The Floor - US + +Licensed under the New BSD license. You may not use this file except in +compliance with this License. + +You may obtain a copy of the License at +https://github.com/GPII/universal/blob/master/LICENSE.txt +*/ +"use strict"; + +var fluid = require("infusion"); + +var gpii = fluid.registerNamespace("gpii"); + +fluid.require("%gpii-universal"); +gpii.loadTestingSupport(); + +require("./migrations/gpii-2966-migrations.js"); +require("./migrations/gpii-3717-migrations.js"); + +fluid.each([ + gpii.migrations.migrationGPII2966(), + gpii.migrations.migrationGPII2966part2(), + gpii.migrations.migrationGPII3717(), + gpii.migrations.migrationGPII3717part2() +], function (item) { + fluid.promise.fireTransformEvent(item.events.continueMigration); +}); diff --git a/scripts/testAllDocumentMigrations.js b/scripts/testAllDocumentMigrations.js new file mode 100644 index 000000000..1cd4e4972 --- /dev/null +++ b/scripts/testAllDocumentMigrations.js @@ -0,0 +1,27 @@ +/*! +Copyright 2019 Raising The Floor - US + +Licensed under the New BSD license. You may not use this file except in +compliance with this License. + +You may obtain a copy of the License at +https://github.com/GPII/universal/blob/master/LICENSE.txt +*/ +"use strict"; + +var fluid = require("infusion"); + +var gpii = fluid.registerNamespace("gpii"); + +fluid.require("%gpii-universal"); +gpii.loadTestingSupport(); + +require("./migrations/gpii-2966-migrations.js"); +require("./migrations/gpii-3717-migrations.js"); + +fluid.test.runTests([ + gpii.migrations.utils.createBeforeAfterTests(gpii.migrations.migrationGPII2966()), + gpii.migrations.utils.createBeforeAfterTests(gpii.migrations.migrationGPII2966part2()), + gpii.migrations.utils.createBeforeAfterTests(gpii.migrations.migrationGPII3717()), + gpii.migrations.utils.createBeforeAfterTests(gpii.migrations.migrationGPII3717part2()) +]); diff --git a/tests/all-tests.js b/tests/all-tests.js index 837b27925..cdca02191 100644 --- a/tests/all-tests.js +++ b/tests/all-tests.js @@ -68,6 +68,7 @@ var testIncludes = [ "../gpii/node_modules/flowManager/test/PSPChannelTests.js", "../gpii/node_modules/flowManager/test/SettingsDataSourceTests.js", "../gpii/node_modules/gpii-db-operation/test/DbDataStoreTests.js", + "../gpii/node_modules/gpii-migrations/test/migrationsTests.js", "../gpii/node_modules/gpii-ini-file/test/iniFileTests.js", "../gpii/node_modules/gpii-oauth2/gpii-oauth2-authz-server/test/authGrantFinderTests.js", "../gpii/node_modules/gpii-oauth2/gpii-oauth2-authz-server/test/authorizationServiceTests.js",