New: Run adapt-migrations directly for all trigger points

errors/errors.json

@@ -112,6 +112,13 @@
     "description": "Update of the framework failed",
     "statusCode": 500
   },
+  "FW_UPDATE_MIGRATION_FAILED": {
+    "data": {
+      "errors": "Array of per-course migration errors"
+    },
+    "description": "Content migration after framework or plugin update encountered errors",
+    "statusCode": 500
+  },
   "FW_INVALID_VERSION": {
     "data": {
       "name": "Incompatible plugin name",

index.js

@@ -3,6 +3,6 @@
  * @namespace adaptframework
  */
 export { default } from './lib/AdaptFrameworkModule.js'
-export { copyFrameworkSource } from './lib/utils.js'
+export { copyFrameworkSource, readFrameworkPluginVersions } from './lib/utils.js'
 export { default as AdaptFrameworkBuild } from './lib/AdaptFrameworkBuild.js'
 export { default as AdaptFrameworkImport } from './lib/AdaptFrameworkImport.js'

lib/AdaptFrameworkImport.js

@@ -1,13 +1,12 @@
-import { App, Hook, spawn, readJson, writeJson } from 'adapt-authoring-core'
+import { App, Hook, readJson, writeJson } from 'adapt-authoring-core'
 import { parseObjectId } from 'adapt-authoring-mongodb'
 import fs from 'node:fs/promises'
 import { glob } from 'glob'
 import octopus from 'adapt-octopus'
 import path from 'upath'
-import { randomBytes } from 'node:crypto'
 import semver from 'semver'
 import { unzip } from 'zipper'
-import { log, logDir, getImportSummary, getImportContentCounts } from './utils.js'
+import { log, logDir, getImportSummary, getImportContentCounts, readFrameworkPluginVersions, collectMigrationScripts, runContentMigration } from './utils.js'
 
 import ComponentTransform from './migrations/component.js'
 import ConfigTransform from './migrations/config.js'
@@ -269,9 +268,8 @@ class AdaptFrameworkImport {
         [this.importCourseAssets, importContent],
         [this.importCoursePlugins, isDryRun && importPlugins],
         [this.importCoursePlugins, !isDryRun && importContent],
-        [this.loadCourseData, isDryRun && importContent],
+        [this.loadCourseData, importContent],
         [this.migrateCourseData, !isDryRun && migrateContent],
-        [this.loadCourseData, !isDryRun && importContent],
         [this.importCourseData, !isDryRun && importContent],
         [this.generateSummary]
       ]
@@ -492,47 +490,61 @@ class AdaptFrameworkImport {
   }
 
   /**
-   * Run grunt task
-   * @return {Promise}
-   */
-  async runGruntMigration (subTask, { outputDir, captureDir, outputFilePath }) {
-    const output = await spawn({
-      cmd: 'npx',
-      args: ['grunt', `migration:${subTask}`, `--outputdir=${outputDir}`, `--capturedir=${captureDir}`],
-      cwd: this.frameworkPath ?? this.framework.path
-    })
-    if (outputFilePath) await fs.writeFile(outputFilePath, output)
-  }
-
-  /**
-   * Handle migrate course data, installs adapt-migrations/capture data/adds updated scripts/migrates data
+   * Migrates course data in-memory using adapt-migrations
    */
   async migrateCourseData () {
     try {
       await this.patchThemeName()
       await this.patchCustomStyle()
 
-      const migrationId = `${this.userId}-${randomBytes(4).toString('hex')}`
-
-      const opts = {
-        outputDir: path.relative(this.framework.path, path.resolve(this.coursePath, '..')),
-        captureDir: path.join(`./${migrationId}-migrations`),
-        outputFilePath: path.join(this.framework.path, 'migrations', `${migrationId}.txt`)
-      }
-      log('debug', 'MIGRATION_ID', migrationId)
-      logDir('captureDir', opts.captureDir)
-      logDir('outputDir', opts.outputDir)
+      const content = this.flattenContentJson()
+      const fromPlugins = Object.values(this.usedContentPlugins).map(p => ({
+        name: p.name,
+        version: p.version
+      }))
+      const toPlugins = await readFrameworkPluginVersions(this.framework.path)
+      const scripts = await collectMigrationScripts(this.framework.path)
 
-      await this.runGruntMigration('capture', opts)
-      await this.runGruntMigration('migrate', opts)
+      const migrated = await runContentMigration({ content, fromPlugins, toPlugins, scripts })
 
-      await fs.rm(path.join(this.framework.path, opts.captureDir), { recursive: true })
+      this.unflattenContentJson(migrated)
+      log('info', 'in-memory content migration completed')
     } catch (error) {
       log('error', 'Migration process failed', error)
       throw App.instance.errors.FW_IMPORT_MIGRATION_FAILED.setData({ reason: error.message })
     }
   }
 
+  /**
+   * Flattens this.contentJson into a flat array for adapt-migrations
+   * @returns {Array}
+   */
+  flattenContentJson () {
+    const content = []
+    if (this.contentJson.course?._id) content.push(this.contentJson.course)
+    if (this.contentJson.config?._id) content.push(this.contentJson.config)
+    for (const item of Object.values(this.contentJson.contentObjects)) {
+      if (item?._id) content.push(item)
+    }
+    return content
+  }
+
+  /**
+   * Writes migrated content back into the contentJson structure
+   * @param {Array} migrated The migrated content array
+   */
+  unflattenContentJson (migrated) {
+    for (const item of migrated) {
+      if (item._type === 'course') {
+        this.contentJson.course = item
+      } else if (item._type === 'config') {
+        this.contentJson.config = item
+      } else {
+        this.contentJson.contentObjects[item._id] = item
+      }
+    }
+  }
+
   /**
    * Imports any specified tags
    * @return {Promise}

lib/AdaptFrameworkModule.js

@@ -4,7 +4,7 @@ import AdaptFrameworkImport from './AdaptFrameworkImport.js'
 import fs from 'node:fs/promises'
 import { getHandler, postHandler, importHandler, postUpdateHandler, getUpdateHandler } from './handlers.js'
 import { loadRouteConfig, registerRoutes } from 'adapt-authoring-server'
-import { runCliCommand } from './utils.js'
+import { runCliCommand, readFrameworkPluginVersions, migrateExistingCourses } from './utils.js'
 import path from 'node:path'
 import semver from 'semver'
 
@@ -206,17 +206,22 @@ class AdaptFrameworkModule extends AbstractModule {
    * @return {Promise}
    */
   async updateFramework (version) {
+    let migrationResult
     try {
       if (version) {
         this.checkVersionCompatibility(version)
       }
+      const fromPlugins = await readFrameworkPluginVersions(this.path)
       await this.runCliCommand('updateFramework', { version: version ?? this.targetVersionRange })
       this._version = await this.runCliCommand('getCurrentFrameworkVersion')
+      const toPlugins = await readFrameworkPluginVersions(this.path)
+      migrationResult = await migrateExistingCourses({ fromPlugins, toPlugins, frameworkDir: this.path })
     } catch (e) {
       this.log('error', `failed to update framework, ${e.message}`)
       throw e.statusCode ? e : this.app.errors.FW_UPDATE_FAILED.setData({ reason: e.message })
     }
-    this.postUpdateHook.invoke()
+    await this.postUpdateHook.invoke()
+    return migrationResult
   }
 
   /**
@@ -301,6 +306,18 @@ class AdaptFrameworkModule extends AbstractModule {
     this.contentMigrations.push(migration)
   }
 
+  /**
+   * Migrates content for specific courses. Called by contentplugin on plugin update.
+   * @param {Object} options
+   * @param {Array<{name: String, version: String}>} options.fromPlugins Plugin versions before the update
+   * @param {Array<{name: String, version: String}>} options.toPlugins Plugin versions after the update
+   * @param {String[]} options.courseIds Course IDs to migrate
+   * @returns {Promise<{migrated: Number, failed: Number, errors: Array}>}
+   */
+  async migrateCourses ({ fromPlugins, toPlugins, courseIds }) {
+    return migrateExistingCourses({ fromPlugins, toPlugins, frameworkDir: this.path, courseIds })
+  }
+
   /**
    * Builds a single Adapt framework course
    * @param {AdaptFrameworkBuildOptions} options

lib/handlers.js

@@ -133,11 +133,12 @@ export async function postUpdateHandler (req, res, next) {
     }
     log('info', 'running framework update')
     const previousVersion = framework.version
-    await framework.updateFramework(req.body.version)
+    const migrationResult = await framework.updateFramework(req.body.version)
     const currentVersion = framework.version !== previousVersion ? framework.version : undefined
     res.json({
       from: previousVersion,
-      to: currentVersion
+      to: currentVersion,
+      migration: migrationResult
     })
   } catch (e) {
     return next(e)

lib/utils.js

@@ -7,3 +7,7 @@ export { retrieveBuildData } from './utils/retrieveBuildData.js'
 export { getImportSummary } from './utils/getImportSummary.js'
 export { slugifyTitle } from './utils/slugifyTitle.js'
 export { copyFrameworkSource } from './utils/copyFrameworkSource.js'
+export { readFrameworkPluginVersions } from './utils/readFrameworkPluginVersions.js'
+export { collectMigrationScripts } from './utils/collectMigrationScripts.js'
+export { runContentMigration } from './utils/runContentMigration.js'
+export { migrateExistingCourses } from './utils/migrateExistingCourses.js'

lib/utils/collectMigrationScripts.js

@@ -0,0 +1,15 @@
+import { glob } from 'glob'
+import path from 'node:path'
+
+/**
+ * Collects all migration script paths from the framework's src directory
+ * @param {String} frameworkDir Absolute path to the framework directory
+ * @returns {Promise<String[]>} Absolute paths to migration scripts
+ */
+export async function collectMigrationScripts (frameworkDir) {
+  const srcDir = path.join(frameworkDir, 'src')
+  return glob([
+    'core/migrations/**/*.js',
+    '*/*/migrations/**/*.js'
+  ], { cwd: srcDir, absolute: true })
+}

lib/utils/migrateExistingCourses.js

@@ -0,0 +1,78 @@
+import { App } from 'adapt-authoring-core'
+import { isDeepStrictEqual } from 'node:util'
+import { collectMigrationScripts } from './collectMigrationScripts.js'
+import { runContentMigration } from './runContentMigration.js'
+import { log } from './log.js'
+
+/**
+ * Migrates content for a set of courses by courseId
+ * @param {Object} options
+ * @param {Array<{name: String, version: String}>} options.fromPlugins Plugin versions before update
+ * @param {Array<{name: String, version: String}>} options.toPlugins Plugin versions after update
+ * @param {String} options.frameworkDir Absolute path to the framework directory
+ * @param {String[]} [options.courseIds] Specific course IDs to migrate (if omitted, migrates all)
+ * @returns {Promise<{migrated: Number, failed: Number, errors: Array}>}
+ */
+export async function migrateExistingCourses ({ fromPlugins, toPlugins, frameworkDir, courseIds }) {
+  const content = await App.instance.waitForModule('content')
+  const scripts = await collectMigrationScripts(frameworkDir)
+
+  if (!scripts.length) {
+    log('debug', 'no migration scripts found, skipping')
+    return { migrated: 0, failed: 0, errors: [] }
+  }
+
+  const courses = courseIds
+    ? await Promise.all(courseIds.map(async _id => content.findOne({ _id })))
+    : await content.find({ _type: 'course' })
+
+  let migrated = 0
+  let failed = 0
+  const errors = []
+
+  for (const course of courses) {
+    try {
+      const courseId = course._id.toString()
+      log('debug', `migrating course ${courseId}`)
+
+      const courseContent = await fetchCourseContent(content, course)
+      const originals = JSON.parse(JSON.stringify(courseContent))
+
+      const migratedContent = await runContentMigration({
+        content: courseContent,
+        fromPlugins: JSON.parse(JSON.stringify(fromPlugins)),
+        toPlugins,
+        scripts
+      })
+
+      let updatedCount = 0
+      for (let i = 0; i < migratedContent.length; i++) {
+        if (!isDeepStrictEqual(originals[i], migratedContent[i])) {
+          await content.update({ _id: migratedContent[i]._id }, migratedContent[i])
+          updatedCount++
+        }
+      }
+      if (updatedCount > 0) {
+        log('info', `migrated ${updatedCount} items in course ${courseId}`)
+      }
+      migrated++
+    } catch (e) {
+      const courseId = course._id?.toString() ?? 'unknown'
+      log('error', `migration failed for course ${courseId}`, e.message)
+      errors.push({ courseId, error: e.message })
+      failed++
+    }
+  }
+
+  log('info', `migration complete: ${migrated} succeeded, ${failed} failed`)
+  return { migrated, failed, errors }
+}
+
+async function fetchCourseContent (content, course) {
+  const config = await content.findOne({ _courseId: course._id, _type: 'config' }, { strict: false })
+  const items = await content.find({ _courseId: course._id, _type: { $nin: ['course', 'config'] } })
+  const result = [course]
+  if (config) result.push(config)
+  result.push(...items)
+  return result
+}

lib/utils/readFrameworkPluginVersions.js

@@ -0,0 +1,21 @@
+import { readJson } from 'adapt-authoring-core'
+import { glob } from 'glob'
+import path from 'node:path'
+
+/**
+ * Reads bower.json files from the framework's src directory to build a list of plugin names and versions
+ * @param {String} frameworkDir Absolute path to the framework directory
+ * @returns {Promise<Array<{name: String, version: String}>>}
+ */
+export async function readFrameworkPluginVersions (frameworkDir) {
+  const srcDir = path.join(frameworkDir, 'src')
+  const bowerPaths = await glob([
+    'core/bower.json',
+    '{components,extensions,menu,theme}/*/bower.json'
+  ], { cwd: srcDir, absolute: true })
+  const plugins = await Promise.all(bowerPaths.map(async p => {
+    const { name, version } = await readJson(p)
+    return { name, version }
+  }))
+  return plugins
+}

lib/utils/runContentMigration.js

@@ -0,0 +1,32 @@
+import { load, migrate, Journal, Logger } from 'adapt-migrations'
+
+/**
+ * Runs adapt-migrations on a content array. Shared by framework update, course import, and plugin update.
+ * @param {Object} options
+ * @param {Array} options.content Flat array of content objects (course, config, contentObjects, etc.)
+ * @param {Array<{name: String, version: String}>} options.fromPlugins Plugin versions before the update
+ * @param {Array<{name: String, version: String}>} options.toPlugins Plugin versions after the update
+ * @param {String[]} options.scripts Absolute paths to migration scripts
+ * @param {String} [options.cachePath] Optional cache path for adapt-migrations
+ * @returns {Promise<Array>} The migrated content array
+ */
+export async function runContentMigration ({ content, fromPlugins, toPlugins, scripts, cachePath }) {
+  const logger = Logger.getInstance()
+
+  await load({ scripts, cachePath, logger })
+
+  const originalFromPlugins = JSON.parse(JSON.stringify(fromPlugins))
+  const journal = new Journal({
+    logger,
+    data: {
+      content,
+      fromPlugins,
+      originalFromPlugins,
+      toPlugins
+    }
+  })
+
+  await migrate({ journal, logger })
+
+  return journal.data.content
+}

package.json

@@ -19,6 +19,7 @@
     "adapt-authoring-mongodb": "^3.0.0",
     "adapt-authoring-spoortracking": "^1.0.2",
     "adapt-cli": "^3.3.3",
+    "adapt-migrations": "^1.4.0",
     "adapt-octopus": "^0.1.2",
     "bytes": "^3.1.2",
     "fs-extra": "11.3.3",