New: Support installing and updating plugins from git URLs (fixes #237)

Allow users to install plugins directly from HTTPS git URLs instead of
requiring them to be registered in the bower registry. Supports optional
branch/tag refs via URL#ref syntax.

Author: taylortom

lib/integration/Plugin.js

@@ -5,6 +5,8 @@ import endpointParser from 'bower-endpoint-parser'
 import semver from 'semver'
 import fs from 'fs-extra'
 import path from 'path'
+import os from 'os'
+import { exec } from 'child_process'
 import getBowerRegistryConfig from './getBowerRegistryConfig.js'
 import { ADAPT_ALLOW_PRERELEASE, PLUGIN_TYPES, PLUGIN_TYPE_FOLDERS, PLUGIN_DEFAULT_TYPE } from '../util/constants.js'
 /** @typedef {import("./Project.js").default} Project */
@@ -38,13 +40,23 @@ export default class Plugin {
     this.project = project
     this.cwd = cwd
     this.BOWER_REGISTRY_CONFIG = getBowerRegistryConfig({ cwd: this.cwd })
-    const endpoint = name + '#' + (isCompatibleEnabled ? '*' : requestedVersion)
-    const ep = endpointParser.decompose(endpoint)
     this.sourcePath = null
-    this.name = ep.name || ep.source
-    this.packageName = (/^adapt-/i.test(this.name) ? '' : 'adapt-') + (!isContrib ? '' : 'contrib-') + slug(this.name, { maintainCase: true })
-    // the constraint given by the user
-    this.requestedVersion = requestedVersion
+
+    const isGitUrl = /^https?:\/\//.test(name)
+    if (isGitUrl) {
+      this.gitUrl = name
+      this.gitRef = (requestedVersion && requestedVersion !== '*') ? requestedVersion : null
+      this.name = ''
+      this.packageName = ''
+      this.requestedVersion = '*'
+    } else {
+      const endpoint = name + '#' + (isCompatibleEnabled ? '*' : requestedVersion)
+      const ep = endpointParser.decompose(endpoint)
+      this.name = ep.name || ep.source
+      this.packageName = (/^adapt-/i.test(this.name) ? '' : 'adapt-') + (!isContrib ? '' : 'contrib-') + slug(this.name, { maintainCase: true })
+      this.requestedVersion = requestedVersion
+    }
+
     // the most recent version of the plugin compatible with the given framework
     this.latestCompatibleSourceVersion = null
     // a non-wildcard constraint resolved to the highest version of the plugin that satisfies the requestedVersion and is compatible with the framework
@@ -128,6 +140,14 @@ export default class Plugin {
     return Boolean(this.sourcePath || this?._projectInfo?._wasInstalledFromPath)
   }
 
+  /**
+   * plugin will be or was installed from a git URL
+   * @returns {boolean}
+   */
+  get isGitSource () {
+    return Boolean(this.gitUrl || this._projectInfo?._wasInstalledFromGitRepo)
+  }
+
   /**
    * check if source path is a zip
    * @returns {boolean}
@@ -185,10 +205,34 @@ export default class Plugin {
   }
 
   async fetchSourceInfo () {
+    if (this.isGitSource) return await this.fetchGitSourceInfo()
     if (this.isLocalSource) return await this.fetchLocalSourceInfo()
     await this.fetchBowerInfo()
   }
 
+  async fetchGitSourceInfo () {
+    if (this._sourceInfo) return this._sourceInfo
+    this._sourceInfo = null
+    const tmpDir = path.join(os.tmpdir(), `adapt-git-${Date.now()}`)
+    try {
+      const branchFlag = this.gitRef ? ` --branch ${this.gitRef}` : ''
+      await new Promise((resolve, reject) => {
+        exec(`git clone --depth 1${branchFlag} ${this.gitUrl} "${tmpDir}"`, (err) => {
+          if (err) return reject(err)
+          resolve()
+        })
+      })
+      const bowerJSONPath = path.join(tmpDir, 'bower.json')
+      if (!fs.existsSync(bowerJSONPath)) return
+      this._sourceInfo = await fs.readJSON(bowerJSONPath)
+      this.name = this._sourceInfo.name
+      this.packageName = this.name
+      this.matchedVersion = this._sourceInfo.version
+    } finally {
+      await fs.rm(tmpDir, { recursive: true, force: true })
+    }
+  }
+
   async fetchLocalSourceInfo () {
     if (this._sourceInfo) return this._sourceInfo
     this._sourceInfo = null
@@ -269,6 +313,10 @@ export default class Plugin {
     if (!this._projectInfo) return
     this.name = this._projectInfo.name
     this.packageName = this.name
+    if (this._projectInfo._wasInstalledFromGitRepo) {
+      this.gitUrl = this._projectInfo._gitUrl
+      this.gitRef = this._projectInfo._gitRef || null
+    }
   }
 
   async findCompatibleVersion (framework) {
@@ -291,7 +339,7 @@ export default class Plugin {
     const getMatchingVersion = async () => {
       if (!this.isPresent) return null
 
-      if (this.isLocalSource) {
+      if (this.isLocalSource || this.isGitSource) {
         const info = this.projectVersion ? this._projectInfo : this._sourceInfo
         const satisfiesConstraint = !this.hasValidRequestVersion || semver.satisfies(info.version, this.requestedVersion, semverOptions)
         const satisfiesFramework = semver.satisfies(framework, info.framework)
@@ -360,7 +408,7 @@ export default class Plugin {
 
   async getRepositoryUrl () {
     if (this._repositoryUrl) return this._repositoryUrl
-    if (this.isLocalSource) return
+    if (this.isLocalSource || this.isGitSource) return
     const url = await new Promise((resolve, reject) => {
       bower.commands.lookup(this.packageName, { cwd: this.cwd, registry: this.BOWER_REGISTRY_CONFIG })
         .on('end', resolve)

lib/integration/PluginManagement/install.js

@@ -63,19 +63,28 @@ async function getInstallTargets ({ logger, project, plugins, isCompatibleEnable
   const itinerary = isEmpty
     ? await project.getManifestDependencies()
     : plugins.reduce((itinerary, arg) => {
-      const [name, version = '*'] = arg.split(/[#@]/)
+      let name, version
+      if (/^https?:\/\//.test(arg)) {
+        const hashIndex = arg.lastIndexOf('#')
+        if (hashIndex !== -1) {
+          name = arg.substring(0, hashIndex)
+          version = arg.substring(hashIndex + 1)
+        } else {
+          name = arg
+          version = '*'
+        }
+      } else {
+        [name, version = '*'] = arg.split(/[#@]/)
+      }
       // Duplicates are removed by assigning to object properties
       itinerary[name] = version
       return itinerary
     }, {})
-  const pluginNames = Object.entries(itinerary).map(([name, version]) => `${name}#${version}`)
-
   /**
    * @type {[Target]}
    */
-  const targets = pluginNames.length
-    ? pluginNames.map(nameVersion => {
-      const [name, requestedVersion] = nameVersion.split(/[#@]/)
+  const targets = Object.keys(itinerary).length
+    ? Object.entries(itinerary).map(([name, requestedVersion]) => {
       return new Target({ name, requestedVersion, isCompatibleEnabled, project, logger })
     })
     : await project.getInstallTargets()

lib/integration/PluginManagement/print.js

@@ -22,9 +22,10 @@ export function versionPrinter (plugin, logger) {
     versionToApply,
     latestCompatibleSourceVersion
   } = plugin
+  const sourceLabel = plugin.isLocalSource ? ' (local)' : plugin.isGitSource ? ' (git)' : ` (latest compatible version is ${greenIfEqual(versionToApply, latestCompatibleSourceVersion)})`
   logger?.log(highlight(plugin.packageName), latestCompatibleSourceVersion === null
     ? '(no version information)'
-    : `${chalk.greenBright(versionToApply)}${plugin.isLocalSource ? ' (local)' : ` (latest compatible version is ${greenIfEqual(versionToApply, latestCompatibleSourceVersion)})`}`
+    : `${chalk.greenBright(versionToApply)}${sourceLabel}`
   )
 }
 
@@ -37,9 +38,10 @@ export function existingVersionPrinter (plugin, logger) {
   const fromTo = preUpdateProjectVersion !== null
     ? `from ${chalk.greenBright(preUpdateProjectVersion)} to ${chalk.greenBright(projectVersion)}`
     : `${chalk.greenBright(projectVersion)}`
+  const sourceLabel = plugin.isLocalSource ? ' (local)' : plugin.isGitSource ? ' (git)' : ` (latest compatible version is ${greenIfEqual(projectVersion, latestCompatibleSourceVersion)})`
   logger?.log(highlight(plugin.packageName), latestCompatibleSourceVersion === null
     ? fromTo
-    : `${fromTo}${plugin.isLocalSource ? ' (local)' : ` (latest compatible version is ${greenIfEqual(projectVersion, latestCompatibleSourceVersion)})`}`
+    : `${fromTo}${sourceLabel}`
   )
 }
 

lib/integration/PluginManagement/update.js

@@ -163,7 +163,7 @@ async function conflictResolution ({ logger, targets, isInteractive }) {
       prompt
     }
   }
-  const preFilteredPlugins = targets.filter(target => !target.isLocalSource)
+  const preFilteredPlugins = targets.filter(target => !target.isLocalSource && !target.isGitSource)
   const allQuestions = [
     add(preFilteredPlugins.filter(target => !target.hasFrameworkCompatibleVersion && target.latestSourceVersion), 'There is no compatible version of the following plugins:', checkVersion),
     add(preFilteredPlugins.filter(target => target.hasFrameworkCompatibleVersion && !target.hasValidRequestVersion), 'The version requested is invalid, there are newer compatible versions of the following plugins:', checkVersion),
@@ -181,28 +181,31 @@ async function conflictResolution ({ logger, targets, isInteractive }) {
  * @param {[Target]} options.targets
  */
 function summariseDryRun ({ logger, targets }) {
-  const preFilteredPlugins = targets.filter(target => !target.isLocalSource)
+  const preFilteredPlugins = targets.filter(target => !target.isLocalSource && !target.isGitSource)
   const localSources = targets.filter(target => target.isLocalSource)
+  const gitSources = targets.filter(target => target.isGitSource && target.isToBeUpdated)
   const toBeInstalled = preFilteredPlugins.filter(target => target.isToBeUpdated)
   const toBeSkipped = preFilteredPlugins.filter(target => !target.isToBeUpdated || target.isSkipped)
   const missing = preFilteredPlugins.filter(target => target.isMissing)
   summarise(logger, localSources, packageNamePrinter, 'The following plugins were installed from a local source and cannot be updated:')
   summarise(logger, toBeSkipped, packageNamePrinter, 'The following plugins will be skipped:')
   summarise(logger, missing, packageNamePrinter, 'There was a problem locating the following plugins:')
-  summarise(logger, toBeInstalled, existingVersionPrinter, 'The following plugins will be updated:')
+  summarise(logger, [...toBeInstalled, ...gitSources], existingVersionPrinter, 'The following plugins will be updated:')
 }
 
 /**
  * @param {Object} options
  * @param {[Target]} options.targets
  */
 function summariseUpdates ({ logger, targets }) {
-  const preFilteredPlugins = targets.filter(target => !target.isLocalSource)
+  const preFilteredPlugins = targets.filter(target => !target.isLocalSource && !target.isGitSource)
   const localSources = targets.filter(target => target.isLocalSource)
-  const installSucceeded = preFilteredPlugins.filter(target => target.isUpdateSuccessful)
+  const gitSucceeded = targets.filter(target => target.isGitSource && target.isUpdateSuccessful)
+  const gitErrored = targets.filter(target => target.isGitSource && target.isUpdateFailure)
+  const installSucceeded = [...preFilteredPlugins.filter(target => target.isUpdateSuccessful), ...gitSucceeded]
   const installSkipped = preFilteredPlugins.filter(target => target.isSkipped)
   const noUpdateAvailable = preFilteredPlugins.filter(target => !target.isToBeUpdated && !target.isSkipped)
-  const installErrored = preFilteredPlugins.filter(target => target.isUpdateFailure)
+  const installErrored = [...preFilteredPlugins.filter(target => target.isUpdateFailure), ...gitErrored]
   const missing = preFilteredPlugins.filter(target => target.isMissing)
   const noneInstalled = (installSucceeded.length === 0)
   const allInstalledSuccessfully = (installErrored.length === 0 && missing.length === 0)

lib/integration/Project.js

@@ -56,7 +56,12 @@ export default class Project {
 
   /** @returns {[Target]} */
   async getInstallTargets () {
-    return Object.entries(await this.getManifestDependencies()).map(([name, requestedVersion]) => new Target({ name, requestedVersion, project: this, logger: this.logger }))
+    return Object.entries(await this.getManifestDependencies()).map(([name, requestedVersion]) => {
+      if (/^https?:\/\//.test(requestedVersion)) {
+        return new Target({ name: requestedVersion, project: this, logger: this.logger })
+      }
+      return new Target({ name, requestedVersion, project: this, logger: this.logger })
+    })
   }
 
   /** @returns {[string]} */
@@ -128,7 +133,9 @@ export default class Project {
     if (this.containsManifestFile) {
       manifest = readValidateJSONSync(this.manifestFilePath)
     }
-    manifest.dependencies[plugin.packageName] = plugin.sourcePath || plugin.requestedVersion || plugin.version
+    manifest.dependencies[plugin.packageName] = plugin.gitUrl
+      ? (plugin.gitUrl + (plugin.gitRef ? '#' + plugin.gitRef : ''))
+      : plugin.sourcePath || plugin.requestedVersion || plugin.version
     fs.writeJSONSync(this.manifestFilePath, manifest, { spaces: 2, replacer: null })
   }
 

lib/integration/Target.js

@@ -121,6 +121,10 @@ export default class Target extends Plugin {
   }
 
   markInstallable () {
+    if (this.isGitSource && this.matchedVersion) {
+      this.versionToApply = this.matchedVersion
+      return
+    }
     if (!this.isApplyLatestCompatibleVersion && !(this.isLocalSource && this.latestSourceVersion)) return
     this.versionToApply = this.matchedVersion
   }
@@ -172,6 +176,30 @@ export default class Target extends Plugin {
       await this.fetchProjectInfo()
       return
     }
+    if (this.isGitSource) {
+      await fs.ensureDir(path.resolve(this.cwd, 'src', pluginTypeFolder))
+      const pluginPath = path.resolve(this.cwd, 'src', pluginTypeFolder, this.packageName)
+      await fs.rm(pluginPath, { recursive: true, force: true })
+      const branchFlag = this.gitRef ? ` --branch ${this.gitRef}` : ''
+      try {
+        await new Promise((resolve, reject) => {
+          exec(`git clone${branchFlag} ${this.gitUrl} "${pluginPath}"`, (err) => {
+            if (err) return reject(err)
+            resolve()
+          })
+        })
+      } catch (error) {
+        throw new Error(`The plugin was found but failed to clone from ${this.gitUrl}. Error ${error}`)
+      }
+      const bowerJSON = await fs.readJSON(path.join(pluginPath, 'bower.json'))
+      bowerJSON._gitUrl = this.gitUrl
+      bowerJSON._gitRef = this.gitRef || undefined
+      bowerJSON._wasInstalledFromGitRepo = true
+      await fs.writeJSON(path.join(pluginPath, '.bower.json'), bowerJSON, { spaces: 2, replacer: null })
+      this._projectInfo = null
+      await this.fetchProjectInfo()
+      return
+    }
     if (clone) {
       // clone install
       const repoDetails = await this.getRepositoryUrl()
@@ -238,6 +266,32 @@ export default class Target extends Plugin {
     const typeFolder = await this.getTypeFolder()
     const outputPath = path.join(this.cwd, 'src', typeFolder)
     const pluginPath = path.join(outputPath, this.name)
+    if (this.isGitSource) {
+      this.preUpdateProjectVersion = this.projectVersion
+      try {
+        await fs.rm(pluginPath, { recursive: true, force: true })
+      } catch (err) {
+        throw new Error(`There was a problem writing to the target directory ${pluginPath}`)
+      }
+      try {
+        await new Promise((resolve, reject) => {
+          exec(`git clone ${this.gitUrl} "${pluginPath}"`, (err) => {
+            if (err) return reject(err)
+            resolve()
+          })
+        })
+      } catch (error) {
+        throw new Error(`The plugin was found but failed to clone from ${this.gitUrl}. Error ${error}`)
+      }
+      const bowerJSON = await fs.readJSON(path.join(pluginPath, 'bower.json'))
+      bowerJSON._gitUrl = this.gitUrl
+      bowerJSON._gitRef = this.gitRef || undefined
+      bowerJSON._wasInstalledFromGitRepo = true
+      await fs.writeJSON(path.join(pluginPath, '.bower.json'), bowerJSON, { spaces: 2, replacer: null })
+      this._projectInfo = null
+      await this.fetchProjectInfo()
+      return
+    }
     try {
       await fs.rm(pluginPath, { recursive: true, force: true })
     } catch (err) {