feat: add e2e tests for config-import. (#700)

* add e2e tests for config-import.

Additionally:
* added `translation` getter in `mapeoProject`
* `deleteAll(translation)` in import config

* rename test

* `deleteTranslations` only delete translations that refer to deleted fields or presets

* better signature for delete functions, simplify `deleteTranslations`

* simplify types

* better error text

* addressing review

* make `deleteTranslations` a closure, pass the logger
* `deleteTranslations` avoid for loop, use Promise.all(map)
* make failing test case instead of logging

* revert `deleteTranslations` being a closure

* add boolean to avoid loading a config in parallel

* refactor `deleteTranslations`

Co-authored-by: Evan Hahn <[email protected]>

* wrap importConfig in a try/catch

* re-invert logic in `deleteTranslations` to please typescript

* fix logic error, change assertion on parallel test

* expose translationApi.dataType

* fix type error

* assert.throws -> assert.rejects

Co-authored-by: Evan Hahn <[email protected]>

* change test message

* fix last test

---------

Co-authored-by: Tomás Ciccola <[email protected]>
Co-authored-by: Evan Hahn <[email protected]>

src/mapeo-project.js

@@ -87,6 +87,8 @@ export class MapeoProject extends TypedEmitter {
   /** @type {TranslationApi} */
   #translationApi
   #l
+  /** @type {Boolean} this avoids loading multiple configs in parallel */
+  #loadingConfig
 
   static EMPTY_PROJECT_SETTINGS = EMPTY_PROJECT_SETTINGS
 
@@ -125,6 +127,7 @@ export class MapeoProject extends TypedEmitter {
     this.#l = Logger.create('project', logger)
     this.#deviceId = getDeviceId(keyManager)
     this.#projectId = projectKeyToId(projectKey)
+    this.#loadingConfig = false
 
     ///////// 1. Setup database
     this.#sqlite = new Database(dbPath)
@@ -717,106 +720,125 @@ export class MapeoProject extends TypedEmitter {
    *  @returns {Promise<Error[]>}
    */
   async importConfig({ configPath }) {
-    // check for already present fields and presets and delete them if exist
-    await deleteAll(this.preset)
-    await deleteAll(this.field)
-
-    const config = await readConfig(configPath)
-    /** @type {Map<string, string>} */
-    const iconNameToId = new Map()
-    /** @type {Map<string, string>} */
-    const fieldNameToId = new Map()
-    /** @type {Map<string,string>} */
-    const presetNameToId = new Map()
-
-    // Do this in serial not parallel to avoid memory issues (avoid keeping all icon buffers in memory)
-    for await (const icon of config.icons()) {
-      const iconId = await this.#iconApi.create(icon)
-      iconNameToId.set(icon.name, iconId)
-    }
-
-    // Ok to create fields and presets in parallel
-    const fieldPromises = []
-    for (const { name, value } of config.fields()) {
-      fieldPromises.push(
-        this.#dataTypes.field.create(value).then(({ docId }) => {
-          fieldNameToId.set(name, docId)
-        })
-      )
-    }
-    await Promise.all(fieldPromises)
+    assert(
+      !this.#loadingConfig,
+      'Cannot run multiple config imports at the same time'
+    )
+    this.#loadingConfig = true
 
-    const presetsWithRefs = []
-    for (const { fieldNames, iconName, value, name } of config.presets()) {
-      const fieldIds = fieldNames.map((fieldName) => {
-        const id = fieldNameToId.get(fieldName)
-        if (!id) {
-          throw new Error(
-            `field ${fieldName} not found (referenced by preset ${value.name})})`
-          )
-        }
-        return id
-      })
-      presetsWithRefs.push({
-        preset: {
-          ...value,
-          iconId: iconName && iconNameToId.get(iconName),
-          fieldIds,
-        },
-        name,
+    try {
+      // check for already present fields and presets and delete them if exist
+      await deleteAll(this.preset)
+      await deleteAll(this.field)
+      // delete only translations that refer to deleted fields and presets
+      await deleteTranslations({
+        logger: this.#l,
+        translation: this.$translation.dataType,
+        presets: this.preset,
+        fields: this.field,
       })
-    }
 
-    const presetPromises = []
-    for (const { preset, name } of presetsWithRefs) {
-      presetPromises.push(
-        this.preset.create(preset).then(({ docId }) => {
-          presetNameToId.set(name, docId)
-        })
-      )
-    }
+      const config = await readConfig(configPath)
+      /** @type {Map<string, string>} */
+      const iconNameToId = new Map()
+      /** @type {Map<string, string>} */
+      const fieldNameToId = new Map()
+      /** @type {Map<string,string>} */
+      const presetNameToId = new Map()
+
+      // Do this in serial not parallel to avoid memory issues (avoid keeping all icon buffers in memory)
+      for await (const icon of config.icons()) {
+        const iconId = await this.#iconApi.create(icon)
+        iconNameToId.set(icon.name, iconId)
+      }
 
-    await Promise.all(presetPromises)
-
-    const translationPromises = []
-    for (const { name, value } of config.translations()) {
-      let docIdRef
-      if (value.schemaNameRef === 'fields') {
-        docIdRef = fieldNameToId.get(name)
-      } else if (value.schemaNameRef === 'presets') {
-        docIdRef = presetNameToId.get(name)
-      } else {
-        throw new Error(`invalid schemaNameRef ${value.schemaNameRef}`)
+      // Ok to create fields and presets in parallel
+      const fieldPromises = []
+      for (const { name, value } of config.fields()) {
+        fieldPromises.push(
+          this.#dataTypes.field.create(value).then(({ docId }) => {
+            fieldNameToId.set(name, docId)
+          })
+        )
       }
-      if (docIdRef) {
-        translationPromises.push(
-          this.$translation.put({
+      await Promise.all(fieldPromises)
+
+      const presetsWithRefs = []
+      for (const { fieldNames, iconName, value, name } of config.presets()) {
+        const fieldIds = fieldNames.map((fieldName) => {
+          const id = fieldNameToId.get(fieldName)
+          if (!id) {
+            throw new Error(
+              `field ${fieldName} not found (referenced by preset ${value.name})})`
+            )
+          }
+          return id
+        })
+        presetsWithRefs.push({
+          preset: {
             ...value,
-            docIdRef,
+            iconId: iconName && iconNameToId.get(iconName),
+            fieldIds,
+          },
+          name,
+        })
+      }
+
+      const presetPromises = []
+      for (const { preset, name } of presetsWithRefs) {
+        presetPromises.push(
+          this.preset.create(preset).then(({ docId }) => {
+            presetNameToId.set(name, docId)
           })
         )
-      } else {
-        throw new Error(
-          `docIdRef for preset or field with name ${name} not found`
-        )
       }
-    }
-    await Promise.all(translationPromises)
-
-    // close the zip handles after we know we won't be needing them anymore
-    await config.close()
-
-    await this.$setProjectSettings({
-      defaultPresets: {
-        point: [...presetNameToId.values()],
-        line: [],
-        area: [],
-        vertex: [],
-        relation: [],
-      },
-    })
 
-    return config.warnings
+      await Promise.all(presetPromises)
+
+      const translationPromises = []
+      for (const { name, value } of config.translations()) {
+        let docIdRef
+        if (value.schemaNameRef === 'fields') {
+          docIdRef = fieldNameToId.get(name)
+        } else if (value.schemaNameRef === 'presets') {
+          docIdRef = presetNameToId.get(name)
+        } else {
+          throw new Error(`invalid schemaNameRef ${value.schemaNameRef}`)
+        }
+        if (docIdRef) {
+          translationPromises.push(
+            this.$translation.put({
+              ...value,
+              docIdRef,
+            })
+          )
+        } else {
+          throw new Error(
+            `docIdRef for preset or field with name ${name} not found`
+          )
+        }
+      }
+      await Promise.all(translationPromises)
+
+      // close the zip handles after we know we won't be needing them anymore
+      await config.close()
+
+      await this.$setProjectSettings({
+        defaultPresets: {
+          point: [...presetNameToId.values()],
+          line: [],
+          area: [],
+          vertex: [],
+          relation: [],
+        },
+      })
+      this.#loadingConfig = false
+      return config.warnings
+    } catch (e) {
+      this.#l.log('error loading config', e)
+      this.#loadingConfig = false
+      return /** @type Error[] */ []
+    }
   }
 }
 
@@ -830,8 +852,7 @@ function extractEditableProjectSettings(projectDoc) {
   return result
 }
 
-// TODO: maybe a better signature than a bunch of any?
-/** @param {DataType<any,any,any,any,any>} dataType */
+/** @param {MapeoProject['field'] | MapeoProject['preset']} dataType */
 async function deleteAll(dataType) {
   const deletions = []
   for (const { docId } of await dataType.getMany()) {
@@ -840,6 +861,33 @@ async function deleteAll(dataType) {
   return Promise.all(deletions)
 }
 
+/**
+ * @param {Object} opts
+ * @param {Logger} opts.logger
+ * @param {MapeoProject['$translation']['dataType']} opts.translation
+ * @param {MapeoProject['preset']} opts.presets
+ * @param {MapeoProject['field']} opts.fields
+ */
+async function deleteTranslations(opts) {
+  const translations = await opts.translation.getMany()
+  await Promise.all(
+    translations.map(async ({ docId, docIdRef, schemaNameRef }) => {
+      if (schemaNameRef === 'presets' || schemaNameRef === 'fields') {
+        let shouldDelete = false
+        try {
+          const toDelete = await opts[schemaNameRef].getByDocId(docIdRef)
+          shouldDelete = toDelete.deleted
+        } catch (e) {
+          opts.logger.log(`referred ${docIdRef} is not found`)
+        }
+        if (shouldDelete) {
+          await opts.translation.delete(docId)
+        }
+      }
+    })
+  )
+}
+
 /**
  * Return a map of namespace -> core keypair
  *

src/translation-api.js

@@ -126,4 +126,7 @@ export default class TranslationApi {
   get [ktranslatedLanguageCodeToSchemaNames]() {
     return this.#translatedLanguageCodeToSchemaNames
   }
+  get dataType() {
+    return this.#dataType
+  }
 }

test-e2e/config-import.js

@@ -0,0 +1,76 @@
+import test from 'node:test'
+import assert from 'node:assert/strict'
+import { createManager } from './utils.js'
+import { defaultConfigPath } from '../tests/helpers/default-config.js'
+
+test(' config import - load default config when passed a path to `createProject`', async (t) => {
+  const manager = createManager('device0', t)
+  const project = await manager.getProject(
+    await manager.createProject({ configPath: defaultConfigPath })
+  )
+  const presets = await project.preset.getMany()
+  const fields = await project.field.getMany()
+  const translations = await project.$translation.dataType.getMany()
+  assert.equal(presets.length, 28, 'correct number of loaded presets')
+  assert.equal(fields.length, 11, 'correct number of loaded fields')
+  assert.equal(
+    translations.length,
+    870,
+    'correct number of loaded translations'
+  )
+})
+
+test('config import - load and re-load config manually', async (t) => {
+  const manager = createManager('device0', t)
+  const project = await manager.getProject(await manager.createProject())
+
+  const warnings = await project.importConfig({ configPath: defaultConfigPath })
+  let presets = await project.preset.getMany()
+  let fields = await project.field.getMany()
+  let translations = await project.$translation.dataType.getMany()
+
+  assert.equal(
+    warnings.length,
+    0,
+    'no warnings when manually loading default config'
+  )
+  assert.equal(presets.length, 28, 'correct number of manually loaded presets')
+  assert.equal(fields.length, 11, 'correct number of manually loaded fields')
+  assert.equal(
+    translations.length,
+    870,
+    'correct number of manually loaded translations'
+  )
+
+  // re load the config
+  await project.importConfig({ configPath: defaultConfigPath })
+  presets = await project.preset.getMany()
+  fields = await project.field.getMany()
+  translations = await project.$translation.dataType.getMany()
+  assert.equal(
+    presets.length,
+    28,
+    're-loading the same config leads to the same number of presets (since they are deleted)'
+  )
+  assert.equal(
+    fields.length,
+    11,
+    're-loading the same config leads to the same number of fields (since they are deleted)'
+  )
+  assert.equal(
+    translations.length,
+    870,
+    're-loading the same config leads to the same number of translations (since they are deleted)'
+  )
+})
+
+test('failing on loading multiple configs in parallel', async (t) => {
+  const manager = createManager('device0', t)
+  const project = await manager.getProject(await manager.createProject())
+  const results = await Promise.allSettled([
+    project.importConfig({ configPath: defaultConfigPath }),
+    project.importConfig({ configPath: defaultConfigPath }),
+  ])
+  assert.equal(results[0]?.status, 'fulfilled', 'first import should work')
+  assert.equal(results[1]?.status, 'rejected', 'second import should fail')
+})