Merge pull request #203 from EdgeApp/paul/addReplication

Paul/add replication

Author: paullinator

.vscode/launch.json

@@ -118,6 +118,20 @@
       "skipFiles": [
         "<node_internals>/**"
       ]
+    },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "setup",
+      "args": [
+        "-r",
+        "sucrase/register",
+        "${workspaceFolder}/src/initDbs.ts"
+      ],
+      "internalConsoleOptions": "openOnSessionStart",
+      "skipFiles": [
+        "<node_internals>/**"
+      ]
     }
 
   ]

package.json

@@ -17,7 +17,8 @@
     "fio:promo": "node -r sucrase/register src/bin/fioPromo/fioPromo.ts",
     "lifi": "node -r sucrase/register src/bin/lifiReporter.ts",
     "precommit": "lint-staged && npm run prepare",
-    "prepare": "./scripts/prepare.sh && npm-run-all clean configure -p build.*",
+    "prepare": "./scripts/prepare.sh && npm-run-all clean configure -p build.* && npm run setup",
+    "setup": "node -r sucrase/register src/initDbs.ts",
     "start": "node -r sucrase/register src/indexQuery.ts",
     "start:cache": "node -r sucrase/register src/indexCache.ts",
     "start:rates": "node -r sucrase/register src/indexRates.ts",
@@ -49,7 +50,7 @@
     "csv-stringify": "^6.2.0",
     "date-fns": "^2.16.1",
     "date-fns-tz": "^1.1.4",
-    "edge-server-tools": "^0.2.19",
+    "edge-server-tools": "^0.2.22",
     "express": "^4.17.3",
     "express-promise-router": "^4.1.1",
     "nano": "^10.1.0",

src/cacheEngine.ts

@@ -3,7 +3,6 @@ import nano from 'nano'
 import { getAnalytics } from './apiAnalytics'
 import { config } from './config'
 import { asDbReq } from './dbutils'
-import { initDbs } from './initDbs'
 import { asApps } from './types'
 import {
   datelog,
@@ -30,8 +29,6 @@ export async function cacheEngine(): Promise<void> {
   datelog('Starting Cache Engine')
   console.time('cacheEngine')
 
-  await initDbs()
-
   const reportsApps = nanoDb.use('reports_apps')
   const reportsTransactions = nanoDb.use('reports_transactions')
   const reportsHour = nanoDb.use('reports_hour')

src/config.ts

@@ -1,5 +1,6 @@
 import { makeConfig } from 'cleaner-config'
 import { asArray, asNumber, asObject, asOptional, asString } from 'cleaners'
+import { asCouchCredentials } from 'edge-server-tools'
 
 export const asConfig = asObject({
   couchDbFullpath: asOptional(
@@ -15,7 +16,9 @@ export const asConfig = asObject({
   soloPartnerIds: asOptional(asArray(asString), null),
 
   timeoutOverrideMins: asOptional(asNumber, 1200),
-  cacheLookbackMonths: asOptional(asNumber, 24)
+  cacheLookbackMonths: asOptional(asNumber, 24),
+  couchMainCluster: asOptional(asString, 'wusa'),
+  couchUris: asOptional(asCouchCredentials)
 })
 
 export const config = makeConfig(asConfig, 'config.json')

src/initDbs.ts

@@ -1,9 +1,13 @@
 import {
+  asReplicatorSetupDocument,
+  connectCouch,
   DatabaseSetup,
   JsDesignDocument,
   makeMangoIndex,
   MangoDesignDocument,
-  setupDatabase
+  setupDatabase,
+  SetupDatabaseOptions,
+  syncedDocument
 } from 'edge-server-tools'
 
 import { config } from './config'
@@ -106,10 +110,40 @@ const databases = [
   monthDatabaseSetup
 ]
 
+export const reportsReplication = syncedDocument(
+  'replication',
+  asReplicatorSetupDocument
+)
+
+const couchSettingsSetup: DatabaseSetup = {
+  name: 'reports_settings',
+  syncedDocuments: [reportsReplication]
+}
+
+const options: SetupDatabaseOptions = {
+  replicatorSetup: reportsReplication
+}
+
 export async function initDbs(): Promise<void> {
-  await Promise.all(
-    databases.map(
-      async setup => await setupDatabase(config.couchDbFullpath, setup)
+  if (config.couchUris != null) {
+    const pool = connectCouch(config.couchMainCluster, config.couchUris)
+    await setupDatabase(pool, couchSettingsSetup, options)
+    await Promise.all(
+      databases.map(async setup => await setupDatabase(pool, setup, options))
+    )
+  } else {
+    await Promise.all(
+      databases.map(
+        async setup =>
+          await setupDatabase(config.couchDbFullpath, setup, options)
+      )
     )
-  )
+  }
+  console.log('Done')
+  process.exit(0)
 }
+
+initDbs().catch(err => {
+  console.error(err)
+  process.exit(1)
+})

src/queryEngine.ts

@@ -2,7 +2,6 @@ import nano from 'nano'
 
 import { config } from './config'
 import { pagination } from './dbutils'
-import { initDbs } from './initDbs'
 import { banxa } from './partners/banxa'
 import { bitaccess } from './partners/bitaccess'
 import { bitrefill } from './partners/bitrefill'
@@ -78,8 +77,6 @@ const snooze: Function = async (ms: number) =>
   await new Promise((resolve: Function) => setTimeout(resolve, ms))
 
 export async function queryEngine(): Promise<void> {
-  await initDbs()
-
   const dbProgress = nanoDb.db.use('reports_progresscache')
   const dbApps = nanoDb.db.use('reports_apps')
 

yarn.lock

@@ -2713,10 +2713,10 @@ ecc-jsbn@~0.1.1:
     jsbn "~0.1.0"
     safer-buffer "^2.1.0"
 
-edge-server-tools@^0.2.19:
-  version "0.2.19"
-  resolved "https://registry.yarnpkg.com/edge-server-tools/-/edge-server-tools-0.2.19.tgz#495a7c691b32c9a2d9abee3115bca38ad72efd5e"
-  integrity sha512-iCHxop9064CwrJtpgmbXlMfx93FuNiwmq63c1q/i+mSk5Q0+QCRVP81GXmuxzm9ts0z2s4smhkbd715QTH2Ebw==
+edge-server-tools@^0.2.22:
+  version "0.2.22"
+  resolved "https://registry.yarnpkg.com/edge-server-tools/-/edge-server-tools-0.2.22.tgz#c4571747a25ee1d3ae27613a718fe2356c21c70c"
+  integrity sha512-C7U3Cc+smlVzPtOy6pwQbyFyrBxwj80hCePmHOK0VwGPCjfrXpYqlw/sdwdRY5gYFfm6vVoq5KrEb3hxhSJNwQ==
   dependencies:
     cleaners "^0.3.11"
     nano "^10.1.3"