From f4fbbacb1fe40b321b6562674ecbf9a125557211 Mon Sep 17 00:00:00 2001
From: Ali Deishidi <sizief@gmail.com>
Date: Thu, 18 Nov 2021 20:20:01 +0330
Subject: [PATCH] feat(server): Adding bulk fetch to get data at once (#20)

---
 README.md             |   5 ++
 src/executor/index.ts | 130 ++++++++++++++++++++++++++++++++++--------
 2 files changed, 110 insertions(+), 25 deletions(-)

diff --git a/README.md b/README.md
index 30bbc6a..711e7e8 100644
--- a/README.md
+++ b/README.md
@@ -152,3 +152,8 @@ yarn cli query "SELECT 1;"
 # or installed global
 tentaclesql query "SELECT 1;"
 ```
+
+### Bulk fetch
+By default Tentacle sends one HTTP request for each table data, however you can change this and fetch all table data in one HTTP request. To enable this you need to pass following paramaters:
+- `BULK_FETCH=true`
+- `BULK_FETCH_URL=url`
diff --git a/src/executor/index.ts b/src/executor/index.ts
index 158f795..94ba340 100644
--- a/src/executor/index.ts
+++ b/src/executor/index.ts
@@ -69,6 +69,33 @@ async function fetchTableData (
   return res.json()
 }
 
+async function fetchTablesData (
+  tableDefinitions: Array<TableDefinition>,
+  headers: any,
+  queryAst: any,
+  method: 'POST' | 'GET' = 'POST'
+): Promise<any> {
+  if (process.env.BULK_FETCH_URL === undefined) {
+    return Error('Bulk fetch requested but bulk fetch url is not defined.')
+  }
+  const res = await fetch(
+    process.env.BULK_FETCH_URL, {
+      headers: headers,
+      method: method,
+      body: JSON.stringify({
+        query_ast: queryAst,
+        names: tableDefinitions.map((tableDefinition: TableDefinition) => tableDefinition.name)
+      })
+    }
+  )
+
+  if (!res.ok) {
+    return Promise.reject(new Error(`Error with the request. Status code: ${res.status}`))
+  }
+
+  return res.json()
+}
+
 async function populateTables (
   db: IDatabaseAdapter,
   usedTables: Array<string>,
@@ -81,39 +108,66 @@ async function populateTables (
   ) => usedTables.includes(tableDefinition.name))
 
   const promises = filteredTableDefinition.map(async (tableDefinition: TableDefinition) => {
-    const schemas = parseSchema(tableDefinition.fields).join(', ')
+    const data = await fetchTableData(tableDefinition, headers, queryAst)
+    syncData(tableDefinition, data, db)
+  })
+  return Promise.all(promises)
+}
 
-    if (!tableDefinition.autodiscover) {
-      db.createTable(tableDefinition, schemas)
-    }
+async function populateTablesInOneHTTPRequest (
+  db: IDatabaseAdapter,
+  usedTables: Array<string>,
+  headers: any,
+  schema: any,
+  queryAst: any
+) {
+  const filteredTableDefinition = schema.filter((
+    tableDefinition: TableDefinition
+  ) => usedTables.includes(tableDefinition.name))
+  const remoteData = await fetchTablesData(filteredTableDefinition, headers, queryAst)
+  filteredTableDefinition.forEach((tableDefinition: TableDefinition) => {
+    const targetTable = remoteData.find((tableData: any) => tableData.name === tableDefinition.name)
+    syncData(
+      tableDefinition,
+      targetTable.data,
+      db
+    )
+  })
+}
 
-    const data = await fetchTableData(tableDefinition, headers, queryAst)
+function syncData (
+  tableDefinition: TableDefinition,
+  data: any,
+  db: IDatabaseAdapter
+) {
+  const schemas = parseSchema(tableDefinition.fields).join(', ')
 
-    const resultKey = tableDefinition.resultKey
-    const dataPointer = resultKey ? data[resultKey] : data
-    const fixedData = dataPointer.map((field: any) => flattenObject(field, '_'))
+  if (!tableDefinition.autodiscover) {
+    db.createTable(tableDefinition, schemas)
+  }
 
-    if (fixedData.length === 0) return
+  const resultKey = tableDefinition.resultKey
+  const dataPointer = resultKey ? data[resultKey] : data
+  const fixedData = dataPointer.map((field: any) => flattenObject(field, '_'))
 
-    // No support for booleans :/
-    mutateDataframe(fixedData, (row, k) => {
-      if (typeof row[k] === 'boolean') row[k] = row[k] ? 'TRUE' : 'FALSE'
-    })
+  if (fixedData.length === 0) return
 
-    if (tableDefinition.autodiscover) {
-      const dynamicDefinition = {
-        name: tableDefinition.name,
-        fields: Object.keys(fixedData[0]).map((key) => ({ key: key }))
-      }
+  // No support for booleans :/
+  mutateDataframe(fixedData, (row, k) => {
+    if (typeof row[k] === 'boolean') row[k] = row[k] ? 'TRUE' : 'FALSE'
+  })
 
-      db.createTable(dynamicDefinition, schemas)
-      db.storeToDb(dynamicDefinition, fixedData)
-    } else {
-      db.storeToDb(tableDefinition, fixedData)
+  if (tableDefinition.autodiscover) {
+    const dynamicDefinition = {
+      name: tableDefinition.name,
+      fields: Object.keys(fixedData[0]).map((key) => ({ key: key }))
     }
-  })
 
-  return Promise.all(promises)
+    db.createTable(dynamicDefinition, schemas)
+    db.storeToDb(dynamicDefinition, fixedData)
+  } else {
+    db.storeToDb(tableDefinition, fixedData)
+  }
 }
 
 const DEFAULT_CONFIG = {
@@ -121,6 +175,32 @@ const DEFAULT_CONFIG = {
   schema: []
 }
 
+async function runPopulateTables (
+  db: IDatabaseAdapter,
+  usedTables: Array<string>,
+  headers: any,
+  schema: any,
+  ast: any
+) {
+  if (process.env.BULK_FETCH) {
+    await populateTablesInOneHTTPRequest(
+      db,
+      usedTables,
+      headers,
+      schema,
+      ast
+    )
+  } else {
+    await populateTables(
+      db,
+      usedTables,
+      headers,
+      schema,
+      ast
+    )
+  }
+}
+
 async function executor (
   sql: string,
   parameters: Parameters,
@@ -153,7 +233,7 @@ async function executor (
     const headersWithHost = getHost() ? { ...headers, host: getHost() } : { ...headers }
     headersWithHost['user-agent'] = `tentaclesql/${version}`
 
-    await populateTables(
+    await runPopulateTables(
       db,
       usedTables,
       headers,