From fa4985bb50809bd26c43022aacae20f282c645d5 Mon Sep 17 00:00:00 2001
From: Genar Trias Ortiz <genar@factorial.co>
Date: Wed, 1 Sep 2021 17:52:56 +0200
Subject: [PATCH] Passing ast as custom header to table fetches for
 optimizations (#11)

* Passing ast as custom header to table fetches for optimizations

We're facing some performance problems while fetching full tables with
huge amounts of rows.
This PR changes how tentacle fetches the tables by passing them the
query ast to the endpoint so we could have a mechanism to optimize this
table generations.

* fix linter

* lint fix

* avoid one extra call to parseSql
---
 src/executor/index.spec.ts             |  6 ++++--
 src/executor/index.ts                  | 10 +++++++---
 src/executor/queryParser/index.spec.ts |  5 +++--
 src/executor/queryParser/index.ts      |  6 ++----
 4 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/src/executor/index.spec.ts b/src/executor/index.spec.ts
index e17ff76..71eaffc 100644
--- a/src/executor/index.spec.ts
+++ b/src/executor/index.spec.ts
@@ -1,6 +1,7 @@
 import fetch from 'node-fetch'
 import executor from './index'
 import { version } from '../../package.json'
+import { parseSql } from './queryParser'
 
 jest.mock('node-fetch')
 
@@ -56,11 +57,12 @@ test('executor', async () => {
   const sql = 'SELECT employees.id + goal_configs.id as value FROM goal_configs JOIN employees ON (employees.id + ?) == goal_configs.id'
 
   const result = await executor(sql, [5], headers)
+  const ast = JSON.stringify(parseSql(sql))
 
   expect(mockedFetch).toHaveBeenCalledTimes(3)
   expect(mockedFetch).toHaveBeenCalledWith('https://api.example.com/schema', { headers })
-  expect(mockedFetch).toHaveBeenCalledWith('https://api.example.com/tables/goal_configs', { headers })
-  expect(mockedFetch).toHaveBeenCalledWith('https://api.example.com/tables/employees', { headers })
+  expect(mockedFetch).toHaveBeenCalledWith('https://api.example.com/tables/goal_configs', { headers: { ...headers, 'x-tentacle-query-ast': ast } })
+  expect(mockedFetch).toHaveBeenCalledWith('https://api.example.com/tables/employees', { headers: { ...headers, 'x-tentacle-query-ast': ast } })
   expect(result).toEqual([{ value: 25 }])
 })
 
diff --git a/src/executor/index.ts b/src/executor/index.ts
index 1dcde19..c1c7074 100644
--- a/src/executor/index.ts
+++ b/src/executor/index.ts
@@ -3,7 +3,7 @@ import { version } from '../../package.json'
 
 import type { Database as DatabaseType } from 'better-sqlite3'
 
-import { extractTables } from './queryParser'
+import { extractTables, parseSql } from './queryParser'
 import { createDatabase } from './database'
 import {
   fetchSchema,
@@ -112,9 +112,12 @@ async function executor (
 
   const db = createDatabase(config.extensions)
 
-  const usedTables = extractTables(sql)
+  const ast = parseSql(sql)
+  const usedTables = extractTables(ast)
+
   if (usedTables.length > 0) {
     delete headers['content-length']
+
     const schema: Schema = await fetchSchema(headers, config.schema)
 
     await populateTables(
@@ -122,7 +125,8 @@ async function executor (
       usedTables, {
         ...headers,
         host: getHost(),
-        'user-agent': `tentaclesql/${version}`
+        'user-agent': `tentaclesql/${version}`,
+        'x-tentacle-query-ast': JSON.stringify(ast)
       },
       schema
     )
diff --git a/src/executor/queryParser/index.spec.ts b/src/executor/queryParser/index.spec.ts
index 944ce84..4974d64 100644
--- a/src/executor/queryParser/index.spec.ts
+++ b/src/executor/queryParser/index.spec.ts
@@ -1,8 +1,9 @@
-import { extractTables } from './index'
+import { extractTables, parseSql } from './index'
 
 test('extractTables', () => {
   const subject = (sql: string) => {
-    return extractTables(sql)
+    const ast = parseSql(sql)
+    return extractTables(ast)
   }
 
   expect(subject('SELECT goal_config_id, name FROM goals_config WHERE goal_config_id > (SELECT COUNT(*) - 100 FROM employee) AND 1=1')).toEqual([
diff --git a/src/executor/queryParser/index.ts b/src/executor/queryParser/index.ts
index dec32f0..315cd81 100644
--- a/src/executor/queryParser/index.ts
+++ b/src/executor/queryParser/index.ts
@@ -15,13 +15,11 @@ function validateQuery (ast: any) {
   if (!['select', 'compound'].includes(ast.statement[0].variant)) throw new Error('Only SELECT queries are supported')
 }
 
-function parse (sql: string) {
+export function parseSql (sql: string) {
   return sqliteParser(sql)
 }
 
-export function extractTables (sql: string) {
-  const ast = parse(sql)
-
+export function extractTables (ast: any) {
   validateQuery(ast)
 
   const tables: Array<string> = []