Passing ast as custom header to table fetches for optimizations (#11)

* Passing ast as custom header to table fetches for optimizations

We're facing some performance problems while fetching full tables with
huge amounts of rows.
This PR changes how tentacle fetches the tables by passing them the
query ast to the endpoint so we could have a mechanism to optimize this
table generations.

* fix linter

* lint fix

* avoid one extra call to parseSql

src/executor/index.spec.ts

@@ -1,6 +1,7 @@
 import fetch from 'node-fetch'
 import executor from './index'
 import { version } from '../../package.json'
+import { parseSql } from './queryParser'
 
 jest.mock('node-fetch')
 
@@ -56,11 +57,12 @@ test('executor', async () => {
   const sql = 'SELECT employees.id + goal_configs.id as value FROM goal_configs JOIN employees ON (employees.id + ?) == goal_configs.id'
 
   const result = await executor(sql, [5], headers)
+  const ast = JSON.stringify(parseSql(sql))
 
   expect(mockedFetch).toHaveBeenCalledTimes(3)
   expect(mockedFetch).toHaveBeenCalledWith('https://api.example.com/schema', { headers })
-  expect(mockedFetch).toHaveBeenCalledWith('https://api.example.com/tables/goal_configs', { headers })
-  expect(mockedFetch).toHaveBeenCalledWith('https://api.example.com/tables/employees', { headers })
+  expect(mockedFetch).toHaveBeenCalledWith('https://api.example.com/tables/goal_configs', { headers: { ...headers, 'x-tentacle-query-ast': ast } })
+  expect(mockedFetch).toHaveBeenCalledWith('https://api.example.com/tables/employees', { headers: { ...headers, 'x-tentacle-query-ast': ast } })
   expect(result).toEqual([{ value: 25 }])
 })
 

src/executor/index.ts

@@ -3,7 +3,7 @@ import { version } from '../../package.json'
 
 import type { Database as DatabaseType } from 'better-sqlite3'
 
-import { extractTables } from './queryParser'
+import { extractTables, parseSql } from './queryParser'
 import { createDatabase } from './database'
 import {
   fetchSchema,
@@ -112,17 +112,21 @@ async function executor (
 
   const db = createDatabase(config.extensions)
 
-  const usedTables = extractTables(sql)
+  const ast = parseSql(sql)
+  const usedTables = extractTables(ast)
+
   if (usedTables.length > 0) {
     delete headers['content-length']
+
     const schema: Schema = await fetchSchema(headers, config.schema)
 
     await populateTables(
       db,
       usedTables, {
         ...headers,
         host: getHost(),
-        'user-agent': `tentaclesql/${version}`
+        'user-agent': `tentaclesql/${version}`,
+        'x-tentacle-query-ast': JSON.stringify(ast)
       },
       schema
     )

src/executor/queryParser/index.spec.ts

@@ -1,8 +1,9 @@
-import { extractTables } from './index'
+import { extractTables, parseSql } from './index'
 
 test('extractTables', () => {
   const subject = (sql: string) => {
-    return extractTables(sql)
+    const ast = parseSql(sql)
+    return extractTables(ast)
   }
 
   expect(subject('SELECT goal_config_id, name FROM goals_config WHERE goal_config_id > (SELECT COUNT(*) - 100 FROM employee) AND 1=1')).toEqual([

src/executor/queryParser/index.ts

@@ -15,13 +15,11 @@ function validateQuery (ast: any) {
   if (!['select', 'compound'].includes(ast.statement[0].variant)) throw new Error('Only SELECT queries are supported')
 }
 
-function parse (sql: string) {
+export function parseSql (sql: string) {
   return sqliteParser(sql)
 }
 
-export function extractTables (sql: string) {
-  const ast = parse(sql)
-
+export function extractTables (ast: any) {
   validateQuery(ast)
 
   const tables: Array<string> = []