HTTP FS demo

electric-sql · Oct 3, 2024 · 0700c43 · 0700c43
1 parent c16a1ce
commit 0700c43
Show file tree

Hide file tree

Showing 19 changed files with 1,179 additions and 242 deletions.
diff --git a/demos/pagila-httpvfs/.gitignore b/demos/pagila-httpvfs/.gitignore
@@ -0,0 +1,27 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?
+
+public/pagila
+public/pagila.tar.gz
diff --git a/demos/pagila-httpvfs/README.md b/demos/pagila-httpvfs/README.md
@@ -0,0 +1,25 @@
+# PGlite + HttpFs Demo
+
+This demo shows how to use PGlite, a WASM build of Postgres running entirely in the browser, with the WIP HttpFs to connect to a remote PGlite database. It's using HTTP range requests to fetch database file pages from the remote server on demand.
+
+The database in this demo is the Pagila sample database.
+
+## Development
+
+Install the dependencies:
+
+```
+pnpm install
+```
+
+Build the database:
+
+```
+pnpm make-database
+```
+
+Start the dev server:
+
+```
+pnpm dev
+```
diff --git a/demos/pagila-httpvfs/eslint.config.js b/demos/pagila-httpvfs/eslint.config.js
@@ -0,0 +1,28 @@
+import js from '@eslint/js'
+import globals from 'globals'
+import reactHooks from 'eslint-plugin-react-hooks'
+import reactRefresh from 'eslint-plugin-react-refresh'
+import tseslint from 'typescript-eslint'
+
+export default tseslint.config(
+  { ignores: ['dist'] },
+  {
+    extends: [js.configs.recommended, ...tseslint.configs.recommended],
+    files: ['**/*.{ts,tsx}'],
+    languageOptions: {
+      ecmaVersion: 2020,
+      globals: globals.browser,
+    },
+    plugins: {
+      'react-hooks': reactHooks,
+      'react-refresh': reactRefresh,
+    },
+    rules: {
+      ...reactHooks.configs.recommended.rules,
+      'react-refresh/only-export-components': [
+        'warn',
+        { allowConstantExport: true },
+      ],
+    },
+  },
+)
diff --git a/demos/pagila-httpvfs/index.html b/demos/pagila-httpvfs/index.html
@@ -0,0 +1,13 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" type="image/svg+xml" href="/icon.svg" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>PGlite + HttpFs Demo</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.tsx"></script>
+  </body>
+</html>
diff --git a/demos/pagila-httpvfs/package.json b/demos/pagila-httpvfs/package.json
@@ -0,0 +1,36 @@
+{
+  "name": "pagila-httpvfs",
+  "private": true,
+  "version": "0.0.0",
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "tsc -b && vite build",
+    "lint": "eslint .",
+    "preview": "vite preview",
+    "make-database": "tsx scripts/make-database.ts"
+  },
+  "dependencies": {
+    "@electric-sql/pglite": "workspace:*",
+    "@electric-sql/pglite-react": "workspace:*",
+    "@electric-sql/pglite-repl": "workspace:*",
+    "react": "^18.3.1",
+    "react-dom": "^18.3.1",
+    "tar": "^7.4.3",
+    "tsx": "^4.19.1"
+  },
+  "devDependencies": {
+    "@eslint/js": "^9.9.0",
+    "@types/node": "^20.11.18",
+    "@types/react": "^18.3.3",
+    "@types/react-dom": "^18.3.0",
+    "@vitejs/plugin-react": "^4.3.1",
+    "eslint": "^9.9.0",
+    "eslint-plugin-react-hooks": "^5.1.0-rc.0",
+    "eslint-plugin-react-refresh": "^0.4.9",
+    "globals": "^15.9.0",
+    "typescript": "^5.5.3",
+    "typescript-eslint": "^8.0.1",
+    "vite": "^5.4.1"
+  }
+}
diff --git a/demos/pagila-httpvfs/public/icon.svg b/demos/pagila-httpvfs/public/icon.svg
diff --git a/demos/pagila-httpvfs/scripts/make-database.ts b/demos/pagila-httpvfs/scripts/make-database.ts
@@ -0,0 +1,60 @@
+import { PGlite } from '@electric-sql/pglite'
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { extract as tarExtract } from 'tar'
+
+const SCHEMA_URL =
+  'https://raw.githubusercontent.com/devrimgunduz/pagila/refs/heads/master/pagila-schema.sql'
+const DATA_URL =
+  'https://raw.githubusercontent.com/devrimgunduz/pagila/refs/heads/master/pagila-insert-data.sql'
+const THIS_DIR = path.dirname(fileURLToPath(import.meta.url))
+
+// Download the schema and data from the internet
+console.log('Downloading schema...')
+const schema = await fetch(SCHEMA_URL).then((r) => r.text())
+console.log('Downloading data...')
+const data = await fetch(DATA_URL).then((r) => r.text())
+
+// Create a new PGlite instance
+console.log('Creating database...')
+const pg = await PGlite.create()
+
+// Initialize the schema
+console.log('Initializing database schema...')
+await pg.exec(schema)
+
+// Split the data into lines and execute each line so as to not run out of memory
+console.log('Inserting database data...')
+const dataLines = data.split('\n').filter((line) => line.trim().length > 0 && !line.startsWith('--'))
+for (const line of dataLines) {
+  try {
+    await pg.exec(line)
+  } catch (e) {
+    console.error(line)
+    console.error(e)
+    process.exit(1)
+  }
+}
+
+console.log('Vacuuming database...')
+await pg.exec('VACUUM ANALYZE')
+await pg.exec('CHECKPOINT')
+
+console.log('Dumping database...')
+const file = await pg.dumpDataDir()
+
+console.log('Writing database...')
+await fs.writeFile(
+  path.join(THIS_DIR, '..', 'public', 'pagila.tar.gz'),
+  Buffer.from(await file.arrayBuffer()),
+)
+
+console.log('Extracting database...')
+await fs.mkdir(path.join(THIS_DIR, '..', 'public', 'pagila'))
+await tarExtract({
+  file: path.join(THIS_DIR, '..', 'public', 'pagila.tar.gz'),
+  cwd: path.join(THIS_DIR, '..', 'public', 'pagila'),
+})
+
+console.log('Done!')
diff --git a/demos/pagila-httpvfs/src/App.css b/demos/pagila-httpvfs/src/App.css
@@ -0,0 +1,20 @@
+#root {
+  max-width: 1280px;
+  margin: 0 auto;
+  padding: 1rem;
+  text-align: center;
+  width: 100%;
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+}
+
+.PGliteRepl-root {
+  height: 60vh;
+  width: 100%;
+  text-align: left;
+}
+
+.intro {
+  max-width: 60rem;
+}
diff --git a/demos/pagila-httpvfs/src/App.tsx b/demos/pagila-httpvfs/src/App.tsx
@@ -0,0 +1,51 @@
+import { PGliteWorker } from '@electric-sql/pglite/worker'
+import { Repl } from '@electric-sql/pglite-repl'
+import { useEffect, useState } from 'react'
+import PGWorker from './pglite-worker.js?worker'
+import './App.css'
+
+let pgPromise: Promise<PGliteWorker>
+
+function App() {
+  pgPromise ??= PGliteWorker.create(
+    new PGWorker({
+      name: 'pglite-worker',
+    }),
+  )
+  const [pg, setPg] = useState<PGliteWorker | null>(null)
+  useEffect(() => {
+    pgPromise.then(setPg)
+  }, [])
+
+  return (
+    <>
+      <h1>
+        <a href="https://pglite.dev">PGlite</a> +{' '}
+        <a href="https://github.com/electric-sql/pglite/tree/main/packages/httpfs">
+          HttpFs
+        </a>
+      </h1>
+      <div className="intro">
+        <p>
+          This demo shows how to use <a href="https://pglite.dev">PGlite</a>, a
+          WASM build of Postgres running entirely in the browser, with the WIP
+          HttpFs to connect to a remote PGlite database. It's using HTTP range
+          requests to fetch database file pages from the remote server on
+          demand.
+        </p>
+        <p>
+          The database in this demo is the{' '}
+          <a href="https://github.com/devrimgunduz/pagila">Pagila</a> sample
+          database.
+        </p>
+        <p>
+          The REPL below supports the same <code>\d</code> commands as{' '}
+          <code>psql</code>.
+        </p>
+      </div>
+      {pg ? <Repl pg={pg} border={true} /> : <p>Loading...</p>}
+    </>
+  )
+}
+
+export default App
diff --git a/demos/pagila-httpvfs/src/index.css b/demos/pagila-httpvfs/src/index.css
@@ -0,0 +1,72 @@
+:root {
+  font-family: Inter, system-ui, Avenir, Helvetica, Arial, sans-serif;
+  line-height: 1.5;
+  font-weight: 400;
+  font-size: 14px;
+
+  color-scheme: light dark;
+  color: rgba(255, 255, 255, 0.87);
+  background-color: #242424;
+
+  font-synthesis: none;
+  text-rendering: optimizeLegibility;
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+}
+
+a {
+  font-weight: 500;
+  color: #646cff;
+  text-decoration: inherit;
+}
+a:hover {
+  color: #535bf2;
+}
+
+body {
+  margin: 0;
+  display: flex;
+  place-items: center;
+}
+
+h1 {
+  font-size: 1.5em;
+  line-height: 1.1;
+  font-weight: 600;
+}
+
+h1 a {
+  font-weight: 600;
+}
+
+button {
+  border-radius: 8px;
+  border: 1px solid transparent;
+  padding: 0.6em 1.2em;
+  font-size: 1em;
+  font-weight: 500;
+  font-family: inherit;
+  background-color: #1a1a1a;
+  cursor: pointer;
+  transition: border-color 0.25s;
+}
+button:hover {
+  border-color: #646cff;
+}
+button:focus,
+button:focus-visible {
+  outline: 4px auto -webkit-focus-ring-color;
+}
+
+@media (prefers-color-scheme: light) {
+  :root {
+    color: #213547;
+    background-color: #ffffff;
+  }
+  a:hover {
+    color: #747bff;
+  }
+  button {
+    background-color: #f9f9f9;
+  }
+}
diff --git a/demos/pagila-httpvfs/src/main.tsx b/demos/pagila-httpvfs/src/main.tsx
@@ -0,0 +1,10 @@
+import { StrictMode } from 'react'
+import { createRoot } from 'react-dom/client'
+import App from './App.tsx'
+import './index.css'
+
+createRoot(document.getElementById('root')!).render(
+  <StrictMode>
+    <App />
+  </StrictMode>,
+)
diff --git a/demos/pagila-httpvfs/src/pglite-worker.ts b/demos/pagila-httpvfs/src/pglite-worker.ts
@@ -0,0 +1,14 @@
+import { worker } from "@electric-sql/pglite/worker";
+import { HttpFs } from "@electric-sql/pglite/httpfs/browser";
+import { PGlite } from "@electric-sql/pglite";
+
+worker({
+  async init() {
+    const pg = await PGlite.create({
+      fs: new HttpFs("/pagila", { 
+        fetchGranularity: 'page', // 'file' or 'page'
+      }),
+    });
+    return pg;
+  },
+});
diff --git a/demos/pagila-httpvfs/src/vite-env.d.ts b/demos/pagila-httpvfs/src/vite-env.d.ts
@@ -0,0 +1 @@
+/// <reference types="vite/client" />
diff --git a/demos/pagila-httpvfs/tsconfig.app.json b/demos/pagila-httpvfs/tsconfig.app.json
@@ -0,0 +1,24 @@
+{
+  "compilerOptions": {
+    "target": "ES2020",
+    "useDefineForClassFields": true,
+    "lib": ["ES2020", "DOM", "DOM.Iterable"],
+    "module": "ESNext",
+    "skipLibCheck": true,
+
+    /* Bundler mode */
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "isolatedModules": true,
+    "moduleDetection": "force",
+    "noEmit": true,
+    "jsx": "react-jsx",
+
+    /* Linting */
+    "strict": true,
+    "noUnusedLocals": true,
+    "noUnusedParameters": true,
+    "noFallthroughCasesInSwitch": true
+  },
+  "include": ["src"]
+}