Merge pull request #16 from deco-cx/try1

rebase fresh

src/build/aot_snapshot.ts

@@ -12,8 +12,8 @@ export class AotSnapshot implements BuildSnapshot {
     this.#dependencies = dependencies;
   }
 
-  get paths(): string[] {
-    return Array.from(this.#files.keys());
+  get paths(): Promise<string[]> {
+    return Promise.resolve(Array.from(this.#files.keys()));
   }
 
   async read(path: string): Promise<ReadableStream<Uint8Array> | null> {
@@ -31,7 +31,7 @@ export class AotSnapshot implements BuildSnapshot {
     return null;
   }
 
-  dependencies(path: string): string[] {
-    return this.#dependencies.get(path) ?? [];
+  dependencies(path: string): Promise<string[]> {
+    return Promise.resolve(this.#dependencies.get(path) ?? []);
   }
 }

src/build/esbuild.ts

@@ -7,7 +7,9 @@ import {
   regexpEscape,
   toFileUrl,
 } from "./deps.ts";
-import { Builder, BuildSnapshot } from "./mod.ts";
+import { getSnapJSON, saveSnapshot } from "./kv.ts";
+import { getFile } from "./kvfs.ts";
+import { Builder, BuildSnapshot, BuildSnapshotJson } from "./mod.ts";
 
 export interface EsbuildBuilderOptions {
   /** The build ID. */
@@ -35,7 +37,11 @@ export class EsbuildBuilder implements Builder {
     this.#options = options;
   }
 
-  async build(): Promise<EsbuildSnapshot> {
+  build(): LazySnapshot {
+    return new LazySnapshot(() => this.#build());
+  }
+
+  async #build(): Promise<EsbuildSnapshot> {
     const opts = this.#options;
     try {
       await initEsbuild();
@@ -151,6 +157,60 @@ function buildIdPlugin(buildId: string): esbuildTypes.Plugin {
   };
 }
 
+export class LazySnapshot implements BuildSnapshot {
+  #snapshot: Promise<BuildSnapshot> | null = null;
+  #snapJSON: BuildSnapshotJson | null = null;
+
+  constructor(private getSnapshot: () => Promise<BuildSnapshot>) {}
+
+  async getSnapJSONMemoized() {
+    if (!this.#snapJSON) {
+      this.#snapJSON = await getSnapJSON();
+    }
+
+    return this.#snapJSON;
+  }
+
+  get paths(): Promise<string[]> {
+    return this.getSnapJSONMemoized().then((snap) =>
+      snap?.files ? Object.keys(snap?.files) : []
+    );
+  }
+
+  async read(path: string) {
+    const snap = await this.#snapshot;
+    const content = snap?.read(path) || await getFile(path);
+
+    if (content) {
+      return content;
+    }
+
+    if (this.#snapshot === null) {
+      const start = performance.now();
+      this.#snapshot = this.getSnapshot()
+        .then((snapshot) => {
+          const dur = (performance.now() - start) / 1e3;
+          console.info(` 📦 Fresh bundle: ${dur.toFixed(2)}s`);
+
+          // Save snapshot in the background
+          saveSnapshot(snapshot).catch(console.error);
+
+          return snapshot;
+        });
+    }
+
+    const snapshot = await this.#snapshot;
+
+    return snapshot.read(path);
+  }
+
+  async dependencies(path: string): Promise<string[]> {
+    const snap = await this.getSnapJSONMemoized();
+
+    return snap?.files[path] ?? [];
+  }
+}
+
 export class EsbuildSnapshot implements BuildSnapshot {
   #files: Map<string, Uint8Array>;
   #dependencies: Map<string, string[]>;
@@ -163,15 +223,15 @@ export class EsbuildSnapshot implements BuildSnapshot {
     this.#dependencies = dependencies;
   }
 
-  get paths(): string[] {
-    return Array.from(this.#files.keys());
+  get paths(): Promise<string[]> {
+    return Promise.resolve(Array.from(this.#files.keys()));
   }
 
   read(path: string): Uint8Array | null {
     return this.#files.get(path) ?? null;
   }
 
-  dependencies(path: string): string[] {
-    return this.#dependencies.get(path) ?? [];
+  dependencies(path: string): Promise<string[]> {
+    return Promise.resolve(this.#dependencies.get(path) ?? []);
   }
 }

src/build/kv.ts

@@ -0,0 +1,62 @@
+import { toSnapshotJSON } from "../dev/build.ts";
+import { getFile, housekeep, isSupported, saveFile } from "./kvfs.ts";
+import { BuildSnapshot, BuildSnapshotJson } from "./mod.ts";
+
+const IS_CHUNK = /\/chunk-[a-zA-Z0-9]*.js/;
+const DEPENDENCIES_SNAP = "snapshot.json";
+
+export const getSnapJSON = async (): Promise<BuildSnapshotJson | null> => {
+  const deps = await getFile(DEPENDENCIES_SNAP);
+
+  if (!deps) {
+    return null;
+  }
+
+  return new Response(deps).json();
+};
+
+export const saveSnapJSON = (json: BuildSnapshotJson) =>
+  saveFile(
+    DEPENDENCIES_SNAP,
+    new TextEncoder().encode(
+      JSON.stringify(json),
+    ),
+  );
+
+export const saveSnapshot = async (
+  snapshot: BuildSnapshot,
+) => {
+  if (!isSupported()) return;
+
+  const paths = await snapshot.paths;
+
+  // We need to save chunks first, islands/plugins last so we address esm.sh build instabilities
+  const chunksFirst = paths.sort((a, b) => {
+    const aIsChunk = IS_CHUNK.test(a);
+    const bIsChunk = IS_CHUNK.test(b);
+    const cmp = a > b ? 1 : a < b ? -1 : 0;
+    return aIsChunk && bIsChunk ? cmp : aIsChunk ? -10 : bIsChunk ? 10 : cmp;
+  });
+
+  let start = performance.now();
+  for (const path of chunksFirst) {
+    const content = await snapshot.read(path);
+
+    if (content instanceof ReadableStream) {
+      console.info("streams are not yet supported on KVFS");
+      return;
+    }
+
+    if (content) await saveFile(path, content);
+  }
+
+  await saveSnapJSON(await toSnapshotJSON(snapshot));
+
+  let dur = (performance.now() - start) / 1e3;
+  console.log(` 💾 Save bundle to Deno.KV: ${dur.toFixed(2)}s`);
+
+  start = performance.now();
+  await housekeep();
+  dur = (performance.now() - start) / 1e3;
+  console.log(` 🧹 Housekeep Deno.KV: ${dur.toFixed(2)}s`);
+};

src/build/kvfs.ts

@@ -0,0 +1,70 @@
+import { BUILD_ID } from "../server/build_id.ts";
+
+const CHUNKSIZE = 65536;
+const NAMESPACE = ["_frsh", "js", BUILD_ID];
+
+// @ts-ignore as `Deno.openKv` is still unstable.
+const kv = await Deno.openKv?.().catch((e) => {
+  console.error(e);
+
+  return null;
+});
+
+export const isSupported = () => kv != null;
+
+export const getFile = async (file: string) => {
+  if (!isSupported()) return null;
+
+  const filepath = [...NAMESPACE, file];
+  const metadata = await kv!.get(filepath).catch(() => null);
+
+  if (metadata?.versionstamp == null) {
+    return null;
+  }
+
+  console.log(` 🚣 Streaming from Deno.KV ${file}`);
+
+  return new ReadableStream<Uint8Array>({
+    start: async (sink) => {
+      for await (const chunk of kv!.list({ prefix: filepath })) {
+        sink.enqueue(chunk.value as Uint8Array);
+      }
+      sink.close();
+    },
+  });
+};
+
+export const saveFile = async (file: string, content: Uint8Array) => {
+  if (!isSupported()) return null;
+
+  const filepath = [...NAMESPACE, file];
+  const metadata = await kv!.get(filepath);
+
+  // Current limitation: As of May 2023, KV Transactions only support a maximum of 10 operations.
+  let transaction = kv!.atomic();
+  let chunks = 0;
+  for (; chunks * CHUNKSIZE < content.length; chunks++) {
+    transaction = transaction.set(
+      [...filepath, chunks],
+      content.slice(chunks * CHUNKSIZE, (chunks + 1) * CHUNKSIZE),
+    );
+  }
+  const result = await transaction
+    .set(filepath, chunks)
+    .check(metadata)
+    .commit();
+
+  return result.ok;
+};
+
+export const housekeep = async () => {
+  if (!isSupported()) return null;
+
+  for await (
+    const item of kv!.list({ prefix: ["_frsh", "js"] })
+  ) {
+    if (item.key.includes(BUILD_ID)) continue;
+
+    await kv!.delete(item.key);
+  }
+};

src/build/mod.ts

@@ -1,3 +1,5 @@
+import { LazySnapshot } from "./esbuild.ts";
+
 export {
   EsbuildBuilder,
   type EsbuildBuilderOptions,
@@ -6,12 +8,12 @@ export {
 } from "./esbuild.ts";
 export { AotSnapshot } from "./aot_snapshot.ts";
 export interface Builder {
-  build(): Promise<BuildSnapshot>;
+  build(): LazySnapshot;
 }
 
 export interface BuildSnapshot {
   /** The list of files contained in this snapshot, not prefixed by a slash. */
-  readonly paths: string[];
+  readonly paths: Promise<string[]>;
 
   /** For a given file, return it's contents.
    * @throws If the file is not contained in this snapshot. */
@@ -26,7 +28,7 @@ export interface BuildSnapshot {
   /** For a given entrypoint, return it's list of dependencies.
    *
    * Returns an empty array if the entrypoint does not exist. */
-  dependencies(path: string): string[];
+  dependencies(path: string): Promise<string[]>;
 }
 
 export interface BuildSnapshotJson {

src/dev/build.ts

@@ -1,13 +1,32 @@
 import { getServerContext } from "../server/context.ts";
 import { join } from "../server/deps.ts";
 import { colors, fs } from "./deps.ts";
-import { BuildSnapshotJson } from "../build/mod.ts";
+import { BuildSnapshot, BuildSnapshotJson } from "../build/mod.ts";
 import { BUILD_ID } from "../server/build_id.ts";
 import { InternalFreshOptions } from "../server/types.ts";
 
+export const toSnapshotJSON = async (snapshot: BuildSnapshot) => {
+  // Write dependency snapshot file to disk
+  const jsonSnapshot: BuildSnapshotJson = {
+    build_id: BUILD_ID,
+    files: {},
+  };
+
+  for (const filePath of await snapshot.paths) {
+    const dependencies = await snapshot.dependencies(filePath);
+    jsonSnapshot.files[filePath] = dependencies;
+  }
+
+  return jsonSnapshot;
+};
+
 export async function build(
   config: InternalFreshOptions,
 ) {
+  throw new Error(
+    "AOT Builds not supported in this version. Use the usual way to deploy freshg",
+  );
+
   // Ensure that build dir is empty
   await fs.emptyDir(config.build.outDir);
 
@@ -18,23 +37,14 @@ export async function build(
   const snapshot = await ctx.buildSnapshot();
 
   // Write output files to disk
-  await Promise.all(snapshot.paths.map(async (fileName) => {
+  for (const fileName of await snapshot.paths) {
     const data = await snapshot.read(fileName);
     if (data === null) return;
 
     return Deno.writeFile(join(config.build.outDir, fileName), data);
-  }));
-
-  // Write dependency snapshot file to disk
-  const jsonSnapshot: BuildSnapshotJson = {
-    build_id: BUILD_ID,
-    files: {},
-  };
-  for (const filePath of snapshot.paths) {
-    const dependencies = snapshot.dependencies(filePath);
-    jsonSnapshot.files[filePath] = dependencies;
   }
 
+  const jsonSnapshot = toSnapshotJSON(snapshot);
   const snapshotPath = join(config.build.outDir, "snapshot.json");
   await Deno.writeTextFile(snapshotPath, JSON.stringify(jsonSnapshot, null, 2));
 

src/server/config.ts

@@ -60,7 +60,7 @@ export async function getFreshConfigWithDefaults(
     manifest,
     build: {
       outDir: "",
-      target: opts.build?.target ?? ["chrome99", "firefox99", "safari15"],
+      target: opts.build?.target ?? ["chrome99", "firefox99", "safari12"],
     },
     plugins: opts.plugins ?? [],
     staticDir: "",

src/server/context.ts

@@ -746,7 +746,7 @@ export class ServerContext {
         app: this.#app,
         layouts,
         imports,
-        dependenciesFn,
+        // dependenciesFn,
         renderFn: this.#renderFn,
         url: new URL(req.url),
         params,
@@ -799,7 +799,7 @@ export class ServerContext {
             app: this.#app,
             layouts,
             imports,
-            dependenciesFn,
+            // dependenciesFn,
             renderFn: this.#renderFn,
             url: new URL(req.url),
             params,

src/server/render.ts

@@ -32,7 +32,7 @@ export interface RenderOptions<Data> {
   app: AppModule;
   layouts: LayoutRoute[];
   imports: string[];
-  dependenciesFn: (path: string) => string[];
+  // dependenciesFn: (path: string) => string[];
   url: URL;
   params: Record<string, string | string[]>;
   renderFn: RenderFunction;
@@ -346,7 +346,7 @@ export async function render<Data>(
     bodyHtml,
     imports: opts.imports,
     csp,
-    dependenciesFn: opts.dependenciesFn,
+    // dependenciesFn: opts.dependenciesFn,
     styles: ctx.styles,
     pluginRenderResults: renderResults,
   });