Incremental rebuilds

package.json

@@ -117,7 +117,12 @@
     "graphql": "15.5.0",
     "graphql-language-service-interface": "2.8.2",
     "**/apollo-language-server/graphql": "^15.0.0",
-    "**/@types/graphql-upload/graphql": "^15.0.0"
+    "**/@types/graphql-upload/graphql": "^15.0.0",
+    "@graphql-tools/utils": "7.8.0-alpha-1dce565c.0",
+    "@graphql-tools/code-file-loader": "6.4.0-alpha-1dce565c.0",
+    "@graphql-tools/graphql-file-loader": "6.3.0-alpha-1dce565c.0",
+    "@graphql-tools/json-file-loader": "6.3.0-alpha-1dce565c.0",
+    "@graphql-tools/module-loader": "6.3.0-alpha-1dce565c.0"
   },
   "dependencies": {
     "dotenv": "8.2.0",

packages/graphql-codegen-cli/package.json

@@ -58,9 +58,9 @@
     "chokidar": "^3.4.3",
     "common-tags": "^1.8.0",
     "cosmiconfig": "^7.0.0",
-    "debounce": "^1.2.0",
     "dependency-graph": "^0.11.0",
     "detect-indent": "^6.0.0",
+    "fs-extra": "^9.1.0",
     "glob": "^7.1.6",
     "graphql-config": "^3.2.0",
     "indent-string": "^4.0.0",
@@ -72,7 +72,6 @@
     "listr-update-renderer": "^0.5.0",
     "log-symbols": "^4.0.0",
     "minimatch": "^3.0.4",
-    "mkdirp": "^1.0.4",
     "string-env-interpolation": "^1.0.1",
     "ts-log": "^2.2.3",
     "tslib": "~2.2.0",
@@ -83,7 +82,6 @@
   },
   "devDependencies": {
     "@types/chokidar": "2.1.3",
-    "@types/debounce": "1.2.0",
     "@types/detect-indent": "6.0.0",
     "@types/glob": "7.1.3",
     "@types/inquirer": "7.3.1",
@@ -92,7 +90,6 @@
     "@types/listr": "0.14.2",
     "@types/log-symbols": "3.0.0",
     "@types/minimatch": "3.0.4",
-    "@types/mkdirp": "1.0.1",
     "@types/valid-url": "1.0.3",
     "bdd-stdin": "0.2.0",
     "dotenv": "8.2.0",

packages/graphql-codegen-cli/src/codegen.ts

@@ -9,7 +9,7 @@ import {
 import { codegen } from '@graphql-codegen/core';
 
 import { Renderer, ErrorRenderer } from './utils/listr-renderer';
-import { GraphQLError, GraphQLSchema, DocumentNode, parse } from 'graphql';
+import { GraphQLError, GraphQLSchema, DocumentNode, parse, Source as GraphQLSource } from 'graphql';
 import { getPluginByName } from './plugins';
 import { getPresetByName } from './presets';
 import { debugLog } from './utils/debugging';
@@ -186,6 +186,24 @@ export async function executeCodegen(input: CodegenContext | Types.Config): Prom
             title: hasPreset
               ? `Generate to ${filename} (using EXPERIMENTAL preset "${outputConfig.preset}")`
               : `Generate ${filename}`,
+            skip() {
+              const skip = !context.shouldGenerate(filename);
+
+              if (skip) {
+                // TODO: support presets
+                // preset.buildGeneratesSection creates a list of files that we can't predict at that point...
+
+                // Super important to still push a result even though it's not generated,
+                // otherwise file is going to be removed in watch mode
+                result.push({
+                  filename,
+                  content: 'Skipped because of cache...',
+                  noEmit: true, // prevent an empty file from being emitted
+                });
+              }
+
+              return skip;
+            },
             task: () => {
               let outputSchemaAst: GraphQLSchema;
               let outputSchema: DocumentNode;
@@ -232,6 +250,20 @@ export async function executeCodegen(input: CodegenContext | Types.Config): Prom
                     task: wrapTask(async () => {
                       debugLog(`[CLI] Generating output`);
 
+                      context.setDependencies(
+                        filename,
+                        []
+                          .concat(outputSchemaAst.extensions?.sources?.map((source: GraphQLSource) => source.name))
+                          .concat(outputDocuments.map(d => d.location))
+                          .filter(Boolean)
+                      );
+                      context.setPointers(filename, [
+                        ...rootSchemas,
+                        ...outputSpecificSchemas,
+                        ...rootDocuments,
+                        ...outputSpecificDocuments,
+                      ]);
+
                       const normalizedPluginsArray = normalizeConfig(outputConfig.plugins);
                       const pluginLoader = config.pluginLoader || makeDefaultLoader(context.cwd);
                       const pluginPackages = await Promise.all(

packages/graphql-codegen-cli/src/config.ts

@@ -1,13 +1,17 @@
 import { cosmiconfig, defaultLoaders } from 'cosmiconfig';
-import { resolve } from 'path';
-import { DetailedError, Types } from '@graphql-codegen/plugin-helpers';
+import { resolve, isAbsolute } from 'path';
+import isGlob from 'is-glob';
 import { env } from 'string-env-interpolation';
 import yargs from 'yargs';
 import { GraphQLConfig } from 'graphql-config';
-import { findAndLoadGraphQLConfig } from './graphql-config';
-import { loadSchema, loadDocuments, defaultSchemaLoadOptions, defaultDocumentsLoadOptions } from './load';
+import minimatch from 'minimatch';
 import { GraphQLSchema } from 'graphql';
 import yaml from 'yaml';
+import { Source } from '@graphql-tools/utils';
+import { cwd } from 'process';
+import { DetailedError, Types } from '@graphql-codegen/plugin-helpers';
+import { findAndLoadGraphQLConfig } from './graphql-config';
+import { loadSchema, loadDocuments, defaultSchemaLoadOptions, defaultDocumentsLoadOptions } from './load';
 
 export type YamlCliFlags = {
   config: string;
@@ -17,6 +21,7 @@ export type YamlCliFlags = {
   project: string;
   silent: boolean;
   errorsOnly: boolean;
+  experimentalCache: boolean;
 };
 
 function generateSearchPlaces(moduleName: string) {
@@ -171,6 +176,11 @@ export function buildOptions() {
       describe: 'Name of a project in GraphQL Config',
       type: 'string' as const,
     },
+    experimentalCache: {
+      describe: 'Experimental cache for watch mode',
+      default: false,
+      type: 'boolean' as const,
+    },
   };
 }
 
@@ -190,14 +200,18 @@ export async function createContext(cliFlags: YamlCliFlags = parseArgv(process.a
 }
 
 export function updateContextWithCliFlags(context: CodegenContext, cliFlags: YamlCliFlags) {
-  const config: Partial<Types.Config & { configFilePath?: string }> = {
+  const config: Partial<Types.Config & { configFilePath?: string; experimentalCache?: boolean }> = {
     configFilePath: context.filepath,
   };
 
   if (cliFlags.watch) {
     config.watch = cliFlags.watch;
   }
 
+  if (cliFlags.experimentalCache) {
+    config.experimentalCache = true;
+  }
+
   if (cliFlags.overwrite === true) {
     config.overwrite = cliFlags.overwrite;
   }
@@ -217,12 +231,165 @@ export function updateContextWithCliFlags(context: CodegenContext, cliFlags: Yam
   context.updateConfig(config);
 }
 
+function isPromiseLike<T>(value: Promise<T> | T): value is Promise<T> {
+  return typeof (value as any).then === 'function';
+}
+
+class Cache {
+  isEnabled: () => boolean = () => false;
+  private invalidated: string[] = [];
+  private sources = new Map<string, Source>();
+  private outputs = new Map<
+    string,
+    {
+      pointers: string[];
+      dependencies: string[];
+    }
+  >();
+
+  constructor(private cwd: string) {}
+
+  cacheSource(fn: (pointer: string, options: any) => Source, pointer: string, options: any): Source;
+  cacheSource(fn: (pointer: string, options: any) => Promise<Source>, pointer: string, options: any): Promise<Source>;
+  cacheSource(fn: (pointer: string, options: any) => Promise<Source> | Source, pointer: string, options: any) {
+    if (!this.isEnabled()) {
+      return fn(pointer, options);
+    }
+
+    const absolutePointer = ensureAbsolutePath(pointer, options);
+
+    if (this.sources.has(absolutePointer)) {
+      return this.sources.get(absolutePointer)!;
+    }
+
+    const result = fn(pointer, options);
+
+    if (isPromiseLike(result)) {
+      return result.then(source => {
+        this.sources.set(absolutePointer, source);
+        return source;
+      });
+    }
+
+    this.sources.set(absolutePointer, result);
+
+    return result;
+  }
+
+  invalidate(filepaths: string[]) {
+    if (!this.isEnabled()) {
+      return;
+    }
+
+    this.invalidated = filepaths.map(filepath => {
+      const absoluteFilepath = ensureAbsolutePath(filepath, {
+        cwd: this.cwd,
+      });
+      this.sources.delete(absoluteFilepath);
+
+      return absoluteFilepath;
+    });
+  }
+
+  shouldGenerate(filepath: string): boolean {
+    if (!this.isEnabled()) {
+      return true;
+    }
+
+    // TODO: if new file - generate everything
+
+    // check if output was already processeed
+    // if not, mark as "to generate"
+    if (this.outputs.has(filepath)) {
+      // if nothing to invalidate, do not generate
+      if (this.invalidated.length) {
+        // filepath matches a pointer
+        if (
+          // iterate over all related pointers
+          this.outputs.get(filepath).pointers.some(
+            pointer =>
+              // collect only matching paths and check if the list is not empty
+              this.invalidated.filter(
+                minimatch.filter(pointer, {
+                  matchBase: true,
+                })
+              ).length > 0
+          )
+        ) {
+          // in case we found matching files, generate the output
+          return true;
+        }
+
+        // filepath is already a dependency of the output
+        if (this.outputs.get(filepath).dependencies.some(f => this.invalidated.includes(f))) {
+          return true;
+        }
+      }
+
+      return false;
+    }
+
+    return true;
+  }
+
+  setDependencies(filepath: string, deps: string[]) {
+    if (!this.isEnabled()) {
+      return;
+    }
+
+    this.ensureOutput(filepath);
+    this.outputs.get(filepath).dependencies = deps.map(dep => ensureAbsolutePath(dep, {}));
+  }
+
+  setPointers(filepath: string, pointers: (Types.Schema | Types.OperationDocument)[]) {
+    if (!this.isEnabled()) {
+      return;
+    }
+
+    this.ensureOutput(filepath);
+    this.outputs.get(filepath).pointers = pointers
+      .map(pointer => {
+        if (typeof pointer === 'string') {
+          // collect only globs
+          return isGlob(pointer) ? ensureAbsolutePath(pointer, { cwd: this.cwd }) : null;
+        }
+
+        if (typeof pointer === 'object') {
+          for (const key in pointer) {
+            const options = pointer[key];
+
+            // if it has a loader, we can't assume it's pure and does not provide any extra contents
+            if (options && 'loader' in options) {
+              return null;
+            }
+
+            return isGlob(key) ? ensureAbsolutePath(key, { cwd: this.cwd }) : null;
+          }
+        }
+
+        return null;
+      })
+      .filter(val => typeof val === 'string');
+  }
+
+  private ensureOutput(filepath: string) {
+    if (!this.outputs.has(filepath)) {
+      this.outputs.set(filepath, {
+        pointers: [],
+        dependencies: [],
+      });
+    }
+  }
+}
+
 export class CodegenContext {
   private _config: Types.Config;
   private _graphqlConfig?: GraphQLConfig;
   private config: Types.Config;
   private _project?: string;
   private _pluginContext: { [key: string]: any } = {};
+  private cache: Cache;
+
   cwd: string;
   filepath: string;
 
@@ -239,6 +406,8 @@ export class CodegenContext {
     this._graphqlConfig = graphqlConfig;
     this.filepath = this._graphqlConfig ? this._graphqlConfig.filepath : filepath;
     this.cwd = this._graphqlConfig ? this._graphqlConfig.dirpath : process.cwd();
+    this.cache = new Cache(this.cwd);
+    this.cache.isEnabled = () => (this.config as any).experimentalCache === true;
   }
 
   useProject(name?: string) {
@@ -264,6 +433,9 @@ export class CodegenContext {
     return {
       ...extraConfig,
       ...this.config,
+      includeSources: true,
+      cacheable: this.cacheable.bind(this),
+      cacheableSync: this.cacheableSync.bind(this),
     };
   }
 
@@ -298,8 +470,41 @@ export class CodegenContext {
 
     return loadDocuments(pointer, config);
   }
+
+  invalidate(filepaths: string[]): void {
+    this.cache.invalidate(filepaths);
+  }
+
+  shouldGenerate(filepath: string) {
+    return this.cache.shouldGenerate(filepath);
+  }
+
+  setDependencies(filepath: string, deps: string[]) {
+    return this.cache.setDependencies(filepath, deps);
+  }
+
+  setPointers(filepath: string, pointers: (Types.Schema | Types.OperationDocument)[]) {
+    return this.cache.setPointers(filepath, pointers);
+  }
+
+  private cacheable(fn: (pointer: string, options: any) => Promise<Source>, pointer: string, options: any) {
+    return this.cache.cacheSource(fn, pointer, options);
+  }
+
+  private cacheableSync(fn: (pointer: string, options: any) => Source, pointer: string, options: any) {
+    return this.cache.cacheSource(fn, pointer, options);
+  }
 }
 
 export function ensureContext(input: CodegenContext | Types.Config): CodegenContext {
   return input instanceof CodegenContext ? input : new CodegenContext({ config: input });
 }
+
+function ensureAbsolutePath(
+  pointer: string,
+  options: {
+    cwd?: string;
+  }
+): string {
+  return isAbsolute(pointer) ? pointer : resolve(options.cwd || cwd(), pointer);
+}