Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Incremental rebuilds #6142

Closed
wants to merge 3 commits into from
Closed
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Experimental cache for watch mode
kamilkisiela committed Apr 9, 2021
commit 2a6fe254a3d6304a81b6ddb01721faf213629b90
7 changes: 6 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -117,7 +117,12 @@
"graphql": "15.5.0",
"graphql-language-service-interface": "2.8.2",
"**/apollo-language-server/graphql": "^15.0.0",
"**/@types/graphql-upload/graphql": "^15.0.0"
"**/@types/graphql-upload/graphql": "^15.0.0",
"@graphql-tools/utils": "7.8.0-alpha-1dce565c.0",
"@graphql-tools/code-file-loader": "6.4.0-alpha-1dce565c.0",
"@graphql-tools/graphql-file-loader": "6.3.0-alpha-1dce565c.0",
"@graphql-tools/json-file-loader": "6.3.0-alpha-1dce565c.0",
"@graphql-tools/module-loader": "6.3.0-alpha-1dce565c.0"
},
"dependencies": {
"dotenv": "8.2.0",
5 changes: 1 addition & 4 deletions packages/graphql-codegen-cli/package.json
Original file line number Diff line number Diff line change
@@ -58,9 +58,9 @@
"chokidar": "^3.4.3",
"common-tags": "^1.8.0",
"cosmiconfig": "^7.0.0",
"debounce": "^1.2.0",
"dependency-graph": "^0.11.0",
"detect-indent": "^6.0.0",
"fs-extra": "^9.1.0",
"glob": "^7.1.6",
"graphql-config": "^3.2.0",
"indent-string": "^4.0.0",
@@ -72,7 +72,6 @@
"listr-update-renderer": "^0.5.0",
"log-symbols": "^4.0.0",
"minimatch": "^3.0.4",
"mkdirp": "^1.0.4",
"string-env-interpolation": "^1.0.1",
"ts-log": "^2.2.3",
"tslib": "~2.2.0",
@@ -83,7 +82,6 @@
},
"devDependencies": {
"@types/chokidar": "2.1.3",
"@types/debounce": "1.2.0",
"@types/detect-indent": "6.0.0",
"@types/glob": "7.1.3",
"@types/inquirer": "7.3.1",
@@ -92,7 +90,6 @@
"@types/listr": "0.14.2",
"@types/log-symbols": "3.0.0",
"@types/minimatch": "3.0.4",
"@types/mkdirp": "1.0.1",
"@types/valid-url": "1.0.3",
"bdd-stdin": "0.2.0",
"dotenv": "8.2.0",
28 changes: 27 additions & 1 deletion packages/graphql-codegen-cli/src/codegen.ts
Original file line number Diff line number Diff line change
@@ -9,7 +9,7 @@ import {
import { codegen } from '@graphql-codegen/core';

import { Renderer, ErrorRenderer } from './utils/listr-renderer';
import { GraphQLError, GraphQLSchema, DocumentNode, parse } from 'graphql';
import { GraphQLError, GraphQLSchema, DocumentNode, parse, Source as GraphQLSource } from 'graphql';
import { getPluginByName } from './plugins';
import { getPresetByName } from './presets';
import { debugLog } from './utils/debugging';
@@ -186,6 +186,24 @@ export async function executeCodegen(input: CodegenContext | Types.Config): Prom
title: hasPreset
? `Generate to ${filename} (using EXPERIMENTAL preset "${outputConfig.preset}")`
: `Generate ${filename}`,
skip() {
const skip = !context.shouldGenerate(filename);

if (skip) {
// TODO: support presets
// preset.buildGeneratesSection creates a list of files that we can't predict at that point...

// Super important to still push a result even though it's not generated,
// otherwise file is going to be removed in watch mode
result.push({
filename,
content: 'Skipped because of cache...',
noEmit: true, // prevent an empty file from being emitted
});
}

return skip;
},
task: () => {
let outputSchemaAst: GraphQLSchema;
let outputSchema: DocumentNode;
@@ -232,6 +250,14 @@ export async function executeCodegen(input: CodegenContext | Types.Config): Prom
task: wrapTask(async () => {
debugLog(`[CLI] Generating output`);

context.setDependencies(
filename,
[]
.concat(outputSchemaAst.extensions?.sources?.map((source: GraphQLSource) => source.name))
.concat(outputDocuments.map(d => d.location))
.filter(Boolean)
);

const normalizedPluginsArray = normalizeConfig(outputConfig.plugins);
const pluginLoader = config.pluginLoader || makeDefaultLoader(context.cwd);
const pluginPackages = await Promise.all(
133 changes: 131 additions & 2 deletions packages/graphql-codegen-cli/src/config.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { cosmiconfig, defaultLoaders } from 'cosmiconfig';
import { resolve } from 'path';
import { resolve, isAbsolute } from 'path';
import { DetailedError, Types } from '@graphql-codegen/plugin-helpers';
import { env } from 'string-env-interpolation';
import yargs from 'yargs';
@@ -8,6 +8,8 @@ import { findAndLoadGraphQLConfig } from './graphql-config';
import { loadSchema, loadDocuments, defaultSchemaLoadOptions, defaultDocumentsLoadOptions } from './load';
import { GraphQLSchema } from 'graphql';
import yaml from 'yaml';
import { Source } from '@graphql-tools/utils';
import { cwd } from 'process';

export type YamlCliFlags = {
config: string;
@@ -17,6 +19,7 @@ export type YamlCliFlags = {
project: string;
silent: boolean;
errorsOnly: boolean;
experimentalCache: boolean;
};

function generateSearchPlaces(moduleName: string) {
@@ -171,6 +174,10 @@ export function buildOptions() {
describe: 'Name of a project in GraphQL Config',
type: 'string' as const,
},
experimentalCache: {
describe: 'Use experimental cache for watch mode',
type: 'boolean' as const,
},
};
}

@@ -190,14 +197,18 @@ export async function createContext(cliFlags: YamlCliFlags = parseArgv(process.a
}

export function updateContextWithCliFlags(context: CodegenContext, cliFlags: YamlCliFlags) {
const config: Partial<Types.Config & { configFilePath?: string }> = {
const config: Partial<Types.Config & { configFilePath?: string; experimentalCache?: boolean }> = {
configFilePath: context.filepath,
};

if (cliFlags.watch) {
config.watch = cliFlags.watch;
}

if (cliFlags.experimentalCache) {
config.experimentalCache = true;
}

if (cliFlags.overwrite === true) {
config.overwrite = cliFlags.overwrite;
}
@@ -217,12 +228,97 @@ export function updateContextWithCliFlags(context: CodegenContext, cliFlags: Yam
context.updateConfig(config);
}

function isPromiseLike<T>(value: Promise<T> | T): value is Promise<T> {
return typeof (value as any).then === 'function';
}

class Cache {
isEnabled: () => boolean = () => false;
private invalidated: string[] = [];
private sources = new Map<string, Source>();
private dependencies = new Map<string, string[]>();

cacheSource(fn: (pointer: string, options: any) => Source, pointer: string, options: any): Source;
cacheSource(fn: (pointer: string, options: any) => Promise<Source>, pointer: string, options: any): Promise<Source>;
cacheSource(fn: (pointer: string, options: any) => Promise<Source> | Source, pointer: string, options: any) {
if (!this.isEnabled()) {
return fn(pointer, options);
}

const absolutePointer = ensureAbsolutePath(pointer, options);

if (this.sources.has(absolutePointer)) {
return this.sources.get(absolutePointer)!;
}

const result = fn(pointer, options);

if (isPromiseLike(result)) {
return result.then(source => {
this.sources.set(absolutePointer, source);
return source;
});
}

this.sources.set(absolutePointer, result);

return result;
}

invalidate(filepaths: string[]) {
if (!this.isEnabled()) {
return;
}

this.invalidated = filepaths.map(filepath => {
const absoluteFilepath = ensureAbsolutePath(filepath, {});
this.sources.delete(absoluteFilepath);

return absoluteFilepath;
});
}

shouldGenerate(filepath: string): boolean {
if (!this.isEnabled()) {
return true;
}

// TODO: if new file - generate everything
if (this.dependencies.has(filepath)) {
if (this.invalidated.length) {
const regenerate = this.dependencies.get(filepath).some(f => this.invalidated.includes(f));

if (regenerate) {
return true;
}
}

return false;
}

return true;
}

setDependencies(filepath: string, deps: string[]) {
if (!this.isEnabled()) {
return;
}

this.dependencies.set(
filepath,
deps.map(dep => ensureAbsolutePath(dep, {}))
);
}
}

export class CodegenContext {
private _config: Types.Config;
private _graphqlConfig?: GraphQLConfig;
private config: Types.Config;
private _project?: string;
private _pluginContext: { [key: string]: any } = {};
private cache = new Cache();

cwd: string;
filepath: string;

@@ -239,6 +335,7 @@ export class CodegenContext {
this._graphqlConfig = graphqlConfig;
this.filepath = this._graphqlConfig ? this._graphqlConfig.filepath : filepath;
this.cwd = this._graphqlConfig ? this._graphqlConfig.dirpath : process.cwd();
this.cache.isEnabled = () => (this.config as any).experimentalCache === true;
}

useProject(name?: string) {
@@ -264,6 +361,9 @@ export class CodegenContext {
return {
...extraConfig,
...this.config,
includeSources: true,
cacheable: this.cacheable.bind(this),
cacheableSync: this.cacheableSync.bind(this),
};
}

@@ -298,8 +398,37 @@ export class CodegenContext {

return loadDocuments(pointer, config);
}

invalidate(filepaths: string[]): void {
this.cache.invalidate(filepaths);
}

shouldGenerate(filepath: string) {
return this.cache.shouldGenerate(filepath);
}

setDependencies(filepath: string, deps: string[]) {
return this.cache.setDependencies(filepath, deps);
}

private cacheable(fn: (pointer: string, options: any) => Promise<Source>, pointer: string, options: any) {
return this.cache.cacheSource(fn, pointer, options);
}

private cacheableSync(fn: (pointer: string, options: any) => Source, pointer: string, options: any) {
return this.cache.cacheSource(fn, pointer, options);
}
}

export function ensureContext(input: CodegenContext | Types.Config): CodegenContext {
return input instanceof CodegenContext ? input : new CodegenContext({ config: input });
}

function ensureAbsolutePath(
pointer: string,
options: {
cwd?: string;
}
): string {
return isAbsolute(pointer) ? pointer : resolve(options.cwd || cwd(), pointer);
}
23 changes: 11 additions & 12 deletions packages/graphql-codegen-cli/src/generate-and-save.ts
Original file line number Diff line number Diff line change
@@ -2,8 +2,7 @@ import { lifecycleHooks } from './hooks';
import { Types } from '@graphql-codegen/plugin-helpers';
import { executeCodegen } from './codegen';
import { createWatcher } from './utils/watcher';
import { fileExists, readSync, writeSync, unlinkFile } from './utils/file-system';
import { sync as mkdirpSync } from 'mkdirp';
import { fileExists, readFile, writeFile, unlinkFile, mkdirp } from './utils/file-system';
import { dirname, join, isAbsolute } from 'path';
import { debugLog } from './utils/debugging';
import { CodegenContext, ensureContext } from './config';
@@ -42,18 +41,20 @@ export async function generate(
const recentOutputHash = new Map<string, string>();
async function writeOutput(generationResult: Types.FileOutput[]) {
if (!saveToFile) {
return generationResult;
return;
}

if (config.watch) {
removeStaleFiles(config, generationResult);
}

await lifecycleHooks(config.hooks).beforeAllFileWrite(generationResult.map(r => r.filename));
const resultsToEmit = generationResult.filter(f => !f.noEmit);

await lifecycleHooks(config.hooks).beforeAllFileWrite(resultsToEmit.map(r => r.filename));

await Promise.all(
generationResult.map(async (result: Types.FileOutput) => {
const exists = fileExists(result.filename);
resultsToEmit.map(async (result: Types.FileOutput) => {
const exists = await fileExists(result.filename);

if (!shouldOverwrite(config, result.filename) && exists) {
return;
@@ -64,7 +65,7 @@ export async function generate(
let previousHash = recentOutputHash.get(result.filename);

if (!previousHash && exists) {
previousHash = hash(readSync(result.filename));
previousHash = hash(await readFile(result.filename));
}

if (previousHash && currentHash === previousHash) {
@@ -81,19 +82,17 @@ export async function generate(
const basedir = dirname(result.filename);
await lifecycleHooks(result.hooks).beforeOneFileWrite(result.filename);
await lifecycleHooks(config.hooks).beforeOneFileWrite(result.filename);
mkdirpSync(basedir);
await mkdirp(basedir);
const absolutePath = isAbsolute(result.filename)
? result.filename
: join(input.cwd || process.cwd(), result.filename);
writeSync(absolutePath, result.content);
await writeFile(absolutePath, result.content);
await lifecycleHooks(result.hooks).afterOneFileWrite(result.filename);
await lifecycleHooks(config.hooks).afterOneFileWrite(result.filename);
})
);

await lifecycleHooks(config.hooks).afterAllFileWrite(generationResult.map(r => r.filename));

return generationResult;
await lifecycleHooks(config.hooks).afterAllFileWrite(resultsToEmit.map(r => r.filename));
}

// watch mode
Loading