Speed up GDJS build (including at editor startup) in development

This reduces the number of copied files (let esbuild build directly in the proper output folder). Also avoid copying the source files after every change (avoiding ~3-4 seconds of copy). Overall, a change in GDJS/Extensions results in 1-3 seconds of build, instead of almost 10 sometimes due to the large number of file copies. Only show in developer changelog
4ian · Jul 12, 2021 · f3a49ad · f3a49ad
1 parent aac3379
commit f3a49ad
Showing 7 changed files with 101 additions and 89 deletions.
diff --git a/GDJS/.gitignore b/GDJS/.gitignore
@@ -1,2 +1 @@
 /node_modules
-/Runtime-dist
diff --git a/GDJS/scripts/build.js b/GDJS/scripts/build.js
@@ -4,16 +4,26 @@ const shell = require('shelljs');
 const {
   getAllInOutFilePaths,
   isUntransformedFile,
-  getOutRootPath,
 } = require('./lib/runtime-files-list');
+const args = require('minimist')(process.argv.slice(2), {
+  string: ['out'],
+});
 const fs = require('fs').promises;
 
 /** @param {string} outPath */
 const renameBuiltFile = (outPath) => {
   return outPath.replace(/\.ts$/, '.js');
 };
 
-shell.mkdir('-p', getOutRootPath());
+const bundledOutPath = args.out;
+if (!bundledOutPath) {
+  shell.echo(
+    `❌ --out (path where to build GDJS Runtime and Extensions) is required.`
+  );
+  shell.exit(1);
+}
+
+shell.mkdir('-p', bundledOutPath);
 
 (async () => {
   const esbuildService = await esbuild.startService();
@@ -22,7 +32,7 @@ shell.mkdir('-p', getOutRootPath());
   const {
     allGDJSInOutFilePaths,
     allExtensionsInOutFilePaths,
-  } = await getAllInOutFilePaths();
+  } = await getAllInOutFilePaths({ bundledOutPath });
 
   // Build (or copy) all the files
   let errored = false;

diff --git a/GDJS/scripts/lib/runtime-files-list.js b/GDJS/scripts/lib/runtime-files-list.js
@@ -7,7 +7,6 @@ const gdjsRootPath = path.join(__dirname, '..', '..');
 
 const extensionsRuntimePath = path.join(gdevelopRootPath, 'Extensions');
 const gdjsRuntimePath = path.join(gdjsRootPath, 'Runtime');
-const bundledOutPath = path.join(gdjsRootPath, 'Runtime-dist');
 
 // The extensions to be included in the bundled Runtime (will be built with esbuild or copied).
 const allowedExtensions = ['.js', '.ts', '.html', '.json', '.xml'];
@@ -87,7 +86,7 @@ const isJsExtensionDeclaration = (filePath) => {
 /** @typedef {{inPath: string; outPath: string;}} InOutPath */
 
 /** Returns a function to generate the file paths in the bundled runtime. */
-const getInOutPaths = (basePath) => (inPath) => {
+const getInOutPaths = (basePath, bundledOutPath) => (inPath) => {
   const relativePath = path.relative(basePath, inPath);
   return {
     inPath,
@@ -96,10 +95,6 @@ const getInOutPaths = (basePath) => (inPath) => {
 };
 
 module.exports = {
-  /**
-   * Returns the path where the bundled Runtime will be generated.
-   */
-  getOutRootPath: () => bundledOutPath,
   /**
    * Check if a file must be copied without being built with esbuild.
    * @param {string} path
@@ -123,9 +118,10 @@ module.exports = {
   },
   /**
    * Get the list of all files part of the runtime, and their destination file when the runtime is bundled.
+   * @param {{bundledOutPath: string}} options
    * @returns {Promise<{allGDJSInOutFilePaths: InOutPath[]; allExtensionsInOutFilePaths: InOutPath[];}>}
    */
-  getAllInOutFilePaths: async () => {
+  getAllInOutFilePaths: async (options) => {
     // List all the files of the runtime
     const allGDJSInFilePaths = await recursive(gdjsRuntimePath, [
       isNotAllowedExtension,
@@ -137,10 +133,10 @@ module.exports = {
 
     // Generate the output file paths
     const allGDJSInOutFilePaths = allGDJSInFilePaths.map(
-      getInOutPaths(gdjsRuntimePath)
+      getInOutPaths(gdjsRuntimePath, options.bundledOutPath)
     );
     const allExtensionsInOutFilePaths = allExtensionsInFilePaths.map(
-      getInOutPaths(gdevelopRootPath)
+      getInOutPaths(gdevelopRootPath, options.bundledOutPath)
     );
 
     return { allGDJSInOutFilePaths, allExtensionsInOutFilePaths };

diff --git a/newIDE/app/package.json b/newIDE/app/package.json
@@ -83,7 +83,7 @@
   },
   "scripts": {
     "postinstall": "cd ../../GDJS && npm install && cd ../newIDE/app && npm run import-resources && npm run make-version-metadata",
-    "import-resources": "npm run import-zipped-external-editors && npm run build-theme-resources && cd scripts && node import-libGD.js && node import-GDJS-Runtime.js --clean && node import-monaco-editor.js && node import-zipped-external-libs.js",
+    "import-resources": "npm run import-zipped-external-editors && npm run build-theme-resources && cd scripts && node import-libGD.js && node import-GDJS-Runtime.js && node import-monaco-editor.js && node import-zipped-external-libs.js",
     "make-version-metadata": "cd scripts && node make-version-metadata.js",
     "make-service-worker": "cd scripts && node make-service-worker.js",
     "start": "npm run import-resources && npm run make-version-metadata && react-scripts start",
@@ -100,7 +100,7 @@
     "analyze-source-map": "source-map-explorer build/static/js/*.js",
     "extract-all-translations": "node scripts/extract-all-translations.js",
     "compile-translations": "node scripts/compile-translations.js",
-    "reload-extensions": "node scripts/import-GDJS-Runtime.js",
+    "reload-extensions": "node scripts/import-GDJS-Runtime.js --skip-clean",
     "build-theme-resources": "node scripts/build-theme-resources.js",
     "create-new-theme": "node scripts/create-new-theme.js",
     "import-zipped-external-editors": "cd scripts && node import-zipped-editor.js piskel 5.0.0-beta82 b8e4d57b160ff93d3680168cd271af795412ea6c4c0da321aee2946345c7fb75 && node import-zipped-editor.js jfxr 5.0.0-beta55 8ac12b557c2ddba958c6f0d3e0c5df8cf3369a65262dcb90cf5c8a7a7d20bdf6 && node import-zipped-editor.js yarn 5.0.0-beta103 155f6d074dbb025b082ede0f9b6acd55ed293457441f4c55f084c2d27fbda61d"

diff --git a/newIDE/app/scripts/import-GDJS-Runtime.js b/newIDE/app/scripts/import-GDJS-Runtime.js
@@ -1,3 +1,8 @@
+/*
+ * This builds GDJS game engine ("Runtime") and the extensions so that it can
+ * be used by the editor (either by the "electron-app" or the "web-app").
+ */
+
 const shell = require('shelljs');
 const path = require('path');
 const copy = require('recursive-copy');
@@ -9,72 +14,69 @@ const destinationPaths = [
   path.join(__dirname, '..', 'node_modules', 'GDJS-for-web-app-only'),
 ];
 
-// Build GDJS
-const output = shell.exec(`node scripts/build.js`, {
-  cwd: path.join(gdevelopRootPath, 'GDJS'),
-});
-if (output.code !== 0) {
-  shell.exit(0);
+// Clean the paths where GDJS Runtime (and extensions) will be copied/bundled.
+if (!args['skip-clean']) {
+  destinationPaths.forEach(destinationPath => {
+    shell.echo(`ℹ️ Cleaning destination first...`);
+    shell.rm('-rf', destinationPath);
+    shell.mkdir('-p', destinationPath);
+  });
 }
 
-// Then copy the Runtime to the IDE
-const copyOptions = {
-  overwrite: true,
-  expand: true,
-  dot: true,
-  junk: true,
-};
-
-shell.echo(
-  `ℹ️ Copying GDJS and extensions runtime built files and sources to ${destinationPaths.join(
-    ', '
-  )}...`
-);
+// Build GDJS runtime (and extensions).
 destinationPaths.forEach(destinationPath => {
-  if (args.clean) {
-    shell.echo(`ℹ️ Cleaning destination first...`);
-    shell.rm('-rf', destinationPath);
+  const outPath = path.join(destinationPath, 'Runtime');
+  const output = shell.exec(`node scripts/build.js --out ${outPath}`, {
+    cwd: path.join(gdevelopRootPath, 'GDJS'),
+  });
+  if (output.code !== 0) {
+    shell.exit(0);
   }
-  shell.mkdir('-p', destinationPath);
-
-  const startTime = Date.now();
-  // TODO: Investigate the use of a smart & faster sync
-  // that only copy files with changed content.
-  return Promise.all([
-    // Copy the built files
-    copy(
-      path.join(gdevelopRootPath, 'GDJS', 'Runtime-dist'),
-      path.join(destinationPath, 'Runtime'),
-      copyOptions
-    ),
-    // Copy the GDJS runtime and extension sources (useful for autocompletions
-    // in the IDE).
-    copy(
-      path.join(gdevelopRootPath, 'GDJS', 'Runtime'),
-      path.join(destinationPath, 'Runtime-sources'),
-      copyOptions
-    ),
-    copy(
-      path.join(gdevelopRootPath, 'Extensions'),
-      path.join(destinationPath, 'Runtime-sources', 'Extensions'),
-      { ...copyOptions, filter: ['**/*.js', '**/*.ts'] }
-    ),
-  ])
-    .then(function([
-      bundledResults,
-      unbundledResults,
-      unbundledExtensionsResults,
-    ]) {
-      const totalFilesCount =
-        bundledResults.length +
-        unbundledResults.length +
-        unbundledExtensionsResults.length;
-      const duration = Date.now() - startTime;
-      console.info(
-        `✅ Runtime files copy done (${totalFilesCount} file(s) copied in ${duration}ms).`
-      );
-    })
-    .catch(function(error) {
-      console.error('❌ Copy failed: ', error);
-    });
 });
+
+// Copy the GDJS runtime and extension sources (used for autocompletions
+// in the IDE). This is optional as this takes a lot of time that would add
+// up whenever any change is made.
+if (!args['skip-sources']) {
+  shell.echo(
+    `ℹ️ Copying GDJS and extensions runtime sources to ${destinationPaths.join(
+      ', '
+    )}...`
+  );
+  destinationPaths.forEach(destinationPath => {
+    const copyOptions = {
+      overwrite: true,
+      expand: true,
+      dot: true,
+      junk: true,
+    };
+
+    const startTime = Date.now();
+
+    // TODO: Investigate the use of a smart & faster sync
+    // that only copy files with changed content.
+    return Promise.all([
+      copy(
+        path.join(gdevelopRootPath, 'GDJS', 'Runtime'),
+        path.join(destinationPath, 'Runtime-sources'),
+        copyOptions
+      ),
+      copy(
+        path.join(gdevelopRootPath, 'Extensions'),
+        path.join(destinationPath, 'Runtime-sources', 'Extensions'),
+        { ...copyOptions, filter: ['**/*.js', '**/*.ts'] }
+      ),
+    ])
+      .then(function([unbundledResults, unbundledExtensionsResults]) {
+        const totalFilesCount =
+          unbundledResults.length + unbundledExtensionsResults.length;
+        const duration = Date.now() - startTime;
+        console.info(
+          `✅ Runtime source files copy done (${totalFilesCount} file(s) copied in ${duration}ms).`
+        );
+      })
+      .catch(function(error) {
+        console.error('❌ Copy failed: ', error);
+      });
+  });
+}
diff --git a/newIDE/app/src/CodeEditor/LocalCodeEditorAutocompletions.js b/newIDE/app/src/CodeEditor/LocalCodeEditorAutocompletions.js
@@ -88,6 +88,7 @@ export const setupAutocompletions = (monaco: any) => {
           folderName =>
             !folderName.endsWith('.txt') &&
             !folderName.endsWith('.md') &&
+            !folderName.endsWith('.flow.js') &&
             !folderName.endsWith('.gitignore')
         )
         .forEach(folderName =>

diff --git a/newIDE/app/src/GameEngineFinder/LocalGDJSDevelopmentWatcher.js b/newIDE/app/src/GameEngineFinder/LocalGDJSDevelopmentWatcher.js
@@ -13,7 +13,7 @@ const chokidar = optionalRequire('chokidar');
  * Returns the folder corresponding to newIDE/app in **development**. Works
  * only when running in Electron.
  */
-const findDevelopmentNewIdeAppPath = () /*: string */ => {
+const findDevelopmentNewIdeAppPath = (): string => {
   if (!electron) return '';
 
   const developmentElectronAppFolder = process.cwd();
@@ -22,6 +22,8 @@ const findDevelopmentNewIdeAppPath = () /*: string */ => {
 
 /**
  * Launch the newIDE script `import-GDJS-Runtime`.
+ * Cleaning the GDJS output folder and copying sources are both
+ * skipped to speed up the build.
  */
 const importGDJSRuntime = (): Promise<void> => {
   if (!child_process || !path) return Promise.reject(new Error('Unsupported'));
@@ -32,7 +34,7 @@ const importGDJSRuntime = (): Promise<void> => {
       `node "${path.join(
         findDevelopmentNewIdeAppPath(),
         'scripts/import-GDJS-Runtime.js'
-      )}"`,
+      )}" --skip-clean --skip-sources`,
       (error, stdout, stderr) => {
         if (error) {
           console.error(`GDJS Runtime update error:\n${error}`);
@@ -94,20 +96,22 @@ export const LocalGDJSDevelopmentWatcher = () => {
         return;
       }
 
-      const gdjsSourcesRuntimePath = path.join(
-        findDevelopmentNewIdeAppPath(),
-        '../../GDJS/Runtime/**/*'
-      );
-      const extensionsSourcesPath = path.join(
-        findDevelopmentNewIdeAppPath(),
-        '../../Extensions/**/*'
+      const relativeWatchPaths = [
+        // Watch all files in GDJS Runtime:
+        '../../GDJS/Runtime/**/*',
+        // Watch only JS/TS source files in extensions:
+        '../../Extensions/**/*.ts',
+        '../../Extensions/**/*.js',
+      ];
+      const watchPaths = relativeWatchPaths.map(watchPath =>
+        path.join(findDevelopmentNewIdeAppPath(), watchPath)
       );
 
       // Reload extensions when the component is first mounted
       importGDJSRuntime().catch(() => {});
 
       const watcher = chokidar
-        .watch([gdjsSourcesRuntimePath, extensionsSourcesPath], {
+        .watch(watchPaths, {
           awaitWriteFinish: true,
           ignoreInitial: true,
         })