From a46e8ecf401ed4f745cc6b629b75227d895c8f12 Mon Sep 17 00:00:00 2001 From: Andrew Sprouse Date: Fri, 14 Jun 2019 01:21:23 -0400 Subject: [PATCH 1/4] Serialized compile to address #299 --- lib/Configuration.js | 7 ++- lib/Configuration.test.js | 12 ++++-- lib/compile.js | 79 ++++++++++++++++++++-------------- lib/validate.js | 3 +- tests/compile.test.js | 90 +++++++++++++++++++++++++-------------- tests/webpack.mock.js | 12 ++++-- 6 files changed, 129 insertions(+), 74 deletions(-) diff --git a/lib/Configuration.js b/lib/Configuration.js index a3fde813e..2022ca913 100644 --- a/lib/Configuration.js +++ b/lib/Configuration.js @@ -14,7 +14,8 @@ const DefaultConfig = { packager: 'npm', packagerOptions: {}, keepOutputDirectory: false, - config: null + config: null, + serializedCompile: false }; class Configuration { @@ -73,6 +74,10 @@ class Configuration { return this._config.keepOutputDirectory; } + get serializedCompile() { + return this._config.serializedCompile; + } + toJSON() { return _.omitBy(this._config, _.isNil); } diff --git a/lib/Configuration.test.js b/lib/Configuration.test.js index 60206602b..e0e4aea40 100644 --- a/lib/Configuration.test.js +++ b/lib/Configuration.test.js @@ -19,7 +19,8 @@ describe('Configuration', () => { packager: 'npm', packagerOptions: {}, keepOutputDirectory: false, - config: null + config: null, + serializedCompile: false }; }); @@ -78,7 +79,8 @@ describe('Configuration', () => { packager: 'npm', packagerOptions: {}, keepOutputDirectory: false, - config: null + config: null, + serializedCompile: false }); }); }); @@ -98,7 +100,8 @@ describe('Configuration', () => { packager: 'npm', packagerOptions: {}, keepOutputDirectory: false, - config: null + config: null, + serializedCompile: false }); }); @@ -117,7 +120,8 @@ describe('Configuration', () => { packager: 'npm', packagerOptions: {}, keepOutputDirectory: false, - config: null + config: null, + serializedCompile: false }); }); }); diff --git a/lib/compile.js b/lib/compile.js index e5c2363cf..683242f7a 100644 --- a/lib/compile.js +++ b/lib/compile.js @@ -5,44 +5,57 @@ const BbPromise = require('bluebird'); const webpack = require('webpack'); const tty = require('tty'); -module.exports = { - compile() { - this.serverless.cli.log('Bundling with Webpack...'); +const defaultStatsConfig = { + colors: tty.isatty(process.stdout.fd), + hash: false, + version: false, + chunks: false, + children: false +}; - const compiler = webpack(this.webpackConfig); - - return BbPromise.fromCallback(cb => compiler.run(cb)).then(stats => { - if (!this.multiCompile) { - stats = { stats: [stats] }; - } - - const compileOutputPaths = []; - const consoleStats = this.webpackConfig.stats || - _.get(this, 'webpackConfig[0].stats') || { - colors: tty.isatty(process.stdout.fd), - hash: false, - version: false, - chunks: false, - children: false - }; - - _.forEach(stats.stats, compileStats => { - const statsOutput = compileStats.toString(consoleStats); - if (statsOutput) { - this.serverless.cli.consoleLog(statsOutput); +function ensureArray(obj) { + return _.isArray(obj) ? obj : [obj]; +} + +function getStatsLogger(statsConfig, consoleLog) { + return stats => consoleLog(stats.toString(statsConfig || defaultStatsConfig)); +} + +function webpackCompile(config, logStats) { + return BbPromise + .fromCallback(cb => webpack(config).run(cb)) + .then(stats => { + // ensure stats in any array in the case of multiCompile + stats = stats.stats ? stats.stats : [stats]; + + _.forEach(stats, compileStats => { + logStats(compileStats); + if (compileStats.hasErrors()) { + throw new Error('Webpack compilation error, see stats above'); } + }); - if (compileStats.compilation.errors.length) { - throw new Error('Webpack compilation error, see above'); - } + return stats; + }); +} - compileOutputPaths.push(compileStats.compilation.compiler.outputPath); - }); +function webpackCompileSerial(configs, logStats) { + return BbPromise + .mapSeries(configs, config => webpackCompile(config, logStats)) + .then(stats => _.flatten(stats)); +} - this.compileOutputPaths = compileOutputPaths; - this.compileStats = stats; +module.exports = { + compile() { + this.serverless.cli.log('Bundling with Webpack...'); - return BbPromise.resolve(); - }); + const configs = ensureArray(this.webpackConfig); + const logStats = getStatsLogger(configs[0].stats, this.serverless.cli.consoleLog); + + return (this.serializedCompile ? webpackCompileSerial : webpackCompile)(configs, logStats) + .then(stats => { + this.compileStats = { stats }; + return BbPromise.resolve(); + }); } }; diff --git a/lib/validate.js b/lib/validate.js index b26bce3d7..8adc9d9f9 100644 --- a/lib/validate.js +++ b/lib/validate.js @@ -167,8 +167,9 @@ module.exports = { // In case of individual packaging we have to create a separate config for each function if (_.has(this.serverless, 'service.package') && this.serverless.service.package.individually) { - this.options.verbose && this.serverless.cli.log('Using multi-compile (individual packaging)'); this.multiCompile = true; + this.serializedCompile = this.configuration.serializedCompile; + this.options.verbose && this.serverless.cli.log(`Using ${this.serializedCompile ? 'serialized' : 'multi'}-compile (individual packaging)`); if (this.webpackConfig.entry && !_.isEqual(this.webpackConfig.entry, entries)) { return BbPromise.reject( diff --git a/tests/compile.test.js b/tests/compile.test.js index c0bcb672a..58f682029 100644 --- a/tests/compile.test.js +++ b/tests/compile.test.js @@ -44,13 +44,10 @@ describe('compile', () => { consoleLog: sandbox.stub() }; - module = _.assign( - { - serverless, - options: {} - }, - baseModule - ); + module = _.assign({ + serverless, + options: {}, + }, baseModule); }); afterEach(() => { @@ -65,8 +62,9 @@ describe('compile', () => { it('should compile with webpack from a context configuration', () => { const testWebpackConfig = 'testconfig'; module.webpackConfig = testWebpackConfig; - return expect(module.compile()).to.be.fulfilled.then(() => { - expect(webpackMock).to.have.been.calledWith(testWebpackConfig); + return expect(module.compile()).to.be.fulfilled + .then(() => { + expect(webpackMock).to.have.been.calledWith([testWebpackConfig]); expect(webpackMock.compilerMock.run).to.have.been.calledOnce; return null; }); @@ -80,29 +78,58 @@ describe('compile', () => { }); it('should work with multi compile', () => { - const testWebpackConfig = 'testconfig'; - const multiStats = [ - { + const testWebpackConfig = ['testconfig']; + const multiStats = { + stats: [{ compilation: { errors: [], compiler: { - outputPath: 'statsMock-outputPath' - } + outputPath: 'statsMock-outputPath', + }, }, - toString: sandbox.stub().returns('testStats') - } - ]; + toString: sandbox.stub().returns('testStats'), + hasErrors: _.constant(false) + }] + }; module.webpackConfig = testWebpackConfig; module.multiCompile = true; webpackMock.compilerMock.run.reset(); webpackMock.compilerMock.run.yields(null, multiStats); - return expect(module.compile()).to.be.fulfilled.then(() => { + return expect(module.compile()).to.be.fulfilled + .then(() => { expect(webpackMock).to.have.been.calledWith(testWebpackConfig); expect(webpackMock.compilerMock.run).to.have.been.calledOnce; return null; }); }); + it('should work with serialized compile', () => { + const testWebpackConfig = ['testconfig']; + const multiStats = { + stats: [{ + compilation: { + errors: [], + compiler: { + outputPath: 'statsMock-outputPath', + }, + }, + toString: sandbox.stub().returns('testStats'), + hasErrors: _.constant(false) + }] + }; + module.webpackConfig = testWebpackConfig; + module.multiCompile = true; + module.serializedCompile = true; + webpackMock.compilerMock.run.reset(); + webpackMock.compilerMock.run.yields(null, multiStats); + return expect(module.compile()).to.be.fulfilled + .then(() => { + expect(webpackMock).to.have.been.calledWith(testWebpackConfig); + expect(webpackMock.compilerMock.run).to.have.been.calledOnce; + return null; + }); + }); + it('should use correct stats option', () => { const testWebpackConfig = { stats: 'minimal' @@ -114,23 +141,24 @@ describe('compile', () => { outputPath: 'statsMock-outputPath' } }, - toString: sandbox.stub().returns('testStats') + toString: sandbox.stub().returns('testStats'), + hasErrors: _.constant(false) }; module.webpackConfig = testWebpackConfig; webpackMock.compilerMock.run.reset(); webpackMock.compilerMock.run.yields(null, mockStats); - return expect(module.compile()) - .to.be.fulfilled.then(() => { - expect(webpackMock).to.have.been.calledWith(testWebpackConfig); - expect(mockStats.toString.firstCall.args).to.eql([testWebpackConfig.stats]); - module.webpackConfig = [testWebpackConfig]; - return expect(module.compile()).to.be.fulfilled; - }) - .then(() => { - expect(webpackMock).to.have.been.calledWith([testWebpackConfig]); - expect(mockStats.toString.args).to.eql([ [testWebpackConfig.stats], [testWebpackConfig.stats] ]); - return null; - }); + return (expect(module.compile()).to.be.fulfilled) + .then(() => { + expect(webpackMock).to.have.been.calledWith([testWebpackConfig]); + expect(mockStats.toString.firstCall.args).to.eql([testWebpackConfig.stats]); + module.webpackConfig = [testWebpackConfig]; + return (expect(module.compile()).to.be.fulfilled); + }) + .then(() => { + expect(webpackMock).to.have.been.calledWith([testWebpackConfig]); + expect(mockStats.toString.args).to.eql([ [testWebpackConfig.stats], [testWebpackConfig.stats] ]); + return null; + }); }); }); diff --git a/tests/webpack.mock.js b/tests/webpack.mock.js index abf911e6b..5bf7194c2 100644 --- a/tests/webpack.mock.js +++ b/tests/webpack.mock.js @@ -6,12 +6,16 @@ const StatsMock = () => ({ compilation: { errors: [], compiler: { - outputPath: 'statsMock-outputPath' - } + outputPath: 'statsMock-outputPath', + }, + }, + toString: sinon.stub().returns('testStats'), + hasErrors() { + return Boolean(this.compilation.errors.length); }, - toString: sinon.stub().returns('testStats') }); + const CompilerMock = (sandbox, statsMock) => ({ run: sandbox.stub().yields(null, statsMock), watch: sandbox.stub().yields(null, statsMock), @@ -19,7 +23,7 @@ const CompilerMock = (sandbox, statsMock) => ({ beforeCompile: { tapPromise: sandbox.stub() } - } + }, }); const webpackMock = sandbox => { From 0cf085d7611dd4899072992eedcc0cc63b6d8e73 Mon Sep 17 00:00:00 2001 From: Andrew Sprouse Date: Fri, 14 Jun 2019 01:41:17 -0400 Subject: [PATCH 2/4] Add serializedCompile documentation --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index 155b14356..882185bed 100644 --- a/README.md +++ b/README.md @@ -512,6 +512,15 @@ if you are trying to override the entry in webpack.config.js with other unsuppor The individual packaging needs more time at the packaging phase, but you'll get that paid back twice at runtime. +#### Individual packaging serializedCompile +```yaml +# serverless.yml +custom: + webpack: + serializedCompile: true +``` +Will run each webpack build one at a time which helps reduce memory usage and in some cases impoves overall build performance. + ## Usage ### Automatic bundling From e47ff975ef3c963b91522776e1b371eb53bc1e39 Mon Sep 17 00:00:00 2001 From: "Miguel A. Calles MBA" <44813512+miguel-a-calles-mba@users.noreply.github.com> Date: Sun, 12 Jul 2020 15:56:04 -0700 Subject: [PATCH 3/4] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 882185bed..9119f484f 100644 --- a/README.md +++ b/README.md @@ -1046,3 +1046,4 @@ me to take it over and continue working on the project. That helped to revive it [link-505]: https://github.com/serverless-heaven/serverless-webpack/issues/505 [link-499]: https://github.com/serverless-heaven/serverless-webpack/issues/499 [link-496]: https://github.com/serverless-heaven/serverless-webpack/pull/496 + From 91da572199ca9bcf959b8d9a76c0e223c762da7a Mon Sep 17 00:00:00 2001 From: "Miguel A. Calles MBA" <44813512+miguel-a-calles-mba@users.noreply.github.com> Date: Sun, 12 Jul 2020 15:56:17 -0700 Subject: [PATCH 4/4] Update README.md --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 9119f484f..882185bed 100644 --- a/README.md +++ b/README.md @@ -1046,4 +1046,3 @@ me to take it over and continue working on the project. That helped to revive it [link-505]: https://github.com/serverless-heaven/serverless-webpack/issues/505 [link-499]: https://github.com/serverless-heaven/serverless-webpack/issues/499 [link-496]: https://github.com/serverless-heaven/serverless-webpack/pull/496 -