chore: start using package-lock.json for repeatable Lambda Layer buil…

…ds (#2627)

Use node v16 for npm v8 for lockfileVersion:2. Move authoritative lambda
layer zip building to "dev-utils/make-lambda-layer-zip.sh" and get it
to use the package-lock file. 

Add a .nvmrc file as a convenience and a hint at the suggested base
node version to use for local dev.

Closes: #2626

.ci/Jenkinsfile

@@ -17,7 +17,7 @@ pipeline {
     NOTIFY_TO = '[email protected]'
     NPMRC_SECRET = 'secret/jenkins-ci/npmjs/elasticmachine'
     TOTP_SECRET = 'totp/code/npmjs-elasticmachine'
-    NODE_VERSION = 'v14.17.5'
+    BUILD_NODE_VERSION = 'v16.15.0'
   }
   options {
     timeout(time: 3, unit: 'HOURS')
@@ -291,7 +291,7 @@ pipeline {
               setEnvVar('RELEASE_NOTES_URL', getReleaseNotesUrl())
               deleteDir()
               unstash 'source'
-              withNodeJSEnv(version: "${env.NODE_VERSION}"){
+              withNodeJSEnv(version: "${env.BUILD_NODE_VERSION}"){
                 dir("${BASE_DIR}"){
                   cmd(label: 'make dist', script: 'make -C .ci dist')
                 }
@@ -318,18 +318,18 @@ pipeline {
             }
           }
         }
-        stage('Release Notes') {
+        stage('GitHub Release') {
           steps {
             withGhEnv(forceInstallation: true, version: '2.4.0') {
               dir("${BASE_DIR}"){
-                cmd(label: 'make release-notes', script: 'make -C .ci release-notes')
+                cmd(label: 'make github-release', script: 'make -C .ci github-release')
               }
             }
           }
         }
         stage('Publish to npm') {
           steps {
-            withNodeJSEnv(version: "${env.NODE_VERSION}") {
+            withNodeJSEnv(version: "${env.BUILD_NODE_VERSION}") {
               withNpmrc(secret: "${env.NPMRC_SECRET}", path: "${env.WORKSPACE}/${env.BASE_DIR}") {
                 withTotpVault(secret: "${env.TOTP_SECRET}", code_var_name: 'TOTP_CODE') {
                   dir("${BASE_DIR}") {
@@ -383,7 +383,7 @@ pipeline {
       environment {
         HOME = "${env.WORKSPACE}"
         RESULT_FILE = 'apm-agent-benchmark-results.json'
-        NODE_VERSION = '14'
+        BENCH_NODE_VERSION = '14'
       }
       when {
         beforeAgent true
@@ -402,7 +402,7 @@ pipeline {
             deleteDir()
             unstash 'source'
             dir(BASE_DIR){
-              sh '.ci/scripts/run-benchmarks.sh "${RESULT_FILE}" "${NODE_VERSION}"'
+              sh '.ci/scripts/run-benchmarks.sh "${RESULT_FILE}" "${BENCH_NODE_VERSION}"'
             }
           }
         }

.ci/Makefile

@@ -1,23 +1,21 @@
 SHELL = /bin/bash -eo pipefail
 
 AWS_FOLDER = ../build/aws
+LAMBDA_LAYER_ZIP_FILE = ../build/lambda-layer-zip/elastic-apm-node-lambda-layer.zip
 
 export AWS_FOLDER
 
-build:
-	rm -rf $(AWS_FOLDER) || true
-	mkdir -p $(AWS_FOLDER)
-	cd $(AWS_FOLDER); \
-		npm init -y; \
-		npm install --global-style https://github.com/elastic/apm-agent-nodejs#$(BRANCH_NAME); \
-		mkdir nodejs; \
-		mv node_modules nodejs
+.PHONY: dist
+dist: validate-branch-name
+	../dev-utils/make-lambda-layer-zip.sh
+	cp $(LAMBDA_LAYER_ZIP_FILE) $(AWS_FOLDER)/elastic-apm-node-lambda-layer-$(BRANCH_NAME).zip
+
+.PHONY: clean
+clean:
+	rm -rf $(AWS_FOLDER)
+
 env:
 	env
-dist: validate-branch-name build
-	rm -f $(BRANCH_NAME).zip || true
-	cd $(AWS_FOLDER) ; \
-		zip -r $(BRANCH_NAME).zip nodejs
 
 # List all the AWS regions
 get-all-aws-regions:
@@ -48,7 +46,7 @@ publish: validate-layer-name validate-aws-default-region
 		--description "AWS Lambda Extension Layer for the Elastic APM Node.js Agent" \
 		--license "Apache-2.0" \
 		--compatible-runtimes nodejs14.x nodejs12.x nodejs10.x \
-		--zip-file "fileb://./$(AWS_FOLDER)/$(BRANCH_NAME).zip"
+		--zip-file "fileb://./$(AWS_FOLDER)/elastic-apm-node-lambda-layer-$(BRANCH_NAME).zip"
 
 # Grant public access to the given LAYER in the given AWS region
 grant-public-layer-access: validate-layer-name validate-aws-default-region
@@ -72,14 +70,13 @@ get-version: validate-aws-default-region
 create-arn-file: validate-suffix-arn-file validate-release-notes-url
 	@./scripts/create-arn-table.sh
 
-release-notes: validate-branch-name validate-suffix-arn-file
+github-release: validate-branch-name validate-suffix-arn-file
 	@gh release list
 	@gh \
 		release \
 		create $(BRANCH_NAME) \
 		--title '$(BRANCH_NAME)' \
-		--notes-file $(AWS_FOLDER)/$(SUFFIX_ARN_FILE) \
-		$(AWS_FOLDER)/$(BRANCH_NAME).zip
+		--notes-file $(AWS_FOLDER)/$(SUFFIX_ARN_FILE)
 
 validate-branch-name:
 ifndef BRANCH_NAME

.gitignore

@@ -1,7 +1,6 @@
 # Files to ignore
 .DS_Store
 /.vscode
-/package-lock.json
 /test/babel/out.js
 /test/types/transpile/index.js
 /test/types/transpile-default/index.js

.npmrc

@@ -1,5 +1,3 @@
-package-lock=false
-
 # Workaround unresolvable peerDependencies between express-graphql, graphql,
 # and apollo-server-express. npm v7 (included with node v15) makes these
 # peerDependencies issues an install error. Until the community catches up

.nvmrc

@@ -0,0 +1 @@
+16.15.0

CHANGELOG.asciidoc

@@ -19,6 +19,9 @@ Notes:
         [float]
         ===== Bug fixes
 
+        [float]
+        ===== Chores
+
 2. When making a release, change the "==== Unreleased" section header to:
 
         [[release-notes-x.x.x]]
@@ -51,6 +54,12 @@ Notes:
 [float]
 ===== Bug fixes
 
+[float]
+===== Chores
+
+- Add a package-lock.json file to ensure repeatable builds of the AWS Lambda
+  layer and to assist with security issue auditing. ({issue}2626[#2626])
+
 
 [[release-notes-3.33.0]]
 ==== 3.33.0 2022/05/05

CONTRIBUTING.md

@@ -175,6 +175,7 @@ If you have access to make releases, the process is as follows:
    changes.
 1. Make a PR titled "x.y.z" (the new version) which updates:
     - the version in `package.json`,
+    - the version in `package-lock.json`,
     - "CHANGELOG.asciidoc": Add missing changelog entries, if any. Then change
       the "Unreleased" section title to:
         ```

dev-utils/make-lambda-layer-zip.sh

@@ -1,10 +1,9 @@
 #!/bin/bash
 #
-# Make an APM Node.js agent lambda layer zip file with local repo changes
-# that could be published to AWS and used for a dev/test lambda.
+# Make a Node.js APM Agent lambda layer zip file that can be published to AWS.
+# https://docs.aws.amazon.com/lambda/latest/dg/configuration-layers.html
 #
-# Note: This is for development-only, the blessed path for building and
-# publishing Lambda layers for this agent is in ".ci/Makefile".
+# Note: This has the side-effect of modifying "./node_modules/...".
 #
 
 if [ "$TRACE" != "" ]; then
@@ -25,21 +24,33 @@ function fatal {
 
 TOP=$(cd $(dirname $0)/../ >/dev/null; pwd)
 BUILD_DIR="$TOP/build/lambda-layer-zip"
-TIMESTAMP=$(date -u '+%Y%m%dT%H%M%S')
+
+# Guard against accidentally using this script with a too-old npm.
+if [[ $(npm --version | cut -d. -f1) -lt 8 ]]; then
+    fatal "npm version is too old for 'npm ci --omit=dev': $(npm --version)"
+fi
 
 rm -rf "$BUILD_DIR"
 mkdir -p "$BUILD_DIR"
 cd "$BUILD_DIR"
 
-npm --loglevel=warn pack "$TOP" # creates "elastic-apm-node-$ver.tgz"
-npm init -y
-npm install --global-style elastic-apm-node-*.tgz
-
-mkdir nodejs
-mv node_modules nodejs
-zip -q -r elastic-apm-node-lambda-layer-dev-$TIMESTAMP.zip nodejs
-echo "Created build/lambda-layer-zip/elastic-apm-node-lambda-layer-dev-$TIMESTAMP.zip"
+mkdir -p nodejs/node_modules/elastic-apm-node
+(cd nodejs/node_modules/elastic-apm-node;
+    # Use 'npm pack' to get the published files as a start.
+    npm --loglevel=warn pack "$TOP"; # creates "elastic-apm-node-$ver.tgz"
+    tar --strip-components=1 -xf elastic-apm-node-*.tgz;
+    rm elastic-apm-node-*.tgz;
+    cp $TOP/package-lock.json ./;
+    # Then install the "package-lock.json"-dictated dependencies (excluding
+    # devDependencies).  Use '--ignore-scripts' to have confidence no code but
+    # ours and npm's is running.
+    npm ci --omit=dev --ignore-scripts;
+    rm package-lock.json)
+
+echo ""
+zip -q -r elastic-apm-node-lambda-layer.zip nodejs
+echo "Created build/lambda-layer-zip/elastic-apm-node-lambda-layer.zip"
 
 echo
-echo "Note: You can use the following command to publish this layer for dev work:"
-echo "  aws lambda --output json publish-layer-version --layer-name '$USER-play-elastic-apm-nodejs' --description '$USER dev Elastic APM Node.js agent lambda layer' --zip-file 'fileb://build/lambda-layer-zip/elastic-apm-node-lambda-layer-dev-$TIMESTAMP.zip'"
+echo "The lambda layer can be published as follows for dev work:"
+echo "    aws lambda --output json publish-layer-version --layer-name '$USER-dev-elastic-apm-node' --description '$USER dev Elastic APM Node.js agent lambda layer' --zip-file 'fileb://build/lambda-layer-zip/elastic-apm-node-lambda-layer.zip'"