From 4bc52826f1a481d6162791ba0b006eb9e71a1606 Mon Sep 17 00:00:00 2001 From: Kevin O'Donnell Date: Thu, 8 Jul 2021 01:17:43 -0400 Subject: [PATCH] ci: update stadard build patterns Signed-off-by: Kevin O'Donnell --- .gitignore | 2 + .pre-commit-config.yaml | 2 +- Jenkinsfile | 80 ++++---- Makefile | 36 ++-- standard_defs.mk | 404 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 468 insertions(+), 56 deletions(-) create mode 100644 standard_defs.mk diff --git a/.gitignore b/.gitignore index 1521c8b..43bab3b 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ dist +build +markers diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1d75010..47a3754 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -76,7 +76,7 @@ repos: rev: v1.1.10 hooks: - id: forbid-tabs - exclude: "^(Makefile|.*.go)$" + exclude: "^(Makefile|.*.mk|.*.go)$" - repo: git://github.com/doublify/pre-commit-go rev: 208a4aaa7f86b44e961eaaea526743b920e187a0 hooks: diff --git a/Jenkinsfile b/Jenkinsfile index f6e4d59..615b35b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -47,61 +47,63 @@ pipeline { stage('Build') { steps { - sh 'make clean all' + sh 'make clean build' } } - stage('Publish') { + stage('Package') { steps { - withCredentials([ - usernamePassword(credentialsId: 'btp-build-nexus', - usernameVariable:'USER', - passwordVariable:'PASSWORD')]) { - sh ''' - for f in $(find dist -name '*.tar.gz' ); do - job_base_name=$(dirname "$JOB_NAME") - base=$(basename $f) + sh "make package" + } + } - if [ "$BRANCH_NAME" = "main" ]; then - curl -u "$USER:$PASSWORD" --upload-file $f \ - ${PUBLISH_BASE_URL}/${job_base_name}/$base - else - curl -u "$USER:$PASSWORD" --upload-file $f \ - ${PUBLISH_BASE_URL}/${job_base_name}/$base - fi - done + stage("Analyze") { + steps { + withCredentials([string(credentialsId: 'fossa.full.token', variable: 'FOSSA_API_KEY')]) { + sh ''' + make analyze ''' } } } - stage('Create Archives') { + stage('Test') { steps { - sh ''' - REPO=$(git remote show -n origin | grep Fetch | awk -F'[/.]' '{print $6}') - VERSION=`git describe --dirty` - git archive HEAD --format=zip -9 --output=$REPO-$VERSION.zip - git archive HEAD --format=tgz -9 --output=$REPO-$VERSION.tgz - ''' - archiveArtifacts artifacts: '**/*.zip' - archiveArtifacts artifacts: '**/*.tgz' + sh "make test" } } - } - - post { - always { - sh 'make clean || true ' - } - success { - echo "Build success" + stage('Create Archive') { + steps { + sh "make archive" + archiveArtifacts 'build/*.tgz, build/*.zip' } - aborted { - error "Aborted, exiting now" + } + + stage("Publish") { + when { + expression { env.BRANCH_NAME == "main" } } - failure { - error "Failed, exiting now" + steps { + withCredentials([string(credentialsId: 'btp-build-github-pat', + variable: 'GITHUB_TOKEN')]) { + sh ''' + make clean publish + ''' + } } + } + } + + post { + success { + echo "Successfully completed" + } + aborted { + error "Aborted, exiting now" + } + failure { + error "Failed, exiting now" + } } } diff --git a/Makefile b/Makefile index ec2f173..c18a243 100644 --- a/Makefile +++ b/Makefile @@ -1,22 +1,35 @@ +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +include $(MAKEFILE_DIR)/standard_defs.mk -VERSION := $(shell git describe --dirty |cut -c2-) +all: package + +clean: + rm -rf dist + +package: package_scripts + +publish: package gh-create-draft-release + if [ "$(RELEASABLE)" = "yes" ];then \ + $(GH_RELEASE) upload $(VERSION) dist/*.tgz; \ + fi + +.PHONY: package_scripts +package_scripts: dist/doc-$(VERSION).tar.gz dist/bin-$(VERSION).tar.gz dist/lib-$(VERSION).tar.gz -.PHONY: all -all: dist dist/doc-$(VERSION).tar.gz: mkdir -p dist/doc/bash for inc in $$(find bash -name \*.sh); do \ - mdname=$$(echo $$inc | sed -e 's/\.sh/\.md/') ; \ - bash/bashadoc $$inc > dist/doc/$$mdname ; done + mdname=$$(echo $$inc | sed -e 's/\.sh/\.md/') ; \ + bash/bashadoc $$inc > dist/doc/$$mdname ; done tar -zcf dist/doc-$(VERSION).tar.gz -C dist doc rm -rf dist/doc dist/bin-$(VERSION).tar.gz: mkdir -p dist/bin for s in $$(find bash -type f -exec grep -q "includer" {} \; -print|grep -v ".sh$$"); do \ - base=$$(basename $$s) ; \ - bash/pack-script -v -f $$s -o dist/bin/$$base ; \ + base=$$(basename $$s) ; \ + bash/pack-script -v -f $$s -o dist/bin/$$base ; \ done tar -zcf dist/bin-$(VERSION).tar.gz -C dist bin rm -rf dist/bin @@ -26,12 +39,3 @@ dist/lib-$(VERSION).tar.gz: cp bash/*.sh dist/lib tar -zcf dist/lib-$(VERSION).tar.gz -C dist lib rm -rf dist/lib - -.PHONY: clean -clean: - rm -rf dist - -.PHONY: test - -.PHONY: dist -dist: dist/doc-$(VERSION).tar.gz dist/bin-$(VERSION).tar.gz dist/lib-$(VERSION).tar.gz diff --git a/standard_defs.mk b/standard_defs.mk new file mode 100644 index 0000000..15d1b62 --- /dev/null +++ b/standard_defs.mk @@ -0,0 +1,404 @@ +export ISOLATION_ID ?= local +PWD = $(shell pwd) + +export MAKE_BIN ?= $(PWD)/bin + +ORGANIZATION ?= $(shell git remote show -n origin | grep Fetch | \ + awk '{print $$NF}' | sed -e 's/git@github.com://' | \ + sed -e 's@https://github.com/@@' | awk -F'[/.]' '{print $$1}' ) +REPO ?= $(shell git remote show -n origin | grep Fetch | awk '{print $$NF}' | \ + sed -e 's/git@github.com://' | sed -e 's@https://github.com/@@' | \ + awk -F'[/.]' '{print $$2}' ) +BRANCH_NAME ?= $(shell git symbolic-ref -q HEAD |sed -e 's@refs/heads/@@') +SAFE_BRANCH_NAME ?= $(shell if [ -n "$$BRANCH_NAME" ]; then echo $$BRANCH_NAME;\ + else git symbolic-ref -q HEAD|sed -e 's@refs/heads/@@'|sed -e 's@/@_@g'; fi) +PR_KEY=$(shell echo $(BRANCH_NAME) | cut -c4-) +VERSION ?= $(shell git describe | cut -c2- ) +LONG_VERSION ?= $(shell git describe --long --dirty |cut -c2- ) +UID := $(shell id -u) +GID := $(shell id -g) + +RELEASABLE != if [ "$(LONG_VERSION)" = "$(VERSION)" ] || \ + (echo "$(LONG_VERSION)" | grep -q dirty); then \ + echo "no"; else echo "yes"; fi + +MARKERS = markers + +## +# SonarQube0 +## +SONAR_HOST_URL ?= https://sonarcloud.io +SONAR_AUTH_TOKEN ?= + +## +# Maven related settings +## +MAVEN_SETTINGS ?= $(shell if [ -r $(HOME)/.m2/settings.xml ]; then \ + echo $(HOME)/.m2/settings.xml; else echo ""; fi) + +MAVEN_REVISION != if [ "$(LONG_VERSION)" = "$(VERSION)" ] || \ + (echo "$(LONG_VERSION)" | grep -q dirty); then \ + bump_ver=$(VERSION); \ + if [ -x bin/semver ]; then \ + bump_ver=$$(bin/semver bump patch $(VERSION))-SNAPSHOT; \ + elif command -v semver >/dev/null; then \ + bump_ver=$$(command semver bump patch $(VERSION))-SNAPSHOT; \ + fi; \ + echo $$bump_ver ; \ + else \ + echo $(VERSION); \ + fi + +MAVEN_REPO_BASE ?= https://dev.catenasys.com/repository/catenasys-maven +MAVEN_REPO_TARGET != if [ "$(LONG_VERSION)" = "$(VERSION)" ] || \ + (echo "$(LONG_VERSION)" | grep -q dirty); then echo snapshots; \ + else \ + echo releases;\ + fi +MAVEN_UPDATE_RELEASE_INFO != if [ "$(LONG_VERSION)" = "$(VERSION)" ] || \ + (echo "$(LONG_VERSION)" | grep -q dirty); then \ + echo false; \ + else \ + echo true; \ + fi +MAVEN_DEPLOY_TARGET = \ + $(MAVEN_REPO_TARGET)::default::$(MAVEN_REPO_BASE)-$(MAVEN_REPO_TARGET) + +## +# Docker based toolchain +## +DOCKER_RUN = docker run --rm +DOCKER_RUN_USER = $(DOCKER_RUN) --user $(UID):$(GID) +DOCKER_RUN_ROOT = $(DOCKER_RUN) --user 0:0 +DOCKER_RUN_DEFAULT = $(DOCKER_RUN) +DOCKER_SOCK ?= /var/run/docker.sock + +#TOOLCHAIN_IMAGE = toolchain:$(ISOLATION_ID) +TOOLCHAIN_IMAGE := blockchaintp/toolchain:latest +TOOLCHAIN_HOME := /home/toolchain + +TOOL_VOLS = -v toolchain-home-$(ISOLATION_ID):/home/toolchain \ + -v $(PWD):/project + +TOOL = $(DOCKER_RUN_USER) -e GITHUB_TOKEN -e MAVEN_HOME=/home/toolchain/.m2 $(TOOL_VOLS) \ + -w $${WORKDIR:-/project} +TOOL_DEFAULT = $(DOCKER_RUN_DEFAULT) -e GITHUB_TOKEN \ + -e MAVEN_HOME=/home/toolchain/.m2 $(TOOL_VOLS) -w $${WORKDIR:-/project} + +TOOLCHAIN := $(TOOL) \ + $(shell if [ -n "$(MAVEN_SETTINGS)" ]; then echo -v \ + $(MAVEN_SETTINGS):$(TOOLCHAIN_HOME)/.m2/settings.xml; fi) $(TOOLCHAIN_IMAGE) +DOCKER_MVN := $(TOOLCHAIN) mvn -Drevision=$(MAVEN_REVISION) -B +BUSYBOX := $(DOCKER_RUN_USER) $(TOOL_VOLS) \ + $(shell if [ -n "$(MAVEN_SETTINGS)" ]; then echo -v \ + $(MAVEN_SETTINGS):$(TOOLCHAIN_HOME)/.m2/settings.xml; fi) busybox:latest +BUSYBOX_ROOT := $(DOCKER_RUN_ROOT) $(TOOL_VOLS) \ + $(shell if [ -n "$(MAVEN_SETTINGS)" ]; then echo -v \ + $(MAVEN_SETTINGS):$(TOOLCHAIN_HOME)/.m2/settings.xml; fi) busybox:latest + +DIVE_ANALYZE = $(TOOL) -v $(DOCKER_SOCK):/var/run/docker.sock \ + --user toolchain:$(shell getent group docker|awk -F: '{print $$3}') \ + $(TOOLCHAIN_IMAGE) dive --ci + +## +# BEGIN Standardized directives +## + +## +# Standard targets +## + +# A complete build sequence +.PHONY: all +.DEFAULT: all +all: build test package analyze archive + +# Clean everything but the docker images +.PHONY: clean +clean: clean_dirs + +# Clean everything and the docker image, leaving the repo pristine +.PHONY: distclean +distclean: clean + rm -rf $(MARKERS) + +# All compilation tasks +.PHONY: build +build: $(MARKERS) + +# All tasks which produce a test result +.PHONY: test +test: build + +# Produce packages +.PHONY: package +package: build + +# Linting, Static Analysis, Test Analysis and reporting +.PHONY: analyze +analyze: + +# Archive the source code +.PHONY: archive +archive: $(MARKERS) archive_git + +# Any publishing to external systems other than docker registries, e.g. Maven +.PHONY: publish +publish: package + +# Run up a useful local environment of this build +.PHONY: run +run: package + +.PHONY: acquire_project +acquire_project: + mkdir -p project + if [ -n "$(PROJECT_GIT_URL)" ]; then \ + git clone $(PROJECT_GIT_URL) project ; \ + cd project; \ + git checkout $(PROJECT_BRANCH) ; \ + fi + +.PHONY: clean_project +clean_project: + rm -rf project + +.PHONY: project_% +project_%: + cd project; $(MAKE) -f ../$(firstword $(MAKEFILE_LIST)) $(shell echo $@ | cut -c9-) + +.PHONY: analyze_fossa +analyze_fossa: + if [ -z "$(CHANGE_BRANCH)" ]; then \ + $(TOOL_DEFAULT) -e FOSSA_API_KEY blockchaintp/fossa:latest fossa analyze --verbose \ + --no-ansi -b ${BRANCH_NAME}; \ + $(TOOL_DEFAULT) -e FOSSA_API_KEY blockchaintp/fossa:latest fossa test --verbose \ + --no-ansi -b ${BRANCH_NAME}; \ + else \ + $(TOOL_DEFAULT) -e FOSSA_API_KEY blockchaintp/fossa:latest fossa analyze --verbose \ + --no-ansi -b ${CHANGE_BRANCH}; \ + $(TOOL_DEFAULT) -e FOSSA_API_KEY blockchaintp/fossa:latest fossa test --verbose \ + --no-ansi -b ${CHANGE_BRANCH}; \ + fi + +.PHONY: analyze_go +analyze_go: $(MARKERS)/build_toolchain_docker + $(TOOLCHAIN) bash -c "go vet ./..." + +# Maven Version of Sonar Analysis +.PHONY: analyze_sonar_mvn +analyze_sonar_mvn: $(MARKERS)/build_toolchain_docker + [ -z "$(SONAR_AUTH_TOKEN)" ] || \ + if [ -z "$(CHANGE_BRANCH)" ]; then \ + $(DOCKER_MVN) package sonar:sonar \ + -Dsonar.organization=$(ORGANIZATION) \ + -Dsonar.projectKey=$(ORGANIZATION)_$(REPO) \ + -Dsonar.projectName="$(ORGANIZATION)/$(REPO)" \ + -Dsonar.branch.name=$(BRANCH_NAME) \ + -Dsonar.projectVersion=$(VERSION) \ + -Dsonar.host.url=$(SONAR_HOST_URL) \ + -Dsonar.login=$(SONAR_AUTH_TOKEN) ; \ + else \ + $(DOCKER_MVN) package sonar:sonar \ + -Dsonar.organization=$(ORGANIZATION) \ + -Dsonar.projectKey=$(ORGANIZATION)_$(REPO) \ + -Dsonar.projectName="$(ORGANIZATION)/$(REPO)" \ + -Dsonar.pullrequest.key=$(PR_KEY) \ + -Dsonar.pullrequest.branch=$(CHANGE_BRANCH) \ + -Dsonar.pullrequest.base=$(CHANGE_TARGET) \ + -Dsonar.projectVersion=$(VERSION) \ + -Dsonar.host.url=$(SONAR_HOST_URL) \ + -Dsonar.login=$(SONAR_AUTH_TOKEN) ; \ + fi + +.PHONY: analyze_sonar_generic +analyze_sonar_generic: + [ -z "$(SONAR_AUTH_TOKEN)" ] || \ + if [ -z "$(CHANGE_BRANCH)" ]; then \ + $(DOCKER_RUN_USER) \ + -v $$(pwd):/usr/src \ + sonarsource/sonar-scanner-cli \ + -Dsonar.organization=$(ORGANIZATION) \ + -Dsonar.projectKey=$(ORGANIZATION)_$(REPO) \ + -Dsonar.projectName="$(ORGANIZATION)/$(REPO)" \ + -Dsonar.branch.name=$(BRANCH_NAME) \ + -Dsonar.projectVersion=$(VERSION) \ + -Dsonar.host.url=$(SONAR_HOST_URL) \ + -Dsonar.login=$(SONAR_AUTH_TOKEN) \ + -Dsonar.junit.reportPaths=**/target/surefire-reports; \ + else \ + $(DOCKER_RUN_USER) \ + -v $$(pwd):/usr/src \ + sonarsource/sonar-scanner-cli \ + -Dsonar.organization=$(ORGANIZATION) \ + -Dsonar.projectKey=$(ORGANIZATION)_$(REPO) \ + -Dsonar.projectName="$(ORGANIZATION)/$(REPO)" \ + -Dsonar.pullrequest.key=$(PR_KEY) \ + -Dsonar.pullrequest.branch=$(CHANGE_BRANCH) \ + -Dsonar.pullrequest.base=$(CHANGE_TARGET) \ + -Dsonar.projectVersion=$(VERSION) \ + -Dsonar.host.url=$(SONAR_HOST_URL) \ + -Dsonar.login=$(SONAR_AUTH_TOKEN) \ + -Dsonar.junit.reportPaths=**/target/surefire-reports; \ + fi + +.PHONY: analyze_sonar_js +analyze_sonar_js: + [ -z "$(SONAR_AUTH_TOKEN)" ] || \ + if [ -z "$(CHANGE_BRANCH)" ]; then \ + $(DOCKER_RUN_USER) \ + -v $$(pwd):/usr/src \ + sonarsource/sonar-scanner-cli \ + -Dsonar.organization=$(ORGANIZATION) \ + -Dsonar.projectKey=$(ORGANIZATION)_$(REPO) \ + -Dsonar.projectName="$(ORGANIZATION)/$(REPO)" \ + -Dsonar.branch.name=$(BRANCH_NAME) \ + -Dsonar.projectVersion=$(LONG_VERSION) \ + -Dsonar.host.url=$(SONAR_HOST_URL) \ + -Dsonar.login=$(SONAR_AUTH_TOKEN) \ + -Dsonar.sources=src \ + -Dsonar.tests=test \ + -Dsonar.junit.reportPaths=build/junit.xml \ + -Dsonar.javascript.lcov.reportPaths=build/lcov.info; \ + else \ + $(DOCKER_RUN_USER) \ + -v $$(pwd):/usr/src \ + sonarsource/sonar-scanner-cli \ + -Dsonar.organization=$(ORGANIZATION) \ + -Dsonar.projectKey=$(ORGANIZATION)_$(REPO) \ + -Dsonar.projectName="$(ORGANIZATION)/$(REPO)" \ + -Dsonar.pullrequest.key=$(PR_KEY) \ + -Dsonar.pullrequest.branch=$(CHANGE_BRANCH) \ + -Dsonar.pullrequest.base=$(CHANGE_TARGET) \ + -Dsonar.projectVersion=$(LONG_VERSION) \ + -Dsonar.host.url=$(SONAR_HOST_URL) \ + -Dsonar.login=$(SONAR_AUTH_TOKEN) \ + -Dsonar.sources=src \ + -Dsonar.tests=test \ + -Dsonar.junit.reportPaths=build/junit.xml \ + -Dsonar.javascript.lcov.reportPaths=build/lcov.info; \ + fi + +.PHONY: archive_git +archive_git: build/$(REPO)-$(VERSION).zip build/$(REPO)-$(VERSION).tgz + +build/$(REPO)-$(VERSION).zip: + if [ -d .git ]; then \ + git archive HEAD --format=zip -9 --output=build/$(REPO)-$(VERSION).zip; \ + fi + +build/$(REPO)-$(VERSION).tgz: + if [ -d .git ]; then \ + git archive HEAD --format=zip -9 --output=build/$(REPO)-$(VERSION).tgz; \ + fi + +$(MARKERS)/toolchain_vols: + docker volume create toolchain-home-$(ISOLATION_ID) + $(BUSYBOX_ROOT) chown -R $(UID):$(GID) $(TOOLCHAIN_HOME) + touch $@ + +$(MARKERS)/build_toolchain_docker: $(MARKERS) $(MARKERS)/toolchain_vols + if ! docker image ls -qq $(TOOLCHAIN_IMAGE) > /dev/null; then \ + echo "Pulling toolchain $(TOOLCHAIN_IMAGE)"; \ + docker pull -qq $(TOOLCHAIN_IMAGE); \ + else \ + echo "Toolchain $(TOOLCHAIN_IMAGE) already available"; \ + fi + touch $@ + +.PHONY: clean_toolchain_docker +clean_toolchain_docker: + docker rmi -f $(TOOLCHAIN_IMAGE) + docker volume rm -f toolchain-home-$(ISOLATION_ID) + rm -f $(MARKERS)/toolchain_vols + rm -f $(MARKERS)/build_toolchain_docker + +.PHONY: fix_permissions +fix_permissions: + $(BUSYBOX_ROOT) chown -R $(UID):$(GID) $(TOOLCHAIN_HOME)/.m2/repository || true + $(BUSYBOX_ROOT) chown -R $(UID):$(GID) /project || true + +# This will reset the build status possible causing steps to rerun +.PHONY: clean_markers +clean_markers: + rm -rf $(MARKERS) + rm -rf build + +$(MARKERS)/build_go: $(MARKERS)/build_toolchain_docker + $(TOOLCHAIN) bash -c "if [ -r scripts/build ]; then scripts/build; else go build ./...; fi" + touch $@ + +.PHONY: clean_build_go +clean_build_go: $(MARKERS)/build_toolchain_docker + $(TOOLCHAIN) go clean ./... + rm -f $(MARKERS)/build_go + +$(MARKERS)/build_mvn: $(MARKERS)/build_toolchain_docker + $(DOCKER_MVN) compile + touch $@ + +$(MARKERS)/package_mvn: $(MARKERS)/build_toolchain_docker + $(DOCKER_MVN) package verify + touch $@ + +clean_mvn: $(MARKERS)/build_toolchain_docker + $(DOCKER_MVN) clean + +$(MARKERS)/test_mvn: $(MARKERS)/build_toolchain_docker + $(DOCKER_MVN) test + touch $@ + +$(MARKERS)/test_go: $(MARKERS)/build_toolchain_docker + $(TOOLCHAIN) go test ./... + touch $@ + +$(MARKERS)/publish_mvn: $(MARKERS)/build_toolchain_docker + echo $(DOCKER_MVN) clean deploy -DupdateReleaseInfo=$(MAVEN_UPDATE_RELEASE_INFO) \ + -DaltDeploymentRepository=$(MAVEN_DEPLOY_TARGET) + $(DOCKER_MVN) -Drevision=0.0.0 versions:set -DnewVersion=$(MAVEN_REVISION) + $(DOCKER_MVN) clean deploy -DupdateReleaseInfo=$(MAVEN_UPDATE_RELEASE_INFO) \ + -DaltDeploymentRepository=$(MAVEN_DEPLOY_TARGET) + +.PHONY: effective-pom +effective-pom: $(MARKERS)/build_toolchain_docker + $(DOCKER_MVN) help:effective-pom + +# Any prerequisite directories, which should also be gitignored +$(MARKERS): + mkdir -p $(MARKERS) + mkdir -p build + +.PHONY: clean_dirs_standard +clean_dirs_standard: + rm -rf build + rm -rf $(MARKERS) + +.PHONY: clean_dirs +clean_dirs: clean_dirs_standard + +$(MARKERS)/check_ignores: + git check-ignore build + git check-ignore $(MARKERS) + touch $@ + +.PHONY: what_version +what_version: + @echo VERSION=$(VERSION) + @echo LONG_VERSION=$(LONG_VERSION) + @echo MAVEN_REVISION=$(MAVEN_REVISION) + +## +# END Standardized directives +## + +## +# GitHub release directives +## +GH := $(TOOLCHAIN) gh +GH_RELEASE = $(GH) release + +.PHONY: gh-create-draft-release +gh-create-draft-release: + if [ "$(RELEASABLE)" = "yes" ];then \ + $(GH_RELEASE) create $(VERSION) -t "$(VERSION)" -F CHANGELOG.md; \ + fi