From e7584ce9d2c09bd6bdb75192dad6d3f206fffd5a Mon Sep 17 00:00:00 2001 From: Akash Kollipara Date: Tue, 30 Jul 2024 20:20:16 +0530 Subject: [PATCH] Add build cache to speed up builds --- .github/workflows/github_ci.yml | 13 ++- .gitignore | 1 + Makefile | 2 + mk/ccache.mk | 141 +++++++++++++++++++++++++++++ mk/elf.mk | 4 +- mk/lib.mk | 2 +- mk/lobj.mk | 12 +-- mk/obj.mk | 12 +-- mk/path.mk | 2 +- mk/project.mk | 10 +- mk/slib.mk | 2 +- src/visor/services/kernel/build.mk | 2 +- 12 files changed, 178 insertions(+), 25 deletions(-) create mode 100644 mk/ccache.mk diff --git a/.github/workflows/github_ci.yml b/.github/workflows/github_ci.yml index fcf059dd..b96059d9 100644 --- a/.github/workflows/github_ci.yml +++ b/.github/workflows/github_ci.yml @@ -31,12 +31,12 @@ jobs: - name: Fetch Dependencies run: | - sudo apt install cppcheck -y + sudo apt install cppcheck ccache -y make get_avr_tc make get_riscv_tc - name: Init CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} debug: true @@ -51,10 +51,13 @@ jobs: make demo_qemu_sifive_e make demo_ibex_ss + - name: Build Cache Stats + run: | + make show_ccache_stats + - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v3 - name: Clean Up run: | - make clean - rm -rf toolchain + make clean_workspace diff --git a/.gitignore b/.gitignore index 07daea91..a1d1fe4e 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ tools/ backup/ bkp/ projects/ +.buildcache/ *.elf *.bin *.d diff --git a/Makefile b/Makefile index 6c93a73f..e1416d47 100644 --- a/Makefile +++ b/Makefile @@ -15,11 +15,13 @@ SHELL := /bin/bash N_JOBS ?= $(shell grep -c ^processor /proc/cpuinfo) HOST_ARCH := $(shell lscpu | grep -Po '(?<='Architecture:' )[^"]*') +EN_BUILD_CACHE ?= 1 V ?= 0 PP ?= 0 include mk/help.mk include mk/path.mk +include mk/ccache.mk include mk/tc_get.mk include mk/project.mk diff --git a/mk/ccache.mk b/mk/ccache.mk new file mode 100644 index 00000000..ab594f39 --- /dev/null +++ b/mk/ccache.mk @@ -0,0 +1,141 @@ +# +# CYANCORE LICENSE +# Copyrights (C) 2024, Cyancore Team +# +# File Name : ccache.mk +# Description : This file provides recipes for build cache +# Primary Author : Akash Kollipara [akashkollipara@gmail.com] +# Organisation : Cyancore Core-Team +# + +#*.*.*.*.*.*.*.*.*.*.*.*.*.*.*.*.*.*.*.*.*.*# +#--------------< Build Cache >--------------# +#*.*.*.*.*.*.*.*.*.*.*.*.*.*.*.*.*.*.*.*.*.*# + +CCACHE_DIR := $(CC_ROOT)/.buildcache + +# For Flags read ccache documentation +CCACHE_ABSSTDERR := true +export CCACHE_ABSSTDERR +CCACHE_BASEDIR := +export CCACHE_BASEDIR +CCACHE_DIR := $(CCACHE_DIR)/cache +export CCACHE_DIR +CCACHE_COMPILER := +export CCACHE_COMPILER +CCACHE_COMPILERCHECK := mtime +export CCACHE_COMPILERCHECK +CCACHE_COMPILERTYPE := auto +export CCACHE_COMPILERTYPE +export CCACHE_COMPRESS +CCACHE_COMPRESSLEVEL := 0 +export CCACHE_COMPRESSLEVEL +CCACHE_EXTENSION := +export CCACHE_EXTENSION +# export CCACHE_DEBUG +export CCACHE_NODEBUG +CCACHE_DEBUGDIR := $(CCACHE_DIR)/dbg +export CCACHE_DEBUGDIR +CCACHE_DEBUGLEVEL := 2 +export CCACHE_DEBUGLEVEL +# export CCACHE_DEPEND +export CCACHE_NODEPEND +export CCACHE_DIRECT +# export CCACHE_NODIRECT +# export CCACHE_DISABLE +export CCACHE_NODISABLE +CCACHE_EXTRAFILES := +export CCACHE_EXTRAFILES +# export CCACHE_FILECLONE +export CCACHE_NOFILECLONE +# export CCACHE_HARDLINK +export CCACHE_NOHARDLINK +export CCACHE_HASHDIR +# export CCACHE_NOHASHDIR +CCACHE_IGNOREHEADERS := +export CCACHE_IGNOREHEADERS +CCACHE_IGNOREOPTIONS := +export CCACHE_IGNOREOPTIONS +# export CCACHE_INODECACHE +export CCACHE_NOINODECACHE +# export CCACHE_COMMENTS +export CCACHE_NOCOMMENTS +CCACHE_LOGFILE := $(CCACHE_DIR)/log +export CCACHE_LOGFILE +CCACHE_MAXFILES := 0 +export CCACHE_MAXFILES +CCACHE_MAXSIZE := 1.0 GiB +export CCACHE_MAXSIZE +CCACHE_MSVC_DEP_PREFIX := Note: including file: +export CCACHE_MSVC_DEP_PREFIX +CCACHE_NAMESPACE := cyancore +export CCACHE_NAMESPACE +CCACHE_PATH := +export CCACHE_PATH +# export CCACHE_PCH_EXTSUM +export CCACHE_NOPCH_EXTSUM +CCACHE_PREFIX := +export CCACHE_PREFIX +CCACHE_PREFIX_CPP := +export CCACHE_PREFIX_CPP +# export CCACHE_READONLY +export CCACHE_NOREADONLY +# export CCACHE_READONLY_DIRECT +export CCACHE_NOREADONLY_DIRECT +# export CCACHE_RECACHE +export CCACHE_NORECACHE +# export CCACHE_REMOTE_ONLY +export CCACHE_NOREMOTE_ONLY +CCACHE_REMOTE_STORAGE := +export CCACHE_REMOTE_STORAGE +# export CCACHE_RESHARE +export CCACHE_NORESHARE +export CCACHE_CPP2 +# export CCACHE_NOCPP2 +CCACHE_SLOPPINESS := +export CCACHE_SLOPPINESS +# export CCACHE_NOSTATS +export CCACHE_STATS +export CCACHE_STATSLOG := $(CCACHE_DIR)/stats +export CCACHE_TEMPDIR := $(CCACHE_DIR)/ +CCACHE_UMASK := +export CCACHE_UMASK + + +ifeq ($(EN_BUILD_CACHE),1) +CCACHE := ccache + +T_ALLOWLIST += show_ccache_config show_ccache_stats clean_ccache + +CHECK_CCACHE := $(shell which $(CCACHE)) +ifeq ($(CHECK_CCACHE),) +$(info < ! > ccache is not installed!) +$(error < x > Stopping build.) +endif + +show_ccache_stats: +ifneq ($(realpath $(CCACHE_LOGFILE)),) + $(CCACHE) -s -x -v +else + $(info < ! > No log file present, try to perform build and try!) +endif + +show_ccache_config: --prepare-cache + $(CCACHE) -p + +clean_ccache: +ifneq ($(realpath $(CCACHE_LOGFILE)),) + $(CCACHE) -c -C -z +else + $(info < ! > No log file present, try to perform build and try!) +endif + + +.PHONY: --prepare-cache +--prepare-cache: $(CCACHE_LOGFILE) + +.SECONDEXPANSION: +$(CCACHE_LOGFILE): | $$(@D)/ + touch $@ +endif + diff --git a/mk/elf.mk b/mk/elf.mk index 0ce895d0..a847e263 100644 --- a/mk/elf.mk +++ b/mk/elf.mk @@ -29,7 +29,7 @@ elf: $(ELF) .SECONDEXPANSION: $(ELF): $(DEP_LIBS) $(DEP_OBJS) $(LD_SCRIPT) $(LD_SUPPLEMENT) | $$(SIZE) @echo "Elf: Generating $(@F) ..." - $(LD) -dT $(LD_SCRIPT) $(addprefix -T , $(LD_SUPPLEMENT)) $(LD_FLAGS) \ + $(CCACHE) $(LD) -dT $(LD_SCRIPT) $(addprefix -T , $(LD_SUPPLEMENT)) $(LD_FLAGS) \ -Map=$(@:.elf=.map) -o $@ $(filter %.o, $^) $(DEP_LIB_PATH) $(DEP_LIBS_ARG) $(EXLIB_ARGS) -L $(TL) -lgcc $(OD) -Dx -h --wide $@ > $(@:.elf=.lst) $(OC) -O binary $@ $(@:.elf=.bin) @@ -40,5 +40,5 @@ $(ELF): $(DEP_LIBS) $(DEP_OBJS) $(LD_SCRIPT) $(LD_SUPPLEMENT) | $$(SIZE) $(OUT)/%.ld: %.ld.sx mkdir -p $(@D) @echo "Elf: Preprocessing $(@F) ..." - $(CC) $(CFLAGS) -E -P -o $@ $< + $(CCACHE) $(CC) $(CFLAGS) -E -P -o $@ $< diff --git a/mk/lib.mk b/mk/lib.mk index 05278625..e94493c1 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -22,7 +22,7 @@ AR_FLAGS ?= rcs $(LIB): $(LIB_OBJS) | $$(@D)/ @echo "Lib: Generating $(@F) ..." - $(AR) $(AR_FLAGS) $@ $^ + $(CCACHE) $(AR) $(AR_FLAGS) $@ $^ LIB_INCLUDE_PATH:= LIB_OBJS := diff --git a/mk/lobj.mk b/mk/lobj.mk index 95f50be1..2df808a6 100644 --- a/mk/lobj.mk +++ b/mk/lobj.mk @@ -28,21 +28,21 @@ DEP_SRCS += $(C_SRCS) $(CPP_SRCS) $(CPP_OBJS): $(OUT)/%.o: %.cpp | $$(@D)/ @echo "Elf: Compiling $(@F:.o=.cpp) ..." ifeq ($(PP),1) - $(CCP) $(CPPFLAGS) $(CFLAGS) -E -p $< -o $(@:.o=.pre.cpp) + $(CCACHE) $(CCP) $(CPPFLAGS) $(CFLAGS) -E -p $< -o $(@:.o=.pre.cpp) endif - $(CCP) $(CPPFLAGS) $(CFLAGS) -c $< -o $@ + $(CCACHE) $(CCP) $(CPPFLAGS) $(CFLAGS) -c $< -o $@ $(C_OBJS): $(OUT)/%.o: %.c | $$(@D)/ @echo "Lib: Compiling $(@F:.o=.c) ..." ifeq ($(PP),1) - $(CC) $(CCFLAGS) $(CFLAGS) -E -p $< -o $(@:.o=.pre.c) + $(CCACHE) $(CC) $(CCFLAGS) $(CFLAGS) -E -p $< -o $(@:.o=.pre.c) endif - $(CC) $(CCFLAGS) $(CFLAGS) -c $< -o $@ + $(CCACHE) $(CC) $(CCFLAGS) $(CFLAGS) -c $< -o $@ $(S_OBJS): $(OUT)/%.o: %.S | $$(@D)/ @echo "Lib: Assembling $(@F:.o=.S) ..." - $(CC) -E $(CCFLAGS) $(CFLAGS) -c $< -o $(@:.o=.pre.S) - $(AS) $(ASFLAGS) $(@:.o=.pre.S) -o $@ + $(CCACHE) $(CC) -E $(CCFLAGS) $(CFLAGS) -c $< -o $(@:.o=.pre.S) + $(CCACHE) $(AS) $(ASFLAGS) $(@:.o=.pre.S) -o $@ ifneq ($(PP),1) rm $(@:.o=.pre.S) endif diff --git a/mk/obj.mk b/mk/obj.mk index 11a9a856..89a12000 100644 --- a/mk/obj.mk +++ b/mk/obj.mk @@ -28,21 +28,21 @@ DEP_SRCS += $(C_SRCS) $(CPP_SRCS) $(CPP_OBJS): $(OUT)/%.o: %.cpp | $$(@D)/ @echo "Elf: Compiling $(@F:.o=.cpp) ..." ifeq ($(PP),1) - $(CCP) $(CPPFLAGS) $(CFLAGS) -E -p $< -o $(@:.o=.pre.cpp) + $(CCACHE) $(CCP) $(CPPFLAGS) $(CFLAGS) -E -p $< -o $(@:.o=.pre.cpp) endif - $(CCP) $(CPPFLAGS) $(CFLAGS) -c $< -o $@ + $(CCACHE) $(CCP) $(CPPFLAGS) $(CFLAGS) -c $< -o $@ $(C_OBJS): $(OUT)/%.o: %.c | $$(@D)/ @echo "Elf: Compiling $(@F:.o=.c) ..." ifeq ($(PP),1) - $(CC) $(CCFLAGS) $(CFLAGS) -E -p $< -o $(@:.o=.pre.c) + $(CCACHE) $(CC) $(CCFLAGS) $(CFLAGS) -E -p $< -o $(@:.o=.pre.c) endif - $(CC) $(CCFLAGS) $(CFLAGS) -c $< -o $@ + $(CCACHE) $(CC) $(CCFLAGS) $(CFLAGS) -c $< -o $@ $(S_OBJS): $(OUT)/%.o: %.S | $$(@D)/ @echo "Elf: Assembling $(@F:.o=.S) ..." - $(CC) -E $(CCFLAGS) $(CFLAGS) -c $< -o $(@:.o=.pre.S) - $(AS) $(ASFLAGS) $(@:.o=.pre.S) -o $@ + $(CCACHE) $(CC) -E $(CCFLAGS) $(CFLAGS) -c $< -o $(@:.o=.pre.S) + $(CCACHE) $(AS) $(ASFLAGS) $(@:.o=.pre.S) -o $@ ifneq ($(PP),1) rm $(@:.o=.pre.S) endif diff --git a/mk/path.mk b/mk/path.mk index 24318b40..2e3383a6 100644 --- a/mk/path.mk +++ b/mk/path.mk @@ -23,5 +23,5 @@ OUT ?= $(OUT_PATH)/$(PROJECT) GET_PATH = $(patsubst %/,%,$(dir $(lastword $(MAKEFILE_LIST)))) -$(OUT)/%/: +./%/: mkdir -p $@ diff --git a/mk/project.mk b/mk/project.mk index f9148fdc..795fc940 100644 --- a/mk/project.mk +++ b/mk/project.mk @@ -18,7 +18,7 @@ include mk/qemu.mk include mk/picotool.mk P_TARGETS += default cyancore check version copy_to_remote clean_remote -T_ALLOWLIST += list clean all_projects +T_ALLOWLIST += list clean all_projects clean_workspace setup_workspace PROJECT_LIST := $(shell ls projects/ -I *.template -I *.src) .PHONY: aux_target @@ -33,13 +33,19 @@ all_projects: make $$project; \ done -cyancore: version elf +cyancore: --prepare-cache version elf $(info < / > Done !) clean: $(info < ! > Removing $(PROJECT) binaries ...) rm -rf $(OUT) +setup_workspace: $(SIZE) get_qemu get_all_tc + +clean_workspace: clean + $(info < / > Cleaning up workspace ...) + rm -rf $(CCACHE_DIR) $(TOOLS_ROOT) + list: $(info Available projects are :) echo $(PROJECT_LIST) diff --git a/mk/slib.mk b/mk/slib.mk index 3382b067..ede8f6b5 100644 --- a/mk/slib.mk +++ b/mk/slib.mk @@ -22,7 +22,7 @@ slib: $(SLIB) $(SLIB): $(HEADER) $(LIB_OBJS) echo "Generating $(notdir $@) ..." - $(LD) --shared -Wl,-soname,$(notdir $@) -o $@ $< + $(CCACHE) $(LD) --shared -Wl,-soname,$(notdir $@) -o $@ $< $(HEADER): --dependency cp $@ $(OUT) diff --git a/src/visor/services/kernel/build.mk b/src/visor/services/kernel/build.mk index 41d0f45f..58f18c55 100644 --- a/src/visor/services/kernel/build.mk +++ b/src/visor/services/kernel/build.mk @@ -11,4 +11,4 @@ KERNEL_DIR := $(GET_PATH) -$(eval $(call check_and_include KHELIOS,$(KERNEL_DIR)/helios/build.mk)) +$(eval $(call check_and_include,KHELIOS,$(KERNEL_DIR)/helios/build.mk))