diff --git a/.ci/fetch.sh b/.ci/fetch.sh new file mode 100755 index 00000000..cefad9bd --- /dev/null +++ b/.ci/fetch.sh @@ -0,0 +1,250 @@ +#!/usr/bin/env bash + +# Reliable download wrapper with retry logic and timeout handling +# Prefers curl over wget for better error handling and features + +set -euo pipefail + +# Source common utilities +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=.ci/common.sh +source "${SCRIPT_DIR}/common.sh" + +# Configuration +RETRIES=3 +TIMEOUT=30 +READ_TIMEOUT=60 +WAIT_RETRY=1 + +usage() +{ + local exit_code="${1:-0}" + cat << EOF +Usage: $0 [OPTIONS] URL [OUTPUT_FILE] + +Reliable download wrapper with automatic retry logic. + +OPTIONS: + -h, --help Show this help message + -r, --retries N Number of retry attempts (default: $RETRIES) + -t, --timeout N Timeout in seconds (default: $TIMEOUT) + -o, --output FILE Output file path + -q, --quiet Quiet mode (minimal output) + +EXAMPLES: + $0 https://example.com/file.tar.gz + $0 -o output.zip https://example.com/archive.zip + $0 --retries 5 --timeout 30 https://example.com/large-file.bin + +ENVIRONMENT: + PREFER_WGET=1 Force using wget instead of curl +EOF + exit "$exit_code" +} + +log() +{ + echo -e "${GREEN}[reliable-download]${NC} $*" >&2 +} + +# Parse arguments +QUIET=0 +OUTPUT="" +URL="" + +while [[ $# -gt 0 ]]; do + case $1 in + -h | --help) + usage + ;; + -r | --retries) + RETRIES="$2" + shift 2 + ;; + -t | --timeout) + TIMEOUT="$2" + shift 2 + ;; + -o | --output) + OUTPUT="$2" + shift 2 + ;; + -q | --quiet) + QUIET=1 + shift + ;; + -*) + print_error "Unknown option: $1" + usage 1 + ;; + *) + if [[ -z "$URL" ]]; then + URL="$1" + elif [[ -z "$OUTPUT" ]]; then + OUTPUT="$1" + else + print_error "Too many arguments" + usage 1 + fi + shift + ;; + esac +done + +# Validate URL +if [[ -z "$URL" ]]; then + print_error "URL is required" + usage 1 +fi + +# Determine output file +if [[ -z "$OUTPUT" ]]; then + OUTPUT=$(basename "$URL") +fi + +# Check available tools +HAS_CURL=0 +HAS_WGET=0 + +if command -v curl &> /dev/null; then + HAS_CURL=1 +fi + +if command -v wget &> /dev/null; then + HAS_WGET=1 +fi + +if [[ $HAS_CURL -eq 0 && $HAS_WGET -eq 0 ]]; then + print_error "Neither curl nor wget is available" + exit 1 +fi + +# Prefer curl unless PREFER_WGET is set +USE_CURL=0 +if [[ $HAS_CURL -eq 1 && -z "${PREFER_WGET:-}" ]]; then + USE_CURL=1 +elif [[ $HAS_WGET -eq 0 ]]; then + print_error "wget not available and PREFER_WGET is set" + exit 1 +fi + +# Download function +download() +{ + local attempt=$1 + local url=$2 + local output=$3 + + [[ $QUIET -eq 0 ]] && log "Attempt $attempt/$RETRIES: Downloading $url" + + if [[ $USE_CURL -eq 1 ]]; then + # Verify curl is available + if ! command -v curl > /dev/null 2>&1; then + print_error "curl binary not found in PATH" + return 1 + fi + # curl options: + # -f: Fail silently on HTTP errors + # -L: Follow redirects + # -S: Show error even with -s + # -s: Silent mode (if quiet) + # --retry: Number of retries + # --retry-delay: Wait time between retries + # --connect-timeout: Connection timeout + # --max-time: Total operation timeout + # -o: Output file + local curl_opts=( + -f + -L + --retry "$RETRIES" + --retry-delay "$WAIT_RETRY" + --retry-connrefused + --connect-timeout "$TIMEOUT" + --max-time "$READ_TIMEOUT" + -o "$output" + ) + + if [[ $QUIET -eq 1 ]]; then + curl_opts+=(-s -S) + else + curl_opts+=(--progress-bar) + fi + + curl "${curl_opts[@]}" "$url" + else + # Verify wget is available + if ! command -v wget > /dev/null 2>&1; then + print_error "wget binary not found in PATH" + return 1 + fi + # wget options: + # --retry-connrefused: Retry on connection refused + # --waitretry: Wait between retries + # --read-timeout: Read timeout + # --timeout: Connection timeout + # --tries: Number of attempts + # -O: Output file + local wget_opts=( + --retry-connrefused + --waitretry="$WAIT_RETRY" + --read-timeout="$READ_TIMEOUT" + --timeout="$TIMEOUT" + --tries="$RETRIES" + -O "$output" + ) + + if [[ $QUIET -eq 1 ]]; then + wget_opts+=(-q) + fi + + wget "${wget_opts[@]}" "$url" + fi +} + +# Main download logic with retry using atomic operations +SUCCESS=0 +TEMP_OUTPUT="${OUTPUT}.downloading.$$" + +# Cleanup trap for partial downloads +trap 'rm -f "$TEMP_OUTPUT"' EXIT INT TERM + +for i in $(seq 1 "$RETRIES"); do + if download "$i" "$URL" "$TEMP_OUTPUT"; then + # Atomic move to final location + if mv -f "$TEMP_OUTPUT" "$OUTPUT"; then + SUCCESS=1 + [[ $QUIET -eq 0 ]] && log "Download successful: $OUTPUT" + break + else + print_error "Failed to move temporary file to $OUTPUT" + fi + else + EXIT_CODE=$? + print_warning "Download failed (attempt $i/$RETRIES, exit code: $EXIT_CODE)" + + if [[ $i -lt $RETRIES ]]; then + [[ $QUIET -eq 0 ]] && log "Retrying in ${WAIT_RETRY}s..." + sleep "$WAIT_RETRY" + fi + fi +done + +if [[ $SUCCESS -eq 0 ]]; then + print_error "Download failed after $RETRIES attempts" + rm -f "$TEMP_OUTPUT" # Clean up partial download + exit 1 +fi + +# Verify file was created and is not empty +if [[ ! -f "$OUTPUT" ]]; then + print_error "Output file not created: $OUTPUT" + exit 1 +fi + +if [[ ! -s "$OUTPUT" ]]; then + print_error "Output file is empty: $OUTPUT" + rm -f "$OUTPUT" + exit 1 +fi + +exit 0 diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 5e01e1be..c7873d6e 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -2,13 +2,21 @@ name: Benchmark on: [push, pull_request_target, workflow_dispatch] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.event.pull_request.head.sha || github.sha }} + cancel-in-progress: true + jobs: benchmark: name: Performance regression check if: contains(toJSON(github.event.head_commit.message), 'Merge pull request ') == false + timeout-minutes: 30 runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 + with: + submodules: 'true' + - name: Test changed files id: changed-files uses: tj-actions/changed-files@v46 @@ -19,24 +27,35 @@ jobs: src/emulate.c src/rv32_template.c src/rv32_constopt.c - - name: install-dependencies - if: ${{ steps.changed-files.outputs.any_changed == 'true' || - github.event_name == 'workflow_dispatch'}} + src/cache.c + src/io.c + src/jit.c + + - name: Cache benchmark artifacts + if: steps.changed-files.outputs.any_changed == 'true' || github.event_name == 'workflow_dispatch' + uses: actions/cache@v4 + with: + path: build/ + key: benchmark-artifacts-${{ runner.os }}-${{ hashFiles('mk/artifact.mk') }} + restore-keys: | + benchmark-artifacts-${{ runner.os }}- + + - name: Install dependencies + if: steps.changed-files.outputs.any_changed == 'true' || github.event_name == 'workflow_dispatch' run: | sudo pip3 install numpy --break-system-packages shell: bash - - name: default build - if: ${{ steps.changed-files.outputs.any_changed == 'true' || - github.event_name == 'workflow_dispatch'}} + + - name: Build for benchmark + if: steps.changed-files.outputs.any_changed == 'true' || github.event_name == 'workflow_dispatch' run: make ENABLE_SDL=0 all artifact - name: Run benchmark - if: ${{ steps.changed-files.outputs.any_changed == 'true' || - github.event_name == 'workflow_dispatch'}} + if: steps.changed-files.outputs.any_changed == 'true' || github.event_name == 'workflow_dispatch' run: | tests/bench-aggregator.py + - name: Store benchmark results - if: ${{ steps.changed-files.outputs.any_changed == 'true' || - github.event_name == 'workflow_dispatch'}} + if: steps.changed-files.outputs.any_changed == 'true' || github.event_name == 'workflow_dispatch' uses: benchmark-action/github-action-benchmark@v1 with: name: Benchmarks diff --git a/.github/workflows/build-artifact.yml b/.github/workflows/build-artifact.yml index 3140f8fe..49394928 100644 --- a/.github/workflows/build-artifact.yml +++ b/.github/workflows/build-artifact.yml @@ -6,9 +6,13 @@ on: - master workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: detect-file-change: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Checkout repository uses: actions/checkout@v4 @@ -41,7 +45,7 @@ jobs: build-artifact: needs: [detect-file-change] if: ${{ needs.detect-file-change.outputs.has_changed_files == 'true' || github.event_name == 'workflow_dispatch' }} - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Checkout repository uses: actions/checkout@v4 diff --git a/.github/workflows/build-linux-artifacts.yml b/.github/workflows/build-linux-artifacts.yml index cf4043da..2af9ebb6 100644 --- a/.github/workflows/build-linux-artifacts.yml +++ b/.github/workflows/build-linux-artifacts.yml @@ -6,6 +6,10 @@ on: - master workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: detect-file-change: runs-on: ubuntu-24.04 @@ -40,7 +44,7 @@ jobs: build-linux-image-artifact: needs: [detect-file-change] if: ${{ needs.detect-file-change.outputs.has_changed_linux_image_version == 'true' || github.event_name == 'workflow_dispatch' }} - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Checkout repository uses: actions/checkout@v4 diff --git a/.github/workflows/deploy-wasm.yml b/.github/workflows/deploy-wasm.yml index 1a228f68..c7520cca 100644 --- a/.github/workflows/deploy-wasm.yml +++ b/.github/workflows/deploy-wasm.yml @@ -12,19 +12,25 @@ on: repository_dispatch: # listening to rv32emu-prebuilt events types: [deploy_user_wasm, deploy_system_wasm] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: false # Don't cancel deployments + jobs: wasm-system-deploy: if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' && github.event.action == 'deploy_system_wasm' - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Check out the repo uses: actions/checkout@v4 + with: + submodules: 'true' - name: install-dependencies run: | sudo apt-get update -q=2 - sudo apt-get install -q=2 device-tree-compiler + sudo apt-get install -q=2 curl device-tree-compiler - name: Verify if the JS or HTML files has been modified id: changed-files uses: tj-actions/changed-files@v46 @@ -57,14 +63,14 @@ jobs: run: | make artifact # get from rv32emu-prebuilt - wget -O build/shareware_doom_iwad.zip "https://raw.githubusercontent.com/sysprog21/rv32emu-prebuilt/doom-artifact/shareware_doom_iwad.zip" - unzip -d build/ build/shareware_doom_iwad.zip + .ci/fetch.sh -o build/shareware_doom_iwad.zip "https://raw.githubusercontent.com/sysprog21/rv32emu-prebuilt/doom-artifact/shareware_doom_iwad.zip" + unzip -o -d build/ build/shareware_doom_iwad.zip - name: build with emcc and move application files to /tmp if: ${{ steps.changed-files.outputs.any_modified == 'true' || github.event_name == 'workflow_dispatch' || (github.event_name == 'repository_dispatch' && github.event.action == 'deploy_system_wasm') }} run: | - make CC=emcc ENABLE_SYSTEM=1 ENABLE_SDL=1 INITRD_SIZE=32 -j + make CC=emcc ENABLE_SYSTEM=1 ENABLE_SDL=1 ENABLE_JIT=0 INITRD_SIZE=32 -j mkdir /tmp/rv32emu-system-demo mv assets/wasm/html/system.html /tmp/rv32emu-system-demo/index.html mv assets/wasm/js/coi-serviceworker.min.js /tmp/rv32emu-system-demo @@ -114,12 +120,22 @@ jobs: github_token: ${{ secrets.RV32EMU_DEMO_TOKEN }} branch: main wasm-user-deploy: - needs: wasm-system-deploy # run jobs sequentially since two jobs operate on same reposity: rv32emu-demo - if: always() # ensures wasm-user-deploy runs regardless of the outcome or condition of wasm-system-deploy - runs-on: ubuntu-latest + needs: wasm-system-deploy # run jobs sequentially since two jobs operate on same repository: rv32emu-demo + if: | + !cancelled() && ( + needs.wasm-system-deploy.result == 'success' || + (needs.wasm-system-deploy.result == 'skipped' && github.event_name == 'repository_dispatch' && github.event.action == 'deploy_user_wasm') + ) + runs-on: ubuntu-24.04 steps: - name: Check out the repo uses: actions/checkout@v4 + with: + submodules: 'true' + - name: install-dependencies + run: | + sudo apt-get update -q=2 + sudo apt-get install -q=2 curl device-tree-compiler - name: Verify if the JS or HTML or ELF executable files has been modified id: changed-files uses: tj-actions/changed-files@v46 @@ -152,14 +168,14 @@ jobs: run: | make artifact # get from rv32emu-prebuilt - wget -O build/shareware_doom_iwad.zip "https://raw.githubusercontent.com/sysprog21/rv32emu-prebuilt/doom-artifact/shareware_doom_iwad.zip" - unzip -d build/ build/shareware_doom_iwad.zip + .ci/fetch.sh -o build/shareware_doom_iwad.zip "https://raw.githubusercontent.com/sysprog21/rv32emu-prebuilt/doom-artifact/shareware_doom_iwad.zip" + unzip -o -d build/ build/shareware_doom_iwad.zip - name: build with emcc and move application files to /tmp if: ${{ steps.changed-files.outputs.any_modified == 'true' || github.event_name == 'workflow_dispatch' || (github.event_name == 'repository_dispatch' && github.event.action == 'deploy_user_wasm') }} run: | - make CC=emcc ENABLE_SDL=1 + make CC=emcc ENABLE_SDL=1 ENABLE_JIT=0 mkdir /tmp/rv32emu-demo mv assets/wasm/html/user.html /tmp/rv32emu-demo/index.html mv assets/wasm/js/coi-serviceworker.min.js /tmp/rv32emu-demo diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b700ecdc..fb64fefb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -2,6 +2,10 @@ name: CI on: [push, pull_request] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: detect-code-related-file-changes: runs-on: ubuntu-24.04 @@ -36,6 +40,7 @@ jobs: host-x64: needs: [detect-code-related-file-changes] if: needs.detect-code-related-file-changes.outputs.has_code_related_changes == 'true' + timeout-minutes: 60 strategy: fail-fast: false matrix: @@ -43,17 +48,50 @@ jobs: runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - - name: install-dependencies + with: + submodules: 'true' + + - name: Cache LLVM 18 + id: cache-llvm + uses: actions/cache@v4 + with: + path: /usr/lib/llvm-18 + key: ${{ runner.os }}-${{ runner.arch }}-llvm-18-v2 + restore-keys: | + ${{ runner.os }}-${{ runner.arch }}-llvm-18- + + - name: Cache RISC-V Toolchain + id: cache-toolchain + uses: actions/cache@v4 + with: + path: ${{ github.workspace }}/toolchain + key: ${{ runner.os }}-${{ runner.arch }}-riscv-toolchain-${{ hashFiles('.ci/riscv-toolchain-install.sh') }} + + - name: Install dependencies run: | sudo apt-get update -q=2 - sudo apt-get install -q=2 libsdl2-dev libsdl2-mixer-dev device-tree-compiler expect bc p7zip-full + sudo apt-get install -q=2 curl libsdl2-dev libsdl2-mixer-dev device-tree-compiler expect bc p7zip-full + shell: bash + + - name: Install RISC-V Toolchain + if: steps.cache-toolchain.outputs.cache-hit != 'true' + run: | .ci/riscv-toolchain-install.sh - echo "${{ github.workspace }}/toolchain/bin" >> $GITHUB_PATH - wget https://apt.llvm.org/llvm.sh - sudo chmod +x ./llvm.sh + + - name: Setup RISC-V Toolchain PATH + run: echo "${{ github.workspace }}/toolchain/bin" >> $GITHUB_PATH + + - name: Install LLVM 18 + if: steps.cache-llvm.outputs.cache-hit != 'true' + run: | + .ci/fetch.sh -q -o llvm.sh https://apt.llvm.org/llvm.sh + chmod +x ./llvm.sh sudo ./llvm.sh 18 - echo "/usr/lib/llvm-18/bin" >> $GITHUB_PATH + + - name: Setup LLVM PATH + run: echo "/usr/lib/llvm-18/bin" >> $GITHUB_PATH shell: bash + - name: Install compiler id: install_cc uses: rlalik/setup-cpp-compiler@master @@ -75,7 +113,15 @@ jobs: sudo env TMP_FILE=\${TMP_FILE} BLK_DEV_EXT4=\${BLK_DEV_EXT4} \ BLK_DEV_SIMPLEFS=\${BLK_DEV_SIMPLEFS} .ci/boot-linux-prepare.sh cleanup; \ exit \${EXIT_CODE};" >> "$GITHUB_ENV" - - name: fetch artifact first to reduce HTTP requests + - name: Cache build artifacts + uses: actions/cache@v4 + with: + path: build/ + key: rv32emu-artifacts-${{ runner.os }}-${{ hashFiles('mk/artifact.mk', 'mk/external.mk') }} + restore-keys: | + rv32emu-artifacts-${{ runner.os }}- + + - name: Fetch artifacts env: CC: ${{ steps.install_cc.outputs.cc }} run: | @@ -83,118 +129,51 @@ jobs: make ENABLE_SYSTEM=1 artifact make ENABLE_ARCH_TEST=1 artifact # get from rv32emu-prebuilt - wget -O build/shareware_doom_iwad.zip "https://raw.githubusercontent.com/sysprog21/rv32emu-prebuilt/doom-artifact/shareware_doom_iwad.zip" - unzip -d build/ build/shareware_doom_iwad.zip - if: ${{ always() }} + .ci/fetch.sh -o build/shareware_doom_iwad.zip "https://raw.githubusercontent.com/sysprog21/rv32emu-prebuilt/doom-artifact/shareware_doom_iwad.zip" + unzip -o -d build/ build/shareware_doom_iwad.zip + - name: default build using emcc + if: success() run: | - make CC=emcc $PARALLEL - if: ${{ always() }} + make CC=emcc ENABLE_JIT=0 $PARALLEL + - name: default build for system emulation using emcc + if: success() run: | make distclean - make CC=emcc ENABLE_SYSTEM=1 $PARALLEL + make CC=emcc ENABLE_SYSTEM=1 ENABLE_JIT=0 $PARALLEL make distclean ENABLE_SYSTEM=1 - if: ${{ always() }} - - name: default build with -g - env: - CC: ${{ steps.install_cc.outputs.cc }} - run: | - make distclean - make OPT_LEVEL=-g $PARALLEL - if: ${{ always() }} - - name: default build with -Og - env: - CC: ${{ steps.install_cc.outputs.cc }} - run: | - make distclean - make OPT_LEVEL=-Og $PARALLEL - if: ${{ always() }} - - name: default build with -O0 - env: - CC: ${{ steps.install_cc.outputs.cc }} - run: | - make distclean - make OPT_LEVEL=-O0 $PARALLEL - if: ${{ always() }} - - name: default build with -O1 - env: - CC: ${{ steps.install_cc.outputs.cc }} - run: | - make distclean - make OPT_LEVEL=-O1 $PARALLEL - if: ${{ always() }} - - name: default build with -O2 - env: - CC: ${{ steps.install_cc.outputs.cc }} - run: | - make distclean - make OPT_LEVEL=-O2 $PARALLEL - if: ${{ always() }} - - name: default build with -O3 - env: - CC: ${{ steps.install_cc.outputs.cc }} - run: | - make distclean - make OPT_LEVEL=-O3 $PARALLEL - if: ${{ always() }} - - name: default build with -Ofast - env: - CC: ${{ steps.install_cc.outputs.cc }} - run: | - make distclean - make OPT_LEVEL=-Ofast $PARALLEL - if: ${{ always() }} - - name: default build for system emulation with -g - env: - CC: ${{ steps.install_cc.outputs.cc }} - run: | - make distclean - make OPT_LEVEL=-g ENABLE_SYSTEM=1 $PARALLEL - if: ${{ always() }} - - name: default build for system emulation with -Og - env: - CC: ${{ steps.install_cc.outputs.cc }} - run: | - make distclean - make OPT_LEVEL=-Og ENABLE_SYSTEM=1 $PARALLEL - if: ${{ always() }} - - name: default build for system emulation with -O0 - env: - CC: ${{ steps.install_cc.outputs.cc }} - run: | - make distclean - make OPT_LEVEL=-O0 ENABLE_SYSTEM=1 $PARALLEL - if: ${{ always() }} - - name: default build for system emulation with -O1 - env: - CC: ${{ steps.install_cc.outputs.cc }} - run: | - make distclean - make OPT_LEVEL=-O1 ENABLE_SYSTEM=1 $PARALLEL - if: ${{ always() }} - - name: default build for system emulation with -O2 - env: - CC: ${{ steps.install_cc.outputs.cc }} - run: | - make distclean - make OPT_LEVEL=-O2 ENABLE_SYSTEM=1 $PARALLEL - if: ${{ always() }} - - name: default build for system emulation with -O3 + + - name: Build with various optimization levels + if: success() env: CC: ${{ steps.install_cc.outputs.cc }} run: | - make distclean - make OPT_LEVEL=-O3 ENABLE_SYSTEM=1 $PARALLEL - if: ${{ always() }} - - name: default build for system emulation with -Ofast + set -euo pipefail + for opt_level in -g -Og -O0 -O1 -O2 -O3 -Ofast; do + echo "Building with OPT_LEVEL=$opt_level" + if ! (make distclean && make OPT_LEVEL=$opt_level $PARALLEL); then + echo "ERROR: Build failed with OPT_LEVEL=$opt_level" + exit 1 + fi + done + + - name: Build system emulation with various optimization levels + if: success() env: CC: ${{ steps.install_cc.outputs.cc }} run: | - make distclean - make OPT_LEVEL=-Ofast ENABLE_SYSTEM=1 $PARALLEL - if: ${{ always() }} + set -euo pipefail + for opt_level in -g -Og -O0 -O1 -O2 -O3 -Ofast; do + echo "Building system emulation with OPT_LEVEL=$opt_level" + if ! (make distclean && make OPT_LEVEL=$opt_level ENABLE_SYSTEM=1 $PARALLEL); then + echo "ERROR: System emulation build failed with OPT_LEVEL=$opt_level" + exit 1 + fi + done + - name: check + tests + if: success() env: CC: ${{ steps.install_cc.outputs.cc }} run: | @@ -203,108 +182,116 @@ jobs: make tests $PARALLEL make misalign $PARALLEL make tool $PARALLEL - if: ${{ always() }} + - name: diverse configurations + if: success() env: CC: ${{ steps.install_cc.outputs.cc }} run: | - make distclean && make ENABLE_EXT_M=0 check $PARALLEL - make distclean && make ENABLE_EXT_A=0 check $PARALLEL - make distclean && make ENABLE_EXT_F=0 check $PARALLEL - make distclean && make ENABLE_EXT_C=0 check $PARALLEL - make distclean && make ENABLE_SDL=0 check $PARALLEL - make distclean && make ENABLE_Zicsr=0 check $PARALLEL - make distclean && make ENABLE_MOP_FUSION=0 check $PARALLEL - make distclean && make ENABLE_BLOCK_CHAINING=0 check $PARALLEL - make distclean && make ENABLE_Zba=0 check $PARALLEL - make distclean && make ENABLE_Zbb=0 check $PARALLEL - make distclean && make ENABLE_Zbc=0 check $PARALLEL - make distclean && make ENABLE_Zbs=0 check $PARALLEL - make distclean && make ENABLE_Zifencei=0 check $PARALLEL - if: ${{ always() }} + set -euo pipefail + for config in ENABLE_EXT_M ENABLE_EXT_A ENABLE_EXT_F ENABLE_EXT_C \ + ENABLE_SDL ENABLE_Zicsr ENABLE_MOP_FUSION ENABLE_BLOCK_CHAINING \ + ENABLE_Zba ENABLE_Zbb ENABLE_Zbc ENABLE_Zbs ENABLE_Zifencei; do + echo "Testing with ${config}=0" + if ! (make distclean && make ${config}=0 check $PARALLEL); then + echo "ERROR: Test failed with ${config}=0" + exit 1 + fi + done + - name: misalignment test in block emulation + if: success() env: CC: ${{ steps.install_cc.outputs.cc }} run: | make -C tests/system/alignment/ make distclean && make ENABLE_ELF_LOADER=1 ENABLE_EXT_C=0 ENABLE_SYSTEM=1 misalign-in-blk-emu $PARALLEL - if: ${{ always() }} + - name: MMU test + if: success() env: CC: ${{ steps.install_cc.outputs.cc }} run: | make -C tests/system/mmu/ make distclean && make ENABLE_ELF_LOADER=1 ENABLE_SYSTEM=1 mmu-test $PARALLEL - if: ${{ always() }} + - name: gdbstub test + if: success() env: CC: ${{ steps.install_cc.outputs.cc }} run: | make distclean && make ENABLE_GDBSTUB=1 gdbstub-test $PARALLEL - if: ${{ always() }} + - name: JIT test + if: success() env: CC: ${{ steps.install_cc.outputs.cc }} run: | + set -euo pipefail + # Base JIT test make ENABLE_JIT=1 clean && make ENABLE_JIT=1 check $PARALLEL - make ENABLE_JIT=1 clean && make ENABLE_EXT_A=0 ENABLE_JIT=1 check $PARALLEL - make ENABLE_JIT=1 clean && make ENABLE_EXT_F=0 ENABLE_JIT=1 check $PARALLEL - make ENABLE_JIT=1 clean && make ENABLE_EXT_C=0 ENABLE_JIT=1 check $PARALLEL - make ENABLE_JIT=1 clean && make ENABLE_EXT_M=0 ENABLE_JIT=1 check $PARALLEL - make ENABLE_JIT=1 clean && make ENABLE_Zba=0 ENABLE_JIT=1 check $PARALLEL - make ENABLE_JIT=1 clean && make ENABLE_Zbb=0 ENABLE_JIT=1 check $PARALLEL - make ENABLE_JIT=1 clean && make ENABLE_Zbc=0 ENABLE_JIT=1 check $PARALLEL - make ENABLE_JIT=1 clean && make ENABLE_Zbs=0 ENABLE_JIT=1 check $PARALLEL - make ENABLE_JIT=1 clean && make ENABLE_Zicsr=0 ENABLE_JIT=1 check $PARALLEL - make ENABLE_JIT=1 clean && make ENABLE_Zifencei=0 ENABLE_JIT=1 check $PARALLEL - make ENABLE_JIT=1 clean && make ENABLE_MOP_FUSION=0 ENABLE_JIT=1 check $PARALLEL - make ENABLE_JIT=1 clean && make ENABLE_BLOCK_CHAINING=0 ENABLE_JIT=1 check $PARALLEL - if: ${{ always() }} + + # JIT tests with disabled extensions + for ext in ENABLE_EXT_A ENABLE_EXT_F ENABLE_EXT_C ENABLE_EXT_M \ + ENABLE_Zba ENABLE_Zbb ENABLE_Zbc ENABLE_Zbs \ + ENABLE_Zicsr ENABLE_Zifencei \ + ENABLE_MOP_FUSION ENABLE_BLOCK_CHAINING; do + echo "JIT test with ${ext}=0" + if ! (make ENABLE_JIT=1 clean && make ${ext}=0 ENABLE_JIT=1 check $PARALLEL); then + echo "ERROR: JIT test failed with ${ext}=0" + exit 1 + fi + done + - name: undefined behavior test + if: success() || failure() run: | make distclean && make ENABLE_UBSAN=1 check $PARALLEL make ENABLE_JIT=1 clean && make ENABLE_JIT=1 ENABLE_UBSAN=1 check $PARALLEL - if: ${{ always() }} + - name: boot Linux kernel test + if: success() env: CC: ${{ steps.install_cc.outputs.cc }} run: | make distclean && make INITRD_SIZE=32 ENABLE_SYSTEM=1 $PARALLEL && make ENABLE_SYSTEM=1 artifact $PARALLEL bash -c "${BOOT_LINUX_TEST}" make ENABLE_SYSTEM=1 clean - if: ${{ always() }} + - name: boot Linux kernel test (JIT) + if: success() env: CC: ${{ steps.install_cc.outputs.cc }} run: | make distclean && make INITRD_SIZE=32 ENABLE_SYSTEM=1 ENABLE_JIT=1 ENABLE_MOP_FUSION=0 $PARALLEL && make ENABLE_SYSTEM=1 artifact $PARALLEL bash -c "${BOOT_LINUX_TEST}" make ENABLE_SYSTEM=1 ENABLE_JIT=1 ENABLE_MOP_FUSION=0 clean - if: ${{ always() }} + - name: Architecture test + if: success() env: CC: ${{ steps.install_cc.outputs.cc }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | . .ci/common.sh export LATEST_RELEASE=$(download_with_headers "https://api.github.com/repos/sysprog21/rv32emu-prebuilt/releases" \ - "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ + "Authorization: Bearer ${GH_TOKEN}" \ | grep '"tag_name"' \ | grep "sail" \ | head -n 1 \ | sed -E 's/.*"tag_name": "([^"]+)".*/\1/') .ci/riscv-tests.sh - if: ${{ always() }} host-arm64: needs: [detect-code-related-file-changes] if: needs.detect-code-related-file-changes.outputs.has_code_related_changes == 'true' + timeout-minutes: 45 runs-on: ubuntu-24.04 steps: - name: checkout code uses: actions/checkout@v4 - - name: Set parallel jobs variable - run: | - echo "PARALLEL=-j$(nproc)" >> "$GITHUB_ENV" + with: + submodules: 'true' - name: build artifact # The GitHub Action for non-x86 CPU uses: uraimo/run-on-arch-action@v3 @@ -315,20 +302,25 @@ jobs: # No 'sudo' is available install: | apt update -qq - apt install -yqq make git clang libsdl2-dev libsdl2-mixer-dev lsb-release wget software-properties-common gnupg bc - git config --global --add safe.directory ${{ github.workspace }} - git config --global --add safe.directory ${{ github.workspace }}/src/softfloat - git config --global --add safe.directory ${{ github.workspace }}/src/mini-gdbstub - wget https://apt.llvm.org/llvm.sh - chmod +x ./llvm.sh - ./llvm.sh 18 + apt install -yqq make git curl wget clang libsdl2-dev libsdl2-mixer-dev lsb-release software-properties-common gnupg bc + which wget || echo "WARNING: wget not found after installation" # FIXME: gcc build fails on Aarch64/Linux hosts env: | CC: clang-18 - PARALLEL: ${{ env.PARALLEL }} # Append custom commands here run: | + # Verify and install wget if needed (workaround for install step issues) + if ! command -v wget > /dev/null 2>&1; then + apt update -qq && apt install -yqq wget + fi + git config --global --add safe.directory ${{ github.workspace }} + git config --global --add safe.directory ${{ github.workspace }}/src/softfloat + git config --global --add safe.directory ${{ github.workspace }}/src/mini-gdbstub + wget -O /tmp/llvm.sh https://apt.llvm.org/llvm.sh + chmod +x /tmp/llvm.sh + /tmp/llvm.sh 18 export PATH=/usr/lib/llvm-18/bin:$PATH + PARALLEL=-j$(nproc) make artifact make $PARALLEL make check $PARALLEL @@ -340,6 +332,7 @@ jobs: macOS-arm64: needs: [detect-code-related-file-changes] if: needs.detect-code-related-file-changes.outputs.has_code_related_changes == 'true' + timeout-minutes: 60 strategy: fail-fast: false matrix: @@ -347,11 +340,38 @@ jobs: runs-on: macos-latest # M1 chip steps: - uses: actions/checkout@v4 - - name: install-dependencies + with: + submodules: 'true' + + - name: Cache RISC-V Toolchain + id: cache-toolchain + uses: actions/cache@v4 + with: + path: ${{ github.workspace }}/toolchain + key: ${{ runner.os }}-${{ runner.arch }}-riscv-toolchain-${{ hashFiles('.ci/riscv-toolchain-install.sh') }} + + - name: Cache Homebrew packages + id: cache-brew + uses: actions/cache@v4 + with: + path: | + /opt/homebrew/Cellar/llvm@18 + /opt/homebrew/Cellar/sdl2 + /opt/homebrew/Cellar/dtc + key: ${{ runner.os }}-${{ runner.arch }}-brew-${{ hashFiles('.github/workflows/main.yml') }} + + - name: Install dependencies run: | brew install make dtc expect sdl2 bc e2fsprogs p7zip llvm@18 dcfldd brew install sdl2_mixer || echo "Warning: sdl2_mixer installation failed, continuing without SDL_MIXER support" + + - name: Install RISC-V Toolchain + if: steps.cache-toolchain.outputs.cache-hit != 'true' + run: | .ci/riscv-toolchain-install.sh + + - name: Setup toolchain paths + run: | echo "${{ github.workspace }}/toolchain/bin" >> $GITHUB_PATH echo "$(brew --prefix llvm@18)/bin" >> $GITHUB_PATH - name: Set up Python 3.12 @@ -384,45 +404,48 @@ jobs: - name: fetch artifact first to reduce HTTP requests env: CC: ${{ steps.install_cc.outputs.cc }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | . .ci/common.sh LATEST_RELEASE=$(download_with_headers "https://api.github.com/repos/sysprog21/rv32emu-prebuilt/releases" \ - "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ + "Authorization: Bearer ${GH_TOKEN}" \ | grep '"tag_name"' \ | grep "ELF" \ | head -n 1 \ | sed -E 's/.*"tag_name": "([^"]+)".*/\1/') make LATEST_RELEASE=$LATEST_RELEASE artifact LATEST_RELEASE=$(download_with_headers "https://api.github.com/repos/sysprog21/rv32emu-prebuilt/releases" \ - "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ + "Authorization: Bearer ${GH_TOKEN}" \ | grep '"tag_name"' \ | grep "Linux-Image" \ | head -n 1 \ | sed -E 's/.*"tag_name": "([^"]+)".*/\1/') make LATEST_RELEASE=$LATEST_RELEASE ENABLE_SYSTEM=1 artifact LATEST_RELEASE=$(download_with_headers "https://api.github.com/repos/sysprog21/rv32emu-prebuilt/releases" \ - "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ + "Authorization: Bearer ${GH_TOKEN}" \ | grep '"tag_name"' \ | grep "sail" \ | head -n 1 \ | sed -E 's/.*"tag_name": "([^"]+)".*/\1/') make LATEST_RELEASE=$LATEST_RELEASE ENABLE_ARCH_TEST=1 artifact # get from rv32emu-prebuilt - download_to_file "https://raw.githubusercontent.com/sysprog21/rv32emu-prebuilt/doom-artifact/shareware_doom_iwad.zip" \ - "build/shareware_doom_iwad.zip" - unzip -d build/ build/shareware_doom_iwad.zip - if: ${{ always() }} + .ci/fetch.sh -o build/shareware_doom_iwad.zip "https://raw.githubusercontent.com/sysprog21/rv32emu-prebuilt/doom-artifact/shareware_doom_iwad.zip" + unzip -o -d build/ build/shareware_doom_iwad.zip + - name: default build using emcc + if: success() run: | - make CC=emcc $PARALLEL - if: ${{ always() }} + make CC=emcc ENABLE_JIT=0 $PARALLEL + - name: default build for system emulation using emcc + if: success() run: | make distclean - make CC=emcc ENABLE_SYSTEM=1 $PARALLEL + make CC=emcc ENABLE_SYSTEM=1 ENABLE_JIT=0 $PARALLEL make distclean ENABLE_SYSTEM=1 - if: ${{ always() }} + - name: check + tests + if: success() env: CC: ${{ steps.install_cc.outputs.cc }} run: | @@ -431,57 +454,61 @@ jobs: make tests $PARALLEL make misalign $PARALLEL make tool $PARALLEL - if: ${{ always() }} + - name: diverse configurations + if: success() env: CC: ${{ steps.install_cc.outputs.cc }} run: | - make distclean && make ENABLE_EXT_M=0 check $PARALLEL - make distclean && make ENABLE_EXT_A=0 check $PARALLEL - make distclean && make ENABLE_EXT_F=0 check $PARALLEL - make distclean && make ENABLE_EXT_C=0 check $PARALLEL - make distclean && make ENABLE_SDL=0 check $PARALLEL - make distclean && make ENABLE_Zicsr=0 check $PARALLEL - make distclean && make ENABLE_MOP_FUSION=0 check $PARALLEL - make distclean && make ENABLE_BLOCK_CHAINING=0 check $PARALLEL - make distclean && make ENABLE_Zba=0 check $PARALLEL - make distclean && make ENABLE_Zbb=0 check $PARALLEL - make distclean && make ENABLE_Zbc=0 check $PARALLEL - make distclean && make ENABLE_Zbs=0 check $PARALLEL - make distclean && make ENABLE_Zifencei=0 check $PARALLEL - if: ${{ always() }} + set -euo pipefail + for config in ENABLE_EXT_M ENABLE_EXT_A ENABLE_EXT_F ENABLE_EXT_C \ + ENABLE_SDL ENABLE_Zicsr ENABLE_MOP_FUSION ENABLE_BLOCK_CHAINING \ + ENABLE_Zba ENABLE_Zbb ENABLE_Zbc ENABLE_Zbs ENABLE_Zifencei; do + echo "Testing with ${config}=0" + if ! (make distclean && make ${config}=0 check $PARALLEL); then + echo "ERROR: Test failed with ${config}=0" + exit 1 + fi + done + - name: gdbstub test, need RV32 toolchain + if: success() env: CC: ${{ steps.install_cc.outputs.cc }} run: | make distclean && make ENABLE_GDBSTUB=1 gdbstub-test $PARALLEL - if: ${{ always() }} + - name: JIT test + if: success() env: CC: ${{ steps.install_cc.outputs.cc }} run: | + set -euo pipefail + # Base JIT test make ENABLE_JIT=1 clean && make ENABLE_JIT=1 check $PARALLEL - make ENABLE_JIT=1 clean && make ENABLE_EXT_A=0 ENABLE_JIT=1 check $PARALLEL - make ENABLE_JIT=1 clean && make ENABLE_EXT_F=0 ENABLE_JIT=1 check $PARALLEL - make ENABLE_JIT=1 clean && make ENABLE_EXT_C=0 ENABLE_JIT=1 check $PARALLEL - make ENABLE_JIT=1 clean && make ENABLE_EXT_M=0 ENABLE_JIT=1 check $PARALLEL - make ENABLE_JIT=1 clean && make ENABLE_Zba=0 ENABLE_JIT=1 check $PARALLEL - make ENABLE_JIT=1 clean && make ENABLE_Zbb=0 ENABLE_JIT=1 check $PARALLEL - make ENABLE_JIT=1 clean && make ENABLE_Zbc=0 ENABLE_JIT=1 check $PARALLEL - make ENABLE_JIT=1 clean && make ENABLE_Zbs=0 ENABLE_JIT=1 check $PARALLEL - make ENABLE_JIT=1 clean && make ENABLE_Zicsr=0 ENABLE_JIT=1 check $PARALLEL - make ENABLE_JIT=1 clean && make ENABLE_Zifencei=0 ENABLE_JIT=1 check $PARALLEL - make ENABLE_JIT=1 clean && make ENABLE_MOP_FUSION=0 ENABLE_JIT=1 check $PARALLEL - make ENABLE_JIT=1 clean && make ENABLE_BLOCK_CHAINING=0 ENABLE_JIT=1 check $PARALLEL - if: ${{ always() }} + + # JIT tests with disabled extensions + for ext in ENABLE_EXT_A ENABLE_EXT_F ENABLE_EXT_C ENABLE_EXT_M \ + ENABLE_Zba ENABLE_Zbb ENABLE_Zbc ENABLE_Zbs \ + ENABLE_Zicsr ENABLE_Zifencei \ + ENABLE_MOP_FUSION ENABLE_BLOCK_CHAINING; do + echo "JIT test with ${ext}=0" + if ! (make ENABLE_JIT=1 clean && make ${ext}=0 ENABLE_JIT=1 check $PARALLEL); then + echo "ERROR: JIT test failed with ${ext}=0" + exit 1 + fi + done + - name: undefined behavior test + if: (success() || failure()) && steps.install_cc.outputs.cc == 'clang' # gcc on macOS/arm64 does not support sanitizers env: CC: ${{ steps.install_cc.outputs.cc }} - if: ${{ env.CC == 'clang' && always() }} # gcc on macOS/arm64 does not support satinizers run: | make distclean && make ENABLE_UBSAN=1 check $PARALLEL make ENABLE_JIT=1 clean && make ENABLE_JIT=1 ENABLE_UBSAN=1 check $PARALLEL + - name: boot Linux kernel test + if: success() env: CC: ${{ steps.install_cc.outputs.cc }} run: | @@ -489,22 +516,25 @@ jobs: make ENABLE_SYSTEM=1 artifact $PARALLEL bash -c "${BOOT_LINUX_TEST}" make ENABLE_SYSTEM=1 clean - if: ${{ always() }} + - name: boot Linux kernel test (JIT) + if: success() env: CC: ${{ steps.install_cc.outputs.cc }} run: | make distclean && make INITRD_SIZE=32 ENABLE_SYSTEM=1 ENABLE_JIT=1 ENABLE_MOP_FUSION=0 $PARALLEL && make ENABLE_SYSTEM=1 artifact $PARALLEL bash -c "${BOOT_LINUX_TEST}" make ENABLE_SYSTEM=1 ENABLE_JIT=1 ENABLE_MOP_FUSION=0 clean - if: ${{ always() }} + - name: Architecture test + if: success() env: CC: ${{ steps.install_cc.outputs.cc }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | . .ci/common.sh export LATEST_RELEASE=$(download_with_headers "https://api.github.com/repos/sysprog21/rv32emu-prebuilt/releases" \ - "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ + "Authorization: Bearer ${GH_TOKEN}" \ | grep '"tag_name"' \ | grep "sail" \ | head -n 1 \ @@ -512,11 +542,11 @@ jobs: python3 -m venv venv . venv/bin/activate .ci/riscv-tests.sh - if: ${{ always() }} coding-style: needs: [detect-code-related-file-changes] if: needs.detect-code-related-file-changes.outputs.has_code_related_changes == 'true' + timeout-minutes: 15 runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 @@ -531,15 +561,18 @@ jobs: static-analysis: needs: [detect-code-related-file-changes] if: needs.detect-code-related-file-changes.outputs.has_code_related_changes == 'true' - runs-on: ubuntu-latest + timeout-minutes: 45 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 + with: + submodules: 'true' # LLVM static analysis - name: set up scan-build run: | sudo apt-get update -q=2 - sudo apt-get install -q=2 libsdl2-dev libsdl2-mixer-dev - wget https://apt.llvm.org/llvm.sh + sudo apt-get install -q=2 curl libsdl2-dev libsdl2-mixer-dev + .ci/fetch.sh -q -o llvm.sh https://apt.llvm.org/llvm.sh chmod +x ./llvm.sh sudo ./llvm.sh 18 sudo apt-get install -q=2 clang-18 clang-tools-18 @@ -558,27 +591,30 @@ jobs: docker-hub-build-and-publish: needs: [detect-code-related-file-changes] if: needs.detect-code-related-file-changes.outputs.has_code_related_changes == 'true' + timeout-minutes: 60 runs-on: ubuntu-24.04 steps: - name: Check out the repo uses: actions/checkout@v4 + with: + submodules: 'true' - name: Set up QEMU uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Login to Docker Hub uses: docker/login-action@v3 - if: ${{ github.event_name == 'push'}} + if: github.event_name == 'push' with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} - name: Get short commit SHA1 - if: ${{ github.event_name == 'push'}} + if: github.event_name == 'push' shell: bash run: | echo "short_hash=$(git rev-parse --short "$GITHUB_SHA")" >> "$GITHUB_ENV" - name: Build and push - if: ${{ github.event_name == 'push'}} + if: github.event_name == 'push' uses: docker/build-push-action@v6 with: push: true