diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000000..a663805362 --- /dev/null +++ b/.clang-format @@ -0,0 +1,136 @@ +--- +Language: Cpp +# BasedOnStyle: LLVM +AccessModifierOffset: -2 +AlignAfterOpenBracket: Align +AlignConsecutiveMacros: true +AlignConsecutiveAssignments: true +AlignConsecutiveDeclarations: false +AlignEscapedNewlines: Left +AlignOperands: false +AlignTrailingComments: true +AllowAllArgumentsOnNextLine: true +AllowAllConstructorInitializersOnNextLine: true +AllowAllParametersOfDeclarationOnNextLine: true +AllowShortBlocksOnASingleLine: Always +AllowShortCaseLabelsOnASingleLine: true +AllowShortFunctionsOnASingleLine: All +AllowShortLambdasOnASingleLine: All +AllowShortIfStatementsOnASingleLine: Always +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterDefinitionReturnType: None +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: false +AlwaysBreakTemplateDeclarations: MultiLine +BinPackArguments: true +BinPackParameters: true +BraceWrapping: + AfterCaseLabel: false + AfterClass: false + AfterControlStatement: false + AfterEnum: false + AfterFunction: true + AfterNamespace: false + AfterObjCDeclaration: false + AfterStruct: false + AfterUnion: false + AfterExternBlock: false + BeforeCatch: true + BeforeElse: true + IndentBraces: false + SplitEmptyFunction: true + SplitEmptyRecord: true + SplitEmptyNamespace: true +BreakBeforeBinaryOperators: None +BreakBeforeBraces: Stroustrup +BreakBeforeInheritanceComma: false +BreakInheritanceList: AfterColon +BreakBeforeTernaryOperators: false +BreakConstructorInitializersBeforeComma: false +BreakConstructorInitializers: AfterColon +BreakAfterJavaFieldAnnotations: false +BreakStringLiterals: true +ColumnLimit: 100 +CommentPragmas: '^ IWYU pragma:' +CompactNamespaces: false +ConstructorInitializerAllOnOneLineOrOnePerLine: false +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: true +DeriveLineEnding: true +DerivePointerAlignment: false +DisableFormat: false +ExperimentalAutoDetectBinPacking: false +FixNamespaceComments: true +ForEachMacros: + - foreach + - Q_FOREACH + - BOOST_FOREACH +IncludeBlocks: Preserve +IncludeCategories: + - Regex: '^"(llvm|llvm-c|clang|clang-c)/' + Priority: 2 + SortPriority: 0 + - Regex: '^(<|"(gtest|gmock|isl|json)/)' + Priority: 3 + SortPriority: 0 + - Regex: '.*' + Priority: 1 + SortPriority: 0 +IncludeIsMainRegex: '(Test)?$' +IncludeIsMainSourceRegex: '' +IndentCaseLabels: true +IndentGotoLabels: false +IndentPPDirectives: BeforeHash +IndentWidth: 4 +IndentWrappedFunctionNames: true +JavaScriptQuotes: Leave +JavaScriptWrapImports: true +KeepEmptyLinesAtTheStartOfBlocks: true +MacroBlockBegin: '' +MacroBlockEnd: '' +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: None +ObjCBinPackProtocolList: Auto +ObjCBlockIndentWidth: 2 +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: true +PenaltyBreakAssignment: 2 +PenaltyBreakBeforeFirstCallParameter: 19 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakString: 1000 +PenaltyBreakTemplateDeclaration: 10 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 60 +PointerAlignment: Right +ReflowComments: true +SortIncludes: true +SortUsingDeclarations: true +SpaceAfterCStyleCast: false +SpaceAfterLogicalNot: false +SpaceAfterTemplateKeyword: true +SpaceBeforeAssignmentOperators: true +SpaceBeforeCpp11BracedList: false +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true +SpaceBeforeParens: ControlStatements +SpaceBeforeRangeBasedForLoopColon: true +SpaceInEmptyBlock: false +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 1 +SpacesInAngles: false +SpacesInConditionalStatement: false +SpacesInContainerLiterals: true +SpacesInCStyleCastParentheses: false +SpacesInParentheses: false +SpacesInSquareBrackets: false +SpaceBeforeSquareBrackets: false +Standard: Latest +StatementMacros: + - Q_UNUSED + - QT_REQUIRE_VERSION +TabWidth: 4 +UseCRLF: false +UseTab: Never +... \ No newline at end of file diff --git a/.githooks/pre-push b/.githooks/pre-push new file mode 100644 index 0000000000..0b9125c70a --- /dev/null +++ b/.githooks/pre-push @@ -0,0 +1,22 @@ +#!/bin/bash + +FORMAT_SCRIPT="$HOME/esp/utils/scripts/actions-pipeline/code-format/format_modified.sh" + +# Check if the formatting script exists +if [ ! -f "$FORMAT_SCRIPT" ]; then + echo "Error: Formatting script not found!" + exit 1 +fi + +cd "$(dirname "$FORMAT_SCRIPT")" || exit + +chmod +x "$(basename "$FORMAT_SCRIPT")" + +if ! ./$(basename "$FORMAT_SCRIPT") -g -ca; then + echo "Code format check failed." + echo "Fix the formatting issues by running the following script: $FORMAT_SCRIPT" + exit 1 +fi + +# If the format check passes, allow the push to proceed +exit 0 \ No newline at end of file diff --git a/.githooks/setup.sh b/.githooks/setup.sh new file mode 100644 index 0000000000..8260a8a3f3 --- /dev/null +++ b/.githooks/setup.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# Copy pre-push hook script to .git/hooks directory +cp .githooks/pre-push .git/hooks/pre-push +chmod +x .git/hooks/pre-push + +echo "Pre-push hook activated successfully." \ No newline at end of file diff --git a/.github/workflows/regression-test.yaml b/.github/workflows/regression-test.yaml new file mode 100644 index 0000000000..08cc48219a --- /dev/null +++ b/.github/workflows/regression-test.yaml @@ -0,0 +1,86 @@ +name: Regression Test + +on: + pull_request: + branches: + - dev + - main + - master + +jobs: + code-format-check: + name: Code Format Check + runs-on: ubuntu-latest + + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + + container: + image: columbiasld/esp:ubuntu18-small + credentials: + username: ${{ env.DOCKER_USERNAME }} + password: ${{ env.DOCKER_PASSWORD }} + + steps: + - name: Checkout code + uses: actions/checkout@v1 + + - name: Install vsg + run: | + pip3 install vsg + echo 'export PATH="$PATH:/home/espuser/.local/bin/"' >> ~/.bashrc + + - name: Install clang-format-10 + run: | + sudo apt-get update + sudo apt-get install -y clang-format-10 + + - name: Install autopep8 + run: | + pip3 install autopep8 + + - name: Install Verible + run: | + wget https://github.com/chipsalliance/verible/releases/download/v0.0-3545-ge4028f19/verible-v0.0-3545-ge4028f19-linux-static-x86_64.tar.gz + tar -xvf verible-v0.0-3545-ge4028f19-linux-static-x86_64.tar.gz + rm verible-v0.0-3545-ge4028f19-linux-static-x86_64.tar.gz + mv verible-v0.0-3545-ge4028f19/ verible + export PATH=$PATH:/home/espuser/verible/bin + echo 'export PATH="$PATH:/home/espuser/verible/bin"' >> ~/.bashrc + + - name: Source .bashrc + run: | + . ~/.bashrc + + - name: Set directory ownership + run: sudo chown -R $USER:$USER $GITHUB_WORKSPACE + + - name: Configure Git + run: git config --global --add safe.directory $GITHUB_WORKSPACE + + - name: Run code formatting check + run: | + cd $GITHUB_WORKSPACE/utils/scripts/actions-pipeline/code-format + chmod +x format_modified.sh + if ! ./format_modified.sh -g -ca; then + echo "Code format check failed." + echo "Fix the formatting issues by running the following script: /utils/scripts/code-format/format_modified.sh" + exit 1 + fi + regression: + name: Regression Test + runs-on: self-hosted + steps: + - name: Discover modified accelerators + run: | + cd $HOME/esp/utils/scripts/actions-pipeline + ./get_modified_accelerators.sh + - name: Run HLS + run: | + cd $HOME/esp/utils/scripts/actions-pipeline + ./run_sims.sh + - name: Generate bitstream and program FPGA + run: | + cd $HOME/esp/utils/scripts/actions-pipeline + ./run_esp-config.sh \ No newline at end of file diff --git a/utils/scripts/actions-pipeline/README.md b/utils/scripts/actions-pipeline/README.md new file mode 100644 index 0000000000..4be28a0c60 --- /dev/null +++ b/utils/scripts/actions-pipeline/README.md @@ -0,0 +1,146 @@ +# ESP Linting and Regression Testing Workflows +The scripts included in this directory are used as part of a GitHub Actions workflow triggered on PRs to the dev, main or master branches. +Scripts under the /code-format directory are used for linting purposes, while the other scripts are used for regression testing. + +## Scripts +### Code Formatting +There are two scripts under [./code-format](https://github.com/marianabuhazi/esp/blob/regression-flow/utils/scripts/actions-pipeline/code-format/) that can be used by contributors to check their code for formatting violations, as well as to format code in place. These scripts could also be used as part of a Git hook or a GitHub Actions workflow. +The scripts support formatting for the following languages: C/C++, Verilog/SystemVerilog, VHDL, and Python. + +#### Format Newly-Modified Files +[format_modified.sh](https://github.com/marianabuhazi/esp/blob/regression-flow/utils/scripts/actions-pipeline/code-format/format_modified.sh) is a bash script that examines the Git status of the remote repository to identify files that the user has recently modified, added, or deleted. It then determines the file extension type (e.g., C/C++, VHDL, SystemVerilog/Verilog, or Python) and invokes an open-source linting tool for that language with the filename and other necessary arguments. The script offers flexibility through various flags, allowing users to report programming violations, as well as, fix formatting in-place. This tool plays a crucial role in maintaining ESP's readability, consistency, and bug-free nature. +``` +ESP format checker ✨🛠️ +Report violations or format files in-place. +Usage: ./format_modified [OPTIONS] + -h Display this help message + -f Fix formatting for file + -c Check formatting for file + -a Apply to all + -g Run as Github Actions workflow or pre-push hook +``` + +#### Format ESP Repo +The [format_modified.sh](https://github.com/marianabuhazi/esp/blob/regression-flow/utils/scripts/actions-pipeline/code-format/format_modified.sh) script is useful for formatting new modifications to existing code. However, this implies that the code in the repository is already properly structured to begin with. In reality, our ESP repository lacks uniformity. To introduce formatting into the existing code, we have provided another script called [format_repo.sh](https://github.com/marianabuhazi/esp/blob/regression-flow/utils/scripts/actions-pipeline/code-format/format_repo.sh) which traverses through all directories in the ESP repository recursively, skipping over any directories that are Git submodules owned by another organization. The script identifies the file extension of each file within the current directory and formats it in-place by calling the appropriate open-source linter. +format_repo.sh is intended for one-time use. Additionally, it can significantly impact Git blame, potentially attributing all changes to a single contributor (whoever ran the script). To mitigate this, changes made by format_repo.sh should be committed separately and excluded from the blame. +``` +ESP format checker ✨🛠️ +Recursively format all files in the /esp repository for a given file extension. +Usage: [-t {c, cpp, vhdl, v, py}] +Options: + -h, --help Display this help message. + -t, --type {c, cpp, vhdl, v, py} Specify the type of files to format. +``` + +### Git Blame +After processing the Pull Request, it's important to consider what commits we want to exclude from the Git Blame. In particular, it should be any commits that have modified/linted tracked files in bulk. +To make this process easier, I have committed all the linting changes in 4 different commits in the `all-linted-files` branch. +These commit ids can be added to a file which can be excluded from the blame. [This tutorial](https://www.stefanjudis.com/today-i-learned/how-to-exclude-commits-from-git-blame/) is helpful in the process of excluding from the Git Blame. + +image + + +### Regression Testing + +#### Get Modified Accelerators +This script accepts a configuration file called accelerators.json and uses the repository's Git history to determine modified accelerators. The configuration file is simply a JSON file where ESP members can specify what accelerators they wish to test. Imagine that a user makes modifications to many accelerators in a single Pull Request. We may only care about testing a subset of these at a given time. +By default the get_modified_accelerators.sh scripts can run end-to-end tests with each modified accelerator in a 2x2 SoC, but having the configuration file adds extra flexibility to define/override which accelerators are to be tested. + +The script compares the modified files (as specified in the Git history) to the accelerators in the configuration file. It then returns a list of the overlap of accelerators that should be tested, along with references to their HLS and behavioral `make` commands (ie. `make cholesky_stratus-hls` or `make cholesky_stratus-beh`). + +#### Run Simulations +The Run Simulations script accepts a list of testable accelerators via a bash list, and runs HLS for each one. The script first assesses how many accelerators have been scheduled for synthesis. +We consider that in reality a user may only modify one or two accelerators at a time, but designed this project such that if many accelerators are modified in one Pull Request +the program can balance them all effectively across different CPUs on the SOCP server and execute them in parallel. The script iterates through the list of accelerators +and starts and HLS run for each one, asynchronously. Then, it waits for all HLS jobs to return, logging the results for each job as they complete. Once all jobs have +completed, the script exits with success or error code. + +#### Run ESP Config +After the Run Simulations scripts executes, the Run ESP Config script should be executed. Run ESP Config is a bash script that completes the bitstream generation, +FPGA programming, and the test program execution on one of the ESP Xilinx FPGAs. Run ESP Config first looks through the tech/acc folder in the working repository. +This directory has all the work folders for the completed HLS jobs. The script iterates through all the successful accelerator HLS jobs and creates an ESP 2x2 SoC configuration +with that accelerator in the accelerator tile. The script then proceeds to run `make esp-config` and `make vivado-syn` to generate the HDL and bitstream for the +full SoC configuration. If this succeeds, the script will continue to run `make fpga-program` to program the generated bitstream on the FPGA, and execute a test +program using `make fpga-run`. The script opens up a minicom connection to the FPGA simultaneously to running `make fpga-run`. The results of the test program +are read out from the minicom to a log file for users to refer to after completion. + +If more than one accelerator has been modified, the script will start a new end-to-end test by creating the ESP configuration, generating HDL, generating the bitstream, +configuring the FPGA and running the test program with the next accelerator, and so on. + + +## Setting Up Pre-push Hook + +To ensure code formatting compliance, we provide a pre-push Git hook that runs before you push changes to the repository. Follow these steps to set it up: + +1. Navigate to the root directory of the repository. +2. Run the setup script to activate the pre-push hook: + ```bash + .githooks/./setup.sh + ``` + or + ```bash + bash .githooks/setup.sh + ``` + This script will automatically copy the pre-push hook to the appropriate location. + +Now, every time you try to push changes to the repository, the pre-push hook will run automatically to check code formatting compliance. + + +## Setting Up Self-Hosted GitHub Runners +GitHub Actions is a tool integrated into the ESP GitHub repository that consists of YAML workflows that can be triggered by different git operations such as push or pull request. +Normally, GitHub Actions is executed on a GitHub-hosted server and can be operated at no-cost with a limit on the number of workflow runs. +This is useful for many applications but for ESP, we have a variety of licensed tools that are used for HLS and other procedures which are not easily accessed via GitHub-hosted servers. +Instead, it makes sense for ESP to run regression workflows on the SOCP servers for more control and flexibility. For this, GitHub offers self-hosted runners which are easily configurable +via a downloadable executable program. The resources below explain how a self-hosted runner can be created and configured for ESP. + +For testing, I set up the actions-runner program under the ma4107 user in socp01, then executed the `run.sh` script within the project to enable the tool to listen for GitHub Actions +jobs. + +### Resources +- [Adding self-hosted runners to the repository](https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners/adding-self-hosted-runners) +- [Configuring and starting the runner](https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners/adding-self-hosted-runners) + + +## Writing GitHub Actions YAML workflows +YAML workflows execute sccripts upon a push or pull request to a repository's branch. There is one GitHub Actions YAML workflow under [.github/workflows/regresstion-test.yaml](https://github.com/marianabuhazi/esp/blob/regression-flow/.github/workflows/regression-test.yaml) that +sets up the environment to run the tests (columbia-sld's ubuntu-small Docker image) and installs all the open-source linters. Then, it executes code formatting and testing scripts. + +### Installing open-source tools +#### vhdl-style-guide +``` +pip3 install vsg +export PATH="$PATH:/home/espuser/.local/bin/" +source ~/.bashrc +``` + +#### clang-format-10 +``` +sudo apt-get install clang-format-10 +``` + +#### autopep8 +``` +pip3 install autopep8 +``` + +#### verible +``` +wget https://github.com/chipsalliance/verible/releases/download/v0.0-3545-ge4028f19/verible-v0.0-3545-ge4028f19-linux-static-x86_64.tar.gz +tar -xvf verible-v0.0-3545-ge4028f19-linux-static-x86_64.tar.gz +rm verible-v0.0-3545-ge4028f19-linux-static-x86_64.tar.gz +mv verible-v0.0-3545-ge4028f19/ verible +export PATH=$PATH:/home/espuser/verible/bin +source ~/.bashrc +``` + +For more information on writing YAML workflows that are compatible with GitHub Actions, and documentation on these open-source linters check out the resources below. + +### Resources +- [Writing on pull_request workflows](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions) +- [vhdl-style-guide docs](https://vhdl-style-guide.readthedocs.io/en/latest/) +- [clang-format-10 docs](https://releases.llvm.org/10.0.0/tools/clang/docs/ClangFormat.html) +- [autopep8 docs](https://pypi.org/project/autopep8/) +- [verible repo + docs](https://github.com/chipsalliance/verible) + +### Disclaimer +Surely, when these scripts and workflows get integrated into the ESP repository they will need to be modified to match the expectations and demands of the team. There is also lots of room for improvement and adding more advanced linting and testing features! Please contact Marian Abuhazi at ma4107@columbia.edu if you have more questions or want to brainstorm more! Happy to help 🤗 diff --git a/utils/scripts/actions-pipeline/accelerators.json b/utils/scripts/actions-pipeline/accelerators.json new file mode 100644 index 0000000000..ef0dde15f1 --- /dev/null +++ b/utils/scripts/actions-pipeline/accelerators.json @@ -0,0 +1,10 @@ +{ + "accelerators": [ + { + "name": "dummy_stratus", + "path": "accelerators/stratus_hls/dummy_stratus", + "behavioral": "dummy_stratus-exe", + "hls": "dummy_stratus-hls" + } + ] +} \ No newline at end of file diff --git a/utils/scripts/actions-pipeline/code-format/format_modified.sh b/utils/scripts/actions-pipeline/code-format/format_modified.sh new file mode 100644 index 0000000000..2035cf520d --- /dev/null +++ b/utils/scripts/actions-pipeline/code-format/format_modified.sh @@ -0,0 +1,213 @@ +#!/bin/bash + +root_dir=$(git rev-parse --show-toplevel) +is_github_actions=false + +# Output styles +NC='\033[0m' +BOLD='\033[1m' +GREEN='\033[32m' +RED='\033[31m' + +# Display usage instructions +usage() { + echo "ESP format checker ✨🛠️" + echo "Report violations or format files in-place." + echo "" + echo "Usage: $0 [OPTIONS]" + echo " -h Display this help message" + echo "" + echo " -f Fix formatting for file " + echo " -c Check formatting for file " + echo " -a Apply to all" + echo " -g Run as Github Actions workflow or pre-push hook" + echo "" + echo "Examples:" + echo " $0 -fa # Fix all modified files in-place" + echo " $0 -f myfile.py # Fix myfile.py in-place" + echo " $0 -ca # Report violations for all modified files" + echo " $0 -c myfile.py # Report violations for myfile.py" + echo " $0 -g -ca # Report violations as part of a workflow or hook" + exit 1 +} + +format_file() { + local file_to_format="$1" + local action="$2" + local type="${file_to_format##*.}" + + # Parse file extension type and decide which formatting tool to use + # Add the corresponding flags based on whether we are formatting or checking + case "$action" in + Formatt) + case "$type" in + c | h | cpp | hpp) + clang_format_edit="-i" ;; + py) + autopep8_edit="-i" ;; + sv | v) + verible_edit="--inplace" ;; + vhd) + vsg_edit="--fix" ;; + esac + ;; + Check) + case "$type" in + c | h | cpp | hpp) + clang_format_edit="--dry-run" ;; + py) + autopep8_edit="--list-fixes" ;; + sv | v) + verible_edit="--verify";; + vhd) + ;; + esac + ;; + *) + echo "Unknown action: $action" >&2 + usage + ;; +esac + + # Apply formatting tool based on file extension type + local output + case "$type" in + c | h | cpp | hpp) + # Format with clang-format-10 + output=$(clang-format-10 $clang_format_edit "$file_to_format" 2>&1);; + py) + # Format with autopep8 + output=$(python3 -m autopep8 $autopep8_edit -a -a "$file_to_format" 2>&1);; + sv | v) + # Format with verible + output=$(verible-verilog-format $verible_edit --port_declarations_alignment=preserve -assignment_statement_alignment=align --indentation_spaces=4 "$file_to_format" 2>&1) ;; + vhd) + # Format with vhdl-style-guide + output=$(vsg -f "$file_to_format" $vsg_edit -c ~/esp/vhdl-style-guide.yaml 2>&1) ;; + esac + + # If no errors were encountered while formatting, return SUCCESS + # Else return FAILED + if [ $? -eq 0 ] && ! echo "$output" | grep -qi "warning"; then + echo -e "${GREEN}SUCCESS${NC}" + return 0 + else + echo -e "${RED}FAILED${NC}" + echo "$output" | sed 's/^/ /' + echo "" + return 1 + fi +} + +# Based on the command-line flags, determine whether to format in-place or check +# Call the appropriate action +while [[ $# -gt 0 ]]; do + key="$1" + + case $key in + -f) + if [[ -z $2 || $2 == -* ]]; then + echo "Option -f requires an argument." >&2 + usage + fi + action="Formatt" + file_to_format="$2" + shift + ;; + -fa) + action="Formatt" + all_files=true + ;; + -c) + if [[ -z $2 || $2 == -* ]]; then + echo "Option -c requires an argument." >&2 + usage + fi + action="Check" + file_to_format="$2" + shift + ;; + -ca) + action="Check" + all_files=true + ;; + -g) + is_github_actions=true + ;; + -h|--help) + usage + ;; + *) + echo "Unknown option: $1" >&2 + usage + ;; + esac + shift +done + +# User must specify a file to format if the -f flag is called +if [ -n "$file_to_format" ]; then + if [ ! -f "$file_to_format" ]; then + echo "$0: Error: File '$file_to_format' not found." >&2 + exit 1 + fi + + echo -n "$action""ing $file_to_format..." + if format_file "$file_to_format" "$action"; then + echo -e "✨ $action""ing done!" + exit 0 + else + echo -e "❌ $action""ing failed!" + exit 1 + fi + + exit 0 +fi + + +# Check all modified files of the file types: C/C++, Python, Verilog/SystemVerilog, VHDL +if [ "$all_files" = true ]; then + modified_files=$(git status --porcelain | grep -E '^ M|^??' | awk '$2 ~ /\.(c|h|cpp|hpp|py|v|sv|vhd)$/ {print $2}') + + if [ "$is_github_actions" = true ]; then + modified_files=$(git diff --name-only HEAD^..HEAD | grep -E '\.(c|h|cpp|hpp|py|v|sv|vhd)$') + fi + + if [ -z "$modified_files" ]; then + echo -e "🔍 No modified files found." + exit 0 + fi + + # List the number of modified files found + modified_count=$(echo "$modified_files" | wc -l) + echo -e "🔍 Found $modified_count modified files:" + echo "" + for file in $modified_files; do + echo " - $file" + done + echo "" + modified_files=$(echo "$modified_files" | sed "s|^|$root_dir/|") + + error_files="" + success_files="" + + # Format/check each file in place with the appropriate tool + for file in $modified_files; do + echo -n "$action""ing $(basename "$file")..." + if ! format_file "$file" "$action"; then + error_files="$error_files $file" + fi + done + + # Final pass/fail directive + echo "" + if [ -n "$error_files" ]; then + echo -e "❌ $action""ing failed!" + exit 1 + else + echo -e "✨ $action""ing done!" + exit 0 + fi +else + usage +fi \ No newline at end of file diff --git a/utils/scripts/actions-pipeline/code-format/format_repo.sh b/utils/scripts/actions-pipeline/code-format/format_repo.sh new file mode 100644 index 0000000000..0954d05941 --- /dev/null +++ b/utils/scripts/actions-pipeline/code-format/format_repo.sh @@ -0,0 +1,232 @@ +#!/bin/bash +# # # # # # # # # # # # # # # # # # # # # # # # # # # # +# This program recurses through all directories in # +# the /esp repository and formats all files using # +# clang-format-10, autopep8, verible, or vsg. # +# # # # # # # # # # # # # # # # # # # # # # # # # # # # + +# Output styles +NC='\033[0m' +BOLD='\033[1m' +GREEN='\033[32m' +RED='\033[31m' + +# Display usage instructions +display_usage() { + echo "ESP format checker ✨🛠️" + echo "Recursively format all files in the /esp repository for a given file extension." + echo "" + echo "Usage: $0 [-t {c, cpp, vhdl, v, py}]" + echo "Options:" + echo " -h, --help Display this help message." + echo "" + echo " -t, --type {c, cpp, vhdl, v, py} Specify the type of files to format." + echo " Supported options: c, cpp, vhdl, v, py" +} + +# Check if a directory is a submodule +is_submodule() { + local dir="$1" + local gitmodules="$2" + local cwd="$3" + local rel_dir="${dir#$cwd/}" + + # Compare to .gitmodules + grep -q "\[submodule \"$rel_dir\"\]" "$gitmodules" +} + +# Recursively traverse all directory levels and format C/C++ files +descend_and_format() { + local dir="$1" + local gitmodules="$2" + local cwd="$3" + local format_style="$4" + + "$format_style" "$dir" + + # Traverse all directories under /esp + for item in "$dir"/*; do + if [[ -d "$item" ]]; then + # If directory is a submodule, skip unless its sld-owned, then recurse + if is_submodule "$item" "$gitmodules" "$cwd"; then + case "$item" in + */rtl/caches/esp-caches|*/accelerators/stratus_hls/common/inc|*/rtl/caches/spandex-caches) + descend_and_format "$item" "$gitmodules" "$cwd" "$format_style" + ;; + *) + continue + ;; + esac + continue + fi + descend_and_format "$item" "$gitmodules" "$cwd" "$format_style" + fi + done +} + +# Find all .c and .h files in the current directory +find_c_h_files() { + local dir="$1" + for file in "$dir"/*.c "$dir"/*.h; do + local output + if [[ -f "$file" ]]; then + echo -n "Formatting $(basename "$file")..." + output=$(clang-format-10 -i "$file" 2>&1) + if [ ! $? -eq 0 ]; then + echo -e " ${RED}FAILED${NC}" + echo "$output" | sed 's/^/ /' + echo "" + return 1 + else + echo -e " ${GREEN}SUCCESS${NC}" + echo "" + return 0 + fi + fi + done +} + +# Find all .cpp and .hpp files in the current directory +find_cpp_hpp_files() { + local dir="$1" + for file in "$dir"/*.cpp "$dir"/*.hpp; do + if [[ -f "$file" ]]; then + echo -n "Formatting $(basename "$file")..." + output=$(clang-format-10 -i "$file" 2>&1) + if [ ! $? -eq 0 ]; then + echo -e " ${RED}FAILED${NC}" + echo "$output" | sed 's/^/ /' + echo "" + return 1 + else + echo -e " ${GREEN}SUCCESS${NC}" + echo "" + return 0 + fi + fi + done +} + +# Find all .py files in the current directory +find_py_files() { + local dir="$1" + local output + for file in "$dir"/*.py; do + if [[ -f "$file" ]]; then + echo -n "Formatting $(basename "$file")..." + output=$(python3 -m autopep8 -i -a -a "$file" 2>&1) + if [ ! $? -eq 0 ]; then + echo -e " ${RED}FAILED${NC}" + echo "$output" | sed 's/^/ /' + echo "" + return 1 + else + echo -e " ${GREEN}SUCCESS${NC}" + echo "" + return 0 + fi + fi + done +} + +# Find .v files in the current directory +find_v_files() { + local dir="$1" + local output + for file in "$dir"/*.v "$dir"/*.sv; do + if [[ -f "$file" ]]; then + echo -n "Formatting $(basename "$file")..." + output=$(verible-verilog-format --inplace --port_declarations_alignment=preserve -assignment_statement_alignment=align --indentation_spaces=4 "$file" 2>&1) + if [ ! $? -eq 0 ]; then + echo -e " ${RED}FAILED${NC}" + echo "$output" | sed 's/^/ /' + echo "" + return 1 + else + echo -e " ${GREEN}SUCCESS${NC}" + echo "" + return 0 + fi + fi + done +} + +# Function to find .vhd files in the current directory +find_vhd_files() { + local dir="$1" + local output + for file in "$dir"/*.vhd; do + if [[ -f "$file" ]]; then + echo -n "Formatting $(basename "$file")..." + output=$(vsg -f "$file" --fix -c ~/esp/vhdl-style-guide.yaml -of summary 2>&1) + if [ ! $? -eq 0 ]; then + echo -e " ${RED}FAILED${NC}" + echo "$output" | sed 's/^/ /' + echo "" + return 1 + else + echo -e " ${GREEN}SUCCESS${NC}" + echo "" + return 0 + fi + fi + done +} + +# Get cwd and gitmodules +cwd="$(git rev-parse --show-toplevel)" +gitmodules="$cwd/.gitmodules" + +# Parse command line arguments +while [[ $# -gt 0 ]]; do + case "$1" in + -t | --type ) + shift + case "$1" in + c ) format_style="find_c_h_files" + echo "Starting formatting for C files ..." + echo -e "This should be quick! \U0001F680";; + cpp ) format_style="find_cpp_hpp_files" + echo "Starting formatting for C++ files ..." + echo -e "This should be quick! \U0001F680";; + vhdl ) format_style="find_vhd_files" + echo "Starting formatting for VHDL files ..." + echo -e "This may take a while, be patient! \U0001F691";; + v ) format_style="find_v_files" + echo "Starting formatting for Verilog/SystemVerilog files ..." + echo -e "This may take a while, be patient. \U0001F691";; + py ) format_style="find_py_files" + echo "Starting formatting for Python files ..." + echo -e "This should be quick! \U0001F680";; + * ) + echo "Invalid formatting option. Valid options include: c, cpp, vhdl, v, py" + display_usage + exit 1 + ;; + esac + ;; + -h | --h | --help ) + display_usage + exit 0 + ;; + * ) + echo "Invalid option: $1" + display_usage + exit 1 + ;; + esac + shift +done +echo "" + +# Check if format_style is set +if [ -z "$format_style" ]; then + echo "Error: Missing formatting style. Use -t {c, cpp, vhdl, v, py}." + display_usage + exit 1 +fi + +descend_and_format "$cwd" "$gitmodules" "$cwd" "$format_style" +echo "" +echo "" +echo -e "✨ Formatting complete!" \ No newline at end of file diff --git a/utils/scripts/actions-pipeline/get_hls_accelerators.sh b/utils/scripts/actions-pipeline/get_hls_accelerators.sh new file mode 100644 index 0000000000..2e1d55b378 --- /dev/null +++ b/utils/scripts/actions-pipeline/get_hls_accelerators.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# Output styles +NC='\033[0m' +BOLD='\033[1m' +EMOJI_CHECK="\xE2\x9C\x94" + +# Navigate to the acc HLS directory +cd "$HOME/esp/tech/virtex7/" +directory="acc" + +accelerators=() + +# All sub-directories in virtex7/acc represent accelerators post-HLS +# Iterate over them +for dir in "$directory"/*/; do + if [ -d "$dir" ]; then # Check if the directory exists + accelerator_name=$(basename "$dir") + accelerators+=("$accelerator_name") + fi +done + +# Create a dictionary to map accelerator to its dma size +declare -A dma + +# Iterate over available dma sizes +# Pick the greatest one +for accelerator in "${accelerators[@]}"; do + sizes=() + for size_dir in "$directory/$accelerator"/*/; do + size=$(basename "$size_dir") + sizes+=("$size") + done + + sorted_sizes=($(printf "%s\n" "${sizes[@]}" | sort)) + dma["$accelerator"]="${sorted_sizes[-1]#*_*_}" +done + +echo -e "${BOLD}HLS SUCCEEDED FOR...${NC}" +if [ ${#dma[@]} -eq 0 ]; then + echo "0 accelerators" +else + for accelerator in "${!dma[@]}"; do + echo -e " ${EMOJI_CHECK} [${dma[$accelerator]}] $accelerator" + done +fi \ No newline at end of file diff --git a/utils/scripts/actions-pipeline/get_modified_accelerators.sh b/utils/scripts/actions-pipeline/get_modified_accelerators.sh new file mode 100644 index 0000000000..952f3230c0 --- /dev/null +++ b/utils/scripts/actions-pipeline/get_modified_accelerators.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# Output styles +NC='\033[0m' +BOLD='\033[1m' +EMOJI_CHECK="\xE2\x9C\x94" + +# Configuration: +# Defines what accelerators to consider changes for. +ACCELERATORS="accelerators.json" +if [ ! -f "$ACCELERATORS" ]; then + echo -e "${RED}Error:${NC} $ACCELERATORS file not found." + exit 1 +fi + +# Get all the files that were modified compared to master branch. +modified_files=$(git diff master --name-only) + +modified_accelerators=() + +# Loop through each modified file +# Check whether it's an accelerator we should consider +# If it is, add it as a "modified accelerator" +for file in $modified_files; do + while read -r accelerator; do + path=$(jq -r '.path' <<< "$accelerator") + if [[ "$file" == "$path"* ]]; then + accelerator_name=$(jq -r '.name' <<< "$accelerator") + if [[ ! " ${modified_accelerators[@]} " =~ " $accelerator_name " ]]; then + modified_accelerators+=("$accelerator_name") + fi + fi + done < <(jq -c '.accelerators[]' "$ACCELERATORS") +done + +echo -e "${BOLD}MODIFIED ACCELERATORS:${NC}" +for acc in "${modified_accelerators[@]}"; do + echo -e " ${EMOJI_CHECK} $acc" +done \ No newline at end of file diff --git a/utils/scripts/actions-pipeline/run_esp-config.sh b/utils/scripts/actions-pipeline/run_esp-config.sh new file mode 100644 index 0000000000..c7fe08e2eb --- /dev/null +++ b/utils/scripts/actions-pipeline/run_esp-config.sh @@ -0,0 +1,90 @@ +#!/bin/bash + +# Output styles +NC='\033[0m' +BOLD='\033[1m' +EMOJI_CHECK="\xE2\x9C\x94" + +# Source HLS accelerators +source "get_hls_accelerators.sh" + +# Define SoC configuration source + target +defconfig="$HOME/esp/socs/defconfig/esp_xilinx-vc707-xc7vx485t_defconfig" +esp_config="$HOME/esp/socs/xilinx-vc707-xc7vx485t/socgen/esp/.esp_config" + +# FPGA run +fpga_run="$HOME/esp/utils/scripts/actions-pipeline/./run_fpga_program.sh" + +for accelerator in "${!dma[@]}"; do + accelerator_upper=$(echo "$accelerator" | tr '[:lower:]' '[:upper:]') + + # Logging + logs="$HOME/esp/utils/scripts/actions-pipeline/logs/" + esp_config="$logs/config/$accelerator.log" + fpga_program="$logs/program/$accelerator.log" + vivado_syn="$logs/hls/$accelerator.log" + minicom="$logs/run/minicom_$accelerator.log" + run="$logs/run/run_$accelerator.log" + + # Swap in the appropriate accelerator + cp "$defconfig" "$esp_config" + sed -i "s/CONFIG_DSU_IP = C0A80107/CONGIG_DSU_IP = C0A8011C/" "$esp_config" + sed -i "s/CONFIG_DSU_ETH = A6A7A0F8043D/CONGIG_DSU_ETH = A6A7A0F80445/" "$esp_config" + sed -i "s/TILE_1_0 = 2 empty empty 0 0 0/TILE_1_0 = 2 acc $accelerator_upper 0 0 0 ${dma[$accelerator]} 0 sld/" "$esp_config" + sed -i "s/POWER_1_0 = empty 0 0 0 0 0 0 0 0 0 0 0 0/POWER_1_0 = $accelerator_upper 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0/" "$esp_config" + + # Clean the vivado directories and bistream shortcut + echo "" + echo -e "${BOLD}CLEANING VIVADO DIRECTORIES...${NC}" + cd "$HOME/esp/socs/xilinx-vc707-xc7vx485t" + rm top.bit + rm -rf vivado + make clean >/dev/null 2>&1 + echo "" + + # Make esp config + echo "" + echo -e "${BOLD}CREATING SoC CONFIG W/ ACCELERATOR...${NC}" + echo -e " ${EMOJI_CHECK} $accelerator" + make esp-config > "$esp_config" 2>&1 + + # Run SoC HLS and generate bitstream + echo "" + echo -e "${BOLD}STARTING SoC HLS W/ ACCELERATOR...${NC}" + echo -e " ${EMOJI_CHECK} $accelerator" + make vivado-syn > "$vivado_syn" 2>&1 + + # Run FPGA-program if bitstream gen suceeds + if [ -s "top.bit" ]; then + make fpga-program > "$fpga_program" 2>&1 + if grep -q ERROR "$fpga_program"; then + echo "" + echo -e "${BOLD}FPGA-PROGRAM FAILED...${NC}" + echo -e " - $accelerator" + echo "" + else + echo "" + echo -e "${BOLD}FPGA-PROGRAM SUCCEEDED...${NC}" + echo -e " ${EMOJI_CHECK} $accelerator" + echo "" + fi + + # Open Minicom in the foreground + echo -e "${BOLD}OPENING MINICOM...${NC}" + echo "" + socat pty,link=ttyV0,waitslave,mode=777 tcp:goliah.cs.columbia.edu:4332 & + socat_pid=$! + sleep 2 + VIRTUAL_DEVICE=$(readlink ttyV0) + + # Run fpga program in the background + echo -e "${BOLD}WRITING RESULTS TO MINICOM...${NC}" + echo "" + $fpga_run > "$run" 2>&1 & + minicom -p "$VIRTUAL_DEVICE" -C "$minicom" 2>&1 + kill -9 "$socat_pid" + else + echo -e "${BOLD}BITSTREAM GENERATION FAILED...${NC}" + echo -e " - $accelerator" + fi +done \ No newline at end of file diff --git a/utils/scripts/actions-pipeline/run_fpga_program.sh b/utils/scripts/actions-pipeline/run_fpga_program.sh new file mode 100644 index 0000000000..c35db2e9dc --- /dev/null +++ b/utils/scripts/actions-pipeline/run_fpga_program.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +cd "$HOME/esp/socs/xilinx-vc707-xc7vx485t" +sleep 10 + +make fpga-run + +! killall -9 minicom \ No newline at end of file diff --git a/utils/scripts/actions-pipeline/run_sims.sh b/utils/scripts/actions-pipeline/run_sims.sh new file mode 100644 index 0000000000..bda6446722 --- /dev/null +++ b/utils/scripts/actions-pipeline/run_sims.sh @@ -0,0 +1,100 @@ +#!/bin/bash + +# Output styles +NC='\033[0m' +BOLD='\033[1m' +EMOJI_CHECK="\xE2\x9C\x94" + +# Discover all child processes and their descendants +list_descendants() { + local children=$(ps -o pid= --ppid "$1") + for pid in $children; do + list_descendants "$pid" + done + echo "$children" +} + +# Kill all child process and their descendants +cleanup() { + echo "" + echo "" + echo -e "${BOLD}TERMINATING...${NC}" + echo -e " ${EMOJI_CHECK} Killing child processes" + local descendants=$(list_descendants $$) + for pid in $descendants; do + if kill -0 "$pid" &>/dev/null; then + kill "$pid" + fi + done + echo "" + echo -e "${BOLD}DONE.${NC}" + exit +} + +trap cleanup SIGINT + +# Clean hls output directories +echo -e "${BOLD}CLEANING HLS DIRECTORIES...${NC}" +acc_dir="$HOME/esp/tech/virtex7/acc" +echo -e "${EMOJI_CHECK} Clearing tech/virtex/acc${NC}" +rm -rf "$acc_dir" +mkdir -p "$acc_dir" +echo -e "*" > "$acc_dir/.gitignore" +echo -e ".gitignore" >> "$acc_dir/.gitignore" + +# Get modified accelerators +echo "" +echo -e "${BOLD}GETTING MODIFIED ACCs...${NC}" +source get_modified_accelerators.sh +echo "" + +# Source the HLS tools +echo "" +echo -e "${BOLD}EXPORTING HLS TOOLS...${NC}" +source /opt/cad/scripts/tools_env.sh +echo "" + +CORES=$(nproc) +child_processes=() +declare -A core_jobs +declare -A job_names +for ((i=0; i/dev/null 2>&1) & + child_processes+=("$!") + job_names[$!]="$accelerator_name" + + ((core_jobs[$min_core]++)) + min_jobs=${core_jobs[$min_core]} +done + +# Wait for each hls job to finish and update results as each process completes +echo "" +echo -e "${BOLD}WAITING FOR HLS JOBS TO FINISH...${NC}" + +for pid in "${child_processes[@]}"; do + wait "$pid" + + # Print the latest job result with accelerator name + echo -e " ${EMOJI_CHECK} ${job_names[$pid]}" +done \ No newline at end of file diff --git a/vhdl-style-guide.yaml b/vhdl-style-guide.yaml new file mode 100644 index 0000000000..ccc0b31b21 --- /dev/null +++ b/vhdl-style-guide.yaml @@ -0,0 +1,11 @@ +rule: + constant_004: + case: 'upper' + prefix_exceptions: + - 'G_' + suffix_exceptions: + - '_G' + case_exceptions: + = 'IEEE' + length_001 : + length : 120 \ No newline at end of file