diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9f74d050e..680cf5cb6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -118,7 +118,7 @@ jobs: cd ../.. # Run Dependencies Plugin Tests - - name: Run Tests on Dependency Plugin + - name: Run Tests on Dependencies Plugin run: | cd plugins/dependencies rake @@ -207,7 +207,7 @@ jobs: cd ../.. # Run Dependencies Plugin Tests - - name: Run Tests on Dependency Plugin + - name: Run Tests on Dependencies Plugin run: | cd plugins/dependencies rake diff --git a/assets/project_as_gem.yml b/assets/project_as_gem.yml index c6ae0c8f3..891d9462b 100644 --- a/assets/project_as_gem.yml +++ b/assets/project_as_gem.yml @@ -54,11 +54,11 @@ :load_paths: [] :enabled: #- beep # beeps when finished, so you don't waste time waiting for ceedling - - module_generator # handy for quickly creating source, header, and test templates + - module_generator # handy for quickly creating source, header, and test templates #- gcov # test coverage using gcov. Requires gcc, gcov, and a coverage analyzer like gcovr #- bullseye # test coverage using bullseye. Requires bullseye for your platform #- command_hooks # write custom actions to be called at different points during the build process - #- compile_commands_json_db # generate a compile_commands.json file + #- compile_commands_json_db # generate a compile_commands.json file #- dependencies # automatically fetch 3rd party libraries, etc. #- subprojects # managing builds and test for static libraries #- fake_function_framework # use FFF instead of CMock diff --git a/bin/cli_helper.rb b/bin/cli_helper.rb index 249dfc49d..bb432349b 100644 --- a/bin/cli_helper.rb +++ b/bin/cli_helper.rb @@ -203,22 +203,18 @@ def process_stopwatch(tasks:, default_tasks:) def print_rake_tasks() - Rake.application.standard_exception_handling do - # (This required digging into Rake internals a bit.) - Rake.application.define_singleton_method(:name=) {|n| @name = n} - Rake.application.name = 'ceedling' - Rake.application.options.show_tasks = :tasks - Rake.application.options.show_task_pattern = /^(?!.*build).*$/ - Rake.application.display_tasks_and_comments() - end + # (This required digging into Rake internals a bit.) + Rake.application.define_singleton_method(:name=) {|n| @name = n} + Rake.application.name = 'ceedling' + Rake.application.options.show_tasks = :tasks + Rake.application.options.show_task_pattern = /^(?!.*build).*$/ + Rake.application.display_tasks_and_comments() end def run_rake_tasks(tasks) - Rake.application.standard_exception_handling do - Rake.application.collect_command_line_tasks( tasks ) - Rake.application.top_level() - end + Rake.application.collect_command_line_tasks( tasks ) + Rake.application.top_level() end @@ -401,10 +397,10 @@ def vendor_tools(ceedling_root, dest) 'vendor/unity', 'vendor/cmock', 'vendor/c_exception', + 'vendor/diy' ].each do |src| + # Look up licenses using a Glob as capitalization can be inconsistent glob = File.join( ceedling_root, src, 'license.txt' ) - - # Look up licenses (use glob as capitalization can be inconsistent) listing = @file_wrapper.directory_listing( glob ) # Already case-insensitive # Safety check on nil references since we explicitly reference first element diff --git a/lib/ceedling/system_wrapper.rb b/lib/ceedling/system_wrapper.rb index 6531b26dd..7b0c033a9 100644 --- a/lib/ceedling/system_wrapper.rb +++ b/lib/ceedling/system_wrapper.rb @@ -60,6 +60,8 @@ def time_now(format=nil) return Time.now.strftime( format ) end + # If set, `boom` allows a non-zero exit code in results. + # Otherwise, disabled `boom` forces a success exit code but collects errors. def shell_capture3(command:, boom:false) # Beginning with later versions of Ruby2, simple exit codes were replaced # by the more capable and robust Process::Status. @@ -70,16 +72,10 @@ def shell_capture3(command:, boom:false) stdout, stderr = '' # Safe initialization defaults status = nil # Safe initialization default - begin - # Run the command but absorb any exceptions and capture error info instead - stdout, stderr, status = Open3.capture3( command ) - rescue => err - stderr = err.to_s - exit_code = nil - end + stdout, stderr, status = Open3.capture3( command ) - # If boom, then capture the actual exit code, otherwise leave it as zero - # as though execution succeeded + # If boom, then capture the actual exit code. + # Otherwise, leave it as zero as though execution succeeded. exit_code = status.exitstatus.freeze if boom and !status.nil? # (Re)set the global system exit code so everything matches diff --git a/lib/ceedling/tool_validator.rb b/lib/ceedling/tool_validator.rb index 8233cd764..fe11fb460 100644 --- a/lib/ceedling/tool_validator.rb +++ b/lib/ceedling/tool_validator.rb @@ -36,7 +36,8 @@ def validate_executable(tool:, name:, extension:, respect_optional:, boom:) exists = false error = '' - executable = tool[:executable] + # Get unfrozen copy so we can modify for our processing + executable = tool[:executable].dup() # Handle a missing :executable if (executable.nil? or executable.empty?) @@ -88,7 +89,7 @@ def validate_executable(tool:, name:, extension:, respect_optional:, boom:) end # Construct end of error message - error = "does not exist in system search paths." if not exists + error = "does not exist in system search paths" if not exists # If there is a path included, check that explicit filepath exists else @@ -96,7 +97,7 @@ def validate_executable(tool:, name:, extension:, respect_optional:, boom:) exists = true else # Construct end of error message - error = "does not exist on disk." if not exists + error = "does not exist on disk" if not exists end end @@ -121,10 +122,14 @@ def validate_stderr_redirect(tool:, name:, boom:) error = '' redirect = tool[:stderr_redirect] + # If no redirect set at all, it's cool + return if redirect.nil? + + # Otherwise, process the redirect that's been set if redirect.class == Symbol if not StdErrRedirect.constants.map{|constant| constant.to_s}.include?( redirect.to_s.upcase ) options = StdErrRedirect.constants.map{|constant| ':' + constant.to_s.downcase}.join(', ') - error = "#{name} ↳ :stderr_redirect => :#{redirect} is not a recognized option {#{options}}." + error = "#{name} ↳ :stderr_redirect => :#{redirect} is not a recognized option {#{options}}" # Raise exception if requested raise CeedlingException.new( error ) if boom @@ -134,7 +139,7 @@ def validate_stderr_redirect(tool:, name:, boom:) return false end elsif redirect.class != String - raise CeedlingException.new( "#{name} ↳ :stderr_redirect is neither a recognized value nor custom string." ) + raise CeedlingException.new( "#{name} ↳ :stderr_redirect is neither a recognized value nor custom string" ) end return true diff --git a/plugins/beep/lib/beep.rb b/plugins/beep/lib/beep.rb index 4446af03f..c658f69f9 100755 --- a/plugins/beep/lib/beep.rb +++ b/plugins/beep/lib/beep.rb @@ -60,7 +60,9 @@ def post_build command = @ceedling[:tool_executor].build_command_line( @tools[:beep_on_done], [], - ["ceedling build done"]) # Only used by tools with `${1}` replacement argument + # Only used by tools with `${1}` replacement argument + 'ceedling build done' + ) # Verbosity is enabled to allow shell output (primarily for sake of the bell character) @@ -72,8 +74,9 @@ def post_error command = @ceedling[:tool_executor].build_command_line( @tools[:beep_on_error], [], - ["ceedling build error"]) # Only used by tools with `${1}` replacement argument - + # Only used by tools with `${1}` replacement argument + 'ceedling build error' + ) # Verbosity is enabled to allow shell output (primarily for sake of the bell character) @ceedling[:system_wrapper].shell_system( command: command[:line], verbose: true ) diff --git a/plugins/dependencies/README.md b/plugins/dependencies/README.md index 845392252..c5c930015 100644 --- a/plugins/dependencies/README.md +++ b/plugins/dependencies/README.md @@ -24,15 +24,19 @@ containing header files that might want to be included by your release project. So how does all this magic work? -First, you need to add the `:dependencies` plugin to your list. Then, we'll add a new -section called :dependencies. There, you can list as many dependencies as you desire. Each -has a series of fields which help Ceedling to understand your needs. Many of them are -optional. If you don't need that feature, just don't include it! In the end, it'll look -something like this: +First, you need to add the Dependencies plugin to your list of enabled plugins. Then, we'll +add a new comfiguration section called `:dependencies`. There, you can list as many +dependencies as you desire. Each has a series of fields that help Ceedling to understand +your needs. Many of them are optional. If you don't need that feature, just don't include +it! In the end, it'll look something like this: + +```yaml +:plugins: + :enabled: + - dependencies -``` :dependencies: - :libraries: + :deps: - :name: WolfSSL :paths: :fetch: third_party/wolfssl/source @@ -116,11 +120,11 @@ couple of fields: - `:git` -- This tells Ceedling that we want to clone a git repo to our source path. - `:svn` -- This tells Ceedling that we want to checkout a subversion repo to our source path. - `:custom` -- This tells Ceedling that we want to use a custom command or commands to fetch the code. -- `:source` -- This is the path or url to fetch code when using the zip, gzip or git method. -- `:tag`/`:branch` -- This is the specific tag or branch that you wish to retrieve (git only. optional). -- `:hash` -- This is the specific SHA1 hash you want to fetch (git only. optional, requires a deep clone). -- `:revision` -- This is the specific revision you want to fetch (svn only. optional). -- `:executable` -- This is a list of commands to execute when using the `:custom` method +- `:source` -- This is the path or url to fetch code when using the `:zip`, `:gzip` or `:git` method. +- `:tag`/`:branch` -- This is the specific tag or branch that you wish to retrieve (`:git` only, optional). +- `:hash` -- This is the specific SHA1 hash you want to fetch (`:git` only, optional and triggers a deep clone). +- `:revision` -- This is the specific revision you want to fetch (`:svn` only, optional). +- `:executable` -- This is a YAML list of commands to execute when using the `:custom` method Some notes: @@ -131,16 +135,24 @@ Environment Variables --------------------- Many build systems support customization through environment variables. By specifying -an array of environment variables, Ceedling will customize the shell environment before -calling the build process. +an array of environment variables, the Dependencies plugin will customize the shell environment +before calling the build process. + +Note that Ceedling’s project configuration includes a top-level `:environment` sections itself. +The top-level `:environment` section is for all of Ceedling. The `:environment` section nested +within a specific dependency’s configuration is only for the shell environment used to process +that dependency. The format and abilities of the two `:environment` configuration sections are +also different. Environment variables may be specified in three ways. Let's look at one of each: -``` - :environment: - - ARCHITECTURE=ARM9 - - CFLAGS+=-DADD_AWESOMENESS - - CFLAGS-=-DWASTE +```yaml +:dependencies: + : + :environment: + - ARCHITECTURE=ARM9 + - CFLAGS+=-DADD_AWESOMENESS + - CFLAGS-=-DWASTE ``` In the first example, you see the most straightforward method. The environment variable @@ -260,7 +272,7 @@ Custom Tools You can optionally specify a compiler, assembler, and linker, just as you would a release build: -``` +```yaml :tools: :deps_compiler: :executable: gcc @@ -282,7 +294,7 @@ Then, once created, you can reference these tools in your build steps by using t of a series of strings to explain all the steps. Ceedling will understand that it should build all the specified source and/or assembly files into the specified library: -``` +```yaml :dependencies: :deps: - :name: CaptainCrunch diff --git a/plugins/dependencies/config/defaults.yml b/plugins/dependencies/config/defaults.yml index 735d34435..e9a024903 100644 --- a/plugins/dependencies/config/defaults.yml +++ b/plugins/dependencies/config/defaults.yml @@ -11,17 +11,68 @@ :tools: :deps_compiler: - :executable: gcc + :executable: gcc + :name: 'Dependencies compiler' :arguments: - -g - -I"$": COLLECTION_PATHS_DEPS - -D$: COLLECTION_DEFINES_DEPS - -c "${1}" - -o "${2}" - :deps_linker: + + :deps_linker: :executable: ar + :name: 'Dependencies archiver' :arguments: - rcs - ${2} - ${1} + + :deps_zip: + :executable: unzip + :name: 'Dependencies zip unarchiver' + :optional: true + :arguments: + - -o + - ${1} # Filepath + + :deps_targzip: + :executable: tar + :name: 'Dependencies tar gzip unarchiver' + :optional: true + :arguments: + - -xvzf + - ${1} # Filepath + - -C + - ./ + + :deps_git_clone: + :executable: git + :name: 'Dependencies git clone' + :optional: true + :arguments: + - clone + - ${1} # Optional branch with `-b` flag + - ${2} # Optional depth with `--depth` flag + - ${3} # Repository source + - . + + :deps_git_checkout: + :executable: git + :name: 'Dependencies git checkout' + :optional: true + :arguments: + - checkout + - ${1} # Git hash + + :deps_subversion: + :executable: svn + :name: 'Dependencies subversion' + :optional: true + :arguments: + - checkout + - ${1} # Optional branch with `--revision` flag + - ${2} # Repository source + - . + ... diff --git a/plugins/dependencies/lib/dependencies.rb b/plugins/dependencies/lib/dependencies.rb index 84270c8b6..f0690d26b 100644 --- a/plugins/dependencies/lib/dependencies.rb +++ b/plugins/dependencies/lib/dependencies.rb @@ -7,6 +7,7 @@ require 'ceedling/plugin' require 'ceedling/constants' +require 'ceedling/exceptions' require 'pathname' DEPENDENCIES_ROOT_NAME = 'dependencies' @@ -15,7 +16,7 @@ class Dependencies < Plugin - def setup + def setup() # Set up a fast way to look up dependencies by name or static lib path @dependencies = {} @dynamic_libraries = [] @@ -32,6 +33,9 @@ def setup @dynamic_libraries += get_dynamic_libraries_for_dependency(deplib) end + + # Validate fetch tools per the configuration + @dependencies.each {|_, config| validate_fetch_tools( config )} end def config() @@ -54,7 +58,7 @@ def config() end def get_name(deplib) - raise "Each dependency must have a name!" if deplib[:name].nil? + raise CeedlingException.new( "Each dependency must have a name!" ) if deplib[:name].nil? return deplib[:name].gsub(/\W*/,'') end @@ -135,7 +139,7 @@ def get_include_files_for_dependency(deplib) def set_env_if_required(lib_path) blob = @dependencies[lib_path] - raise "Could not find dependency '#{lib_path}'" if blob.nil? + raise CeedlingException.new( "Could not find dependency '#{lib_path}'" ) if blob.nil? return if (blob[:environment].nil?) return if (blob[:environment].empty?) @@ -154,74 +158,135 @@ def set_env_if_required(lib_path) end end - def wrap_command(cmd) - if (cmd.class == String) - cmd = { - :name => cmd.split(/\s+/)[0], - :executable => cmd.split(/\s+/)[0], - :line => cmd, - :options => { :boom => true } - } - end - return cmd + def generate_command_line(cmdline, name=nil) + # Break apart command line at white spaces + cmdline_items = cmdline.split(/\s+/) + + # Even though it may seem redundant to build a command line we already have, + # we do so for possible argument expansion/substitution and, more importantly, logging output. + + # Construct a tool configuration + tool_config = { + # Use tool name if provided, otherwise, grab something from the command line + :name => name.nil? ? cmdline_items[0] : name, + + # Extract executable as first item on the command line + :executable => cmdline_items[0], + + # Extract remaining arguments if there are any + :arguments => (cmdline_items.length > 1) ? cmdline_items[1..-1] : [] + } + + # Construct a command from our tool configuration + command = @ceedling[:tool_executor].build_command_line( tool_config, [] ) + + return command end def fetch_if_required(lib_path) blob = @dependencies[lib_path] - raise "Could not find dependency '#{lib_path}'" if blob.nil? + + raise CeedlingException.new( "Could not find dependency '#{lib_path}'" ) if blob.nil? + if (blob[:fetch].nil?) || (blob[:fetch][:method].nil?) - @ceedling[:loginator].log("No method to fetch #{blob[:name]}", Verbosity::COMPLAIN) + @ceedling[:loginator].log("No fetch method for dependency '#{blob[:name]}'", Verbosity::COMPLAIN) return end - unless (directory(get_source_path(blob))) #&& !Dir.empty?(get_source_path(blob))) + + unless (directory(get_source_path(blob))) @ceedling[:loginator].log("Path #{get_source_path(blob)} is required", Verbosity::COMPLAIN) return end FileUtils.mkdir_p(get_fetch_path(blob)) unless File.exist?(get_fetch_path(blob)) - steps = case blob[:fetch][:method] - when :none - [] - when :zip - [ "unzip -o #{blob[:fetch][:source]}" ] - when :tar_gzip - [ "tar -xvzf #{blob[:fetch][:source]} -C ./" ] - when :git - branch = blob[:fetch][:tag] || blob[:fetch][:branch] || '' - branch = ("-b " + branch) unless branch.empty? - unless blob[:fetch][:hash].nil? - # Do a deep clone to ensure the commit we want is available - retval = [ "git clone #{branch} #{blob[:fetch][:source]} ." ] - # Checkout the specified commit - retval << "git checkout #{blob[:fetch][:hash]}" - else - # Do a thin clone - retval = [ "git clone #{branch} --depth 1 #{blob[:fetch][:source]} ." ] - end - when :svn - revision = blob[:fetch][:revision] || '' - revision = ("--revision " + branch) unless branch.empty? - retval = [ "svn checkout #{revision} #{blob[:fetch][:source]} ." ] - retval - when :custom - blob[:fetch][:executable] - else - raise "Unknown fetch method '#{blob[:fetch][:method]}' for dependency '#{blob[:name]}'" - end + steps = [] + + # Tools already validated within `setup()` + case blob[:fetch][:method] + when :none + # Do nothing + + when :zip + steps << + @ceedling[:tool_executor].build_command_line( + TOOLS_DEPS_ZIP, + [], + blob[:fetch][:source] + ) + + when :tar_gzip + steps << + @ceedling[:tool_executor].build_command_line( + TOOLS_DEPS_TARGZIP, + [], + blob[:fetch][:source] + ) + + when :git + branch = blob[:fetch][:tag] || blob[:fetch][:branch] || '' + branch = '-b ' + branch unless branch.empty? + + unless blob[:fetch][:hash].nil? + # Do a deep clone to ensure the commit we want is available + steps << + @ceedling[:tool_executor].build_command_line( + TOOLS_DEPS_GIT_CLONE, + [], + branch, + '', # No depth + blob[:fetch][:source] + ) + + # Checkout the specified commit + steps << + @ceedling[:tool_executor].build_command_line( + TOOLS_DEPS_GIT_CHECKOUT, + [], + blob[:fetch][:hash] + ) + else + # Do a thin clone + steps << + @ceedling[:tool_executor].build_command_line( + TOOLS_DEPS_GIT_CLONE, + [], + branch, + '--depth 1', + blob[:fetch][:source] + ) + end + + when :svn + revision = blob[:fetch][:revision] || '' + revision = '--revision ' + revision unless revision.empty? + + steps << + @ceedling[:tool_executor].build_command_line( + TOOLS_DEPS_SUBVERSION, + [], + revision, + blob[:fetch][:source] + ) + + when :custom + blob[:fetch][:executable].each.with_index(1) do |cmdline, index| + steps << generate_command_line( cmdline, "Dependencies custom command \##{index}" ) + end + end # Perform the actual fetching @ceedling[:loginator].log("Fetching dependency #{blob[:name]}...", Verbosity::NORMAL) Dir.chdir(get_fetch_path(blob)) do steps.each do |step| - @ceedling[:tool_executor].exec( wrap_command(step) ) + @ceedling[:tool_executor].exec( step ) end end end def build_if_required(lib_path) blob = @dependencies[lib_path] - raise "Could not find dependency '#{lib_path}'" if blob.nil? + raise CeedlingException.new( "Could not find dependency '#{lib_path}'" ) if blob.nil? # We don't clean anything unless we know how to fetch a new copy if (blob[:build].nil? || blob[:build].empty?) @@ -239,7 +304,7 @@ def build_if_required(lib_path) if (step.class == Symbol) exec_dependency_builtin_command(step, blob) else - @ceedling[:tool_executor].exec( wrap_command(step) ) + @ceedling[:tool_executor].exec( generate_command_line(step) ) end end end @@ -247,7 +312,7 @@ def build_if_required(lib_path) def clean_if_required(lib_path) blob = @dependencies[lib_path] - raise "Could not find dependency '#{lib_path}'" if blob.nil? + raise CeedlingException.new( "Could not find dependency '#{lib_path}'" ) if blob.nil? # We don't clean anything unless we know how to fetch a new copy if (blob[:fetch].nil? || blob[:fetch][:method].nil?) @@ -267,7 +332,7 @@ def clean_if_required(lib_path) def deploy_if_required(lib_path) blob = @dependencies[lib_path] - raise "Could not find dependency '#{lib_path}'" if blob.nil? + raise CeedlingException.new( "Could not find dependency '#{lib_path}'" ) if blob.nil? # We don't need to deploy anything if there isn't anything to deploy if (blob[:artifacts].nil? || blob[:artifacts][:dynamic_libraries].nil? || blob[:artifacts][:dynamic_libraries].empty?) @@ -318,7 +383,7 @@ def exec_dependency_builtin_command(step, blob) when :build_lib # We are going to use our defined deps tools to build this library build_lib(blob) else - raise "No such build action as #{step.inspect} for dependency #{blob[:name]}" + raise CeedlingException.new( "No such build action as #{step.inspect} for dependency #{blob[:name]}" ) end end @@ -335,11 +400,11 @@ def build_lib(blob) # Verify there is an artifact that we're building that makes sense libs = [] - raise "No library artifacts specified for dependency #{name}" unless blob.include?(:artifacts) + raise CeedlingException.new( "No library artifacts specified for dependency #{name}" ) unless blob.include?(:artifacts) libs += blob[:artifacts][:static_libraries] if blob[:artifacts].include?(:static_libraries) libs += blob[:artifacts][:static_libraries] if blob[:artifacts].include?(:static_libraries) libs = libs.flatten.uniq - raise "No library artifacts specified for dependency #{name}" if libs.empty? + raise CeedlingException.new( "No library artifacts specified for dependency #{name}" ) if libs.empty? lib = libs[0] # Find all the source, header, and assembly files @@ -350,10 +415,10 @@ def build_lib(blob) end # Do we have what we need to do this? - raise "Nothing to build" if (asm.empty? and src.empty?) - raise "No assembler specified for building dependency #{name}" unless (defined?(TOOLS_DEPS_ASSEMBLER) || asm.empty?) - raise "No compiler specified for building dependency #{name}" unless (defined?(TOOLS_DEPS_COMPILER) || src.empty?) - raise "No linker specified for building dependency #{name}" unless defined?(TOOLS_DEPS_LINKER) + raise CeedlingException.new( "Nothing to build" ) if (asm.empty? and src.empty?) + raise CeedlingException.new( "No assembler specified for building dependency #{name}" ) unless (defined?(TOOLS_DEPS_ASSEMBLER) || asm.empty?) + raise CeedlingException.new( "No compiler specified for building dependency #{name}" ) unless (defined?(TOOLS_DEPS_COMPILER) || src.empty?) + raise CeedlingException.new( "No linker specified for building dependency #{name}" ) unless defined?(TOOLS_DEPS_LINKER) # Build all the source files src.each do |src_file| @@ -421,6 +486,39 @@ def replace_constant(constant, new_value) Object.send(:remove_const, constant.to_sym) if (Object.const_defined? constant) Object.const_set(constant, new_value) end + + ### Private ### + + private + + def validate_fetch_tools(blob) + return if blob[:fetch].nil? || blob[:fetch][:method].nil? + + case blob[:fetch][:method] + when :none + # Do nothing + + when :zip + @ceedling[:tool_validator].validate( tool:TOOLS_DEPS_ZIP, boom:true ) + + when :tar_gzip + @ceedling[:tool_validator].validate( tool:TOOLS_DEPS_TARGZIP, boom:true ) + + when :git + @ceedling[:tool_validator].validate( tool:TOOLS_DEPS_GIT_CLONE, boom:true ) + @ceedling[:tool_validator].validate( tool:TOOLS_DEPS_GIT_CHECKOUT, boom: true ) + + when :svn + @ceedling[:tool_validator].validate( tool:TOOLS_DEPS_SUBVERSION, boom: true ) + + when :custom + # Do nothing + + else + raise CeedlingException.new( "Unknown fetch method '#{blob[:fetch][:method]}' for dependency '#{blob[:name]}'" ) + end + end + end