From 0e474004f2a5d9b8d2f80a0caafe33110243d720 Mon Sep 17 00:00:00 2001 From: nmburgan <13688219+nmburgan@users.noreply.github.com> Date: Sat, 20 Sep 2025 11:08:30 -0700 Subject: [PATCH 1/2] Rubocop: Stop selectively disabling cops we routinely disable --- .rubocop.yml | 9 ++++++--- lib/vanagon/cli.rb | 2 +- lib/vanagon/cli/build.rb | 2 +- lib/vanagon/cli/build_requirements.rb | 2 +- lib/vanagon/cli/list.rb | 2 +- lib/vanagon/component.rb | 4 ++-- lib/vanagon/component/dsl.rb | 6 +++--- lib/vanagon/component/rules.rb | 2 +- lib/vanagon/component/source.rb | 2 +- lib/vanagon/component/source/git.rb | 2 +- lib/vanagon/component/source/http.rb | 2 +- lib/vanagon/component/source/local.rb | 2 +- lib/vanagon/driver.rb | 6 +++--- lib/vanagon/engine/always_be_scheduling.rb | 4 ++-- lib/vanagon/engine/docker.rb | 2 +- lib/vanagon/patch.rb | 2 +- lib/vanagon/platform.rb | 4 ++-- lib/vanagon/platform/deb.rb | 4 ++-- lib/vanagon/platform/macos.rb | 4 ++-- lib/vanagon/platform/solaris_10.rb | 4 ++-- lib/vanagon/platform/solaris_11.rb | 2 +- lib/vanagon/platform/windows.rb | 6 +++--- lib/vanagon/project.rb | 8 ++++---- lib/vanagon/utilities.rb | 2 +- lib/vanagon/utilities/extra_files_signer.rb | 2 +- 25 files changed, 45 insertions(+), 42 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index d8027ad5..ae693637 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -336,9 +336,6 @@ Metrics/ModuleLength: Metrics/ParameterLists: Enabled: false -Metrics/PerceivedComplexity: - Enabled: true - Naming/AccessorMethodName: Enabled: false @@ -815,3 +812,9 @@ Style/FormatStringToken: Style/MultilineTernaryOperator: Enabled: false + +Metrics/AbcSize: + Enabled: false + +Metrics/PerceivedComplexity: + Enabled: false diff --git a/lib/vanagon/cli.rb b/lib/vanagon/cli.rb index e3e9fa11..f9b2651f 100644 --- a/lib/vanagon/cli.rb +++ b/lib/vanagon/cli.rb @@ -42,7 +42,7 @@ class CLI help print this help DOCOPT - def parse(argv) # rubocop:disable Metrics/AbcSize + def parse(argv) parsed_options = parse_options(argv) sub_command = parsed_options[''] sub_argv = parsed_options[''] diff --git a/lib/vanagon/cli/build.rb b/lib/vanagon/cli/build.rb index 0d9cd991..7ad57a2a 100644 --- a/lib/vanagon/cli/build.rb +++ b/lib/vanagon/cli/build.rb @@ -40,7 +40,7 @@ def parse(argv) exit 1 end - def run(options) # rubocop:disable Metrics/AbcSize + def run(options) project = options[:project_name] platform_list = options[:platforms].split(',') target_list = [] diff --git a/lib/vanagon/cli/build_requirements.rb b/lib/vanagon/cli/build_requirements.rb index 419302aa..fe58a953 100644 --- a/lib/vanagon/cli/build_requirements.rb +++ b/lib/vanagon/cli/build_requirements.rb @@ -32,7 +32,7 @@ def parse(argv) exit 1 end - def run(options) # rubocop:disable Metrics/AbcSize + def run(options) platform = options[:platform] project = options[:project_name] driver = Vanagon::Driver.new(platform, project) diff --git a/lib/vanagon/cli/list.rb b/lib/vanagon/cli/list.rb index 5cd938f9..59dfbc14 100644 --- a/lib/vanagon/cli/list.rb +++ b/lib/vanagon/cli/list.rb @@ -29,7 +29,7 @@ def output(list, use_spaces) return list end - def run(options) # rubocop:disable Metrics/AbcSize + def run(options) check_directories(options) default_list = topic_list(File.dirname(__FILE__), '..', 'platform', 'defaults') diff --git a/lib/vanagon/component.rb b/lib/vanagon/component.rb index e918f2c4..c063047e 100644 --- a/lib/vanagon/component.rb +++ b/lib/vanagon/component.rb @@ -156,7 +156,7 @@ def self.load_component(name, configdir, settings, platform) # @param settings [Hash] the settings to be used in the component # @param platform [Vanagon::Platform] the platform to build the component for # @return [Vanagon::Component] the component with the given settings and platform - def initialize(name, settings, platform) # rubocop:disable Metrics/AbcSize + def initialize(name, settings, platform) @name = name @settings = settings @platform = platform @@ -313,7 +313,7 @@ def deprecated_rewrite_url # makefile template # # @param workdir [String] working directory to put the source into - def get_source(workdir) # rubocop:disable Metrics/AbcSize, Metrics/PerceivedComplexity + def get_source(workdir) opts = options.merge({ workdir: workdir, dirname: dirname }) if url || !mirrors.empty? if %w[y yes true 1].include? ENV.fetch('VANAGON_USE_MIRRORS', 'n').downcase diff --git a/lib/vanagon/component/dsl.rb b/lib/vanagon/component/dsl.rb index 1ec2904d..03d30491 100644 --- a/lib/vanagon/component/dsl.rb +++ b/lib/vanagon/component/dsl.rb @@ -176,7 +176,7 @@ def conflicts(pkgname, version = nil) # service_type [String] type of the service (network, application, system, etc) # init_system [String] the init system on which to install service (sysv, systemd) # link_target [String] executable service file should be linked to - def install_service(service_file, default_file = nil, service_name = @component.name, **options) # rubocop:disable Metrics/AbcSize + def install_service(service_file, default_file = nil, service_name = @component.name, **options) init_system = options[:init_system] || @component.platform.servicetype servicedir = @component.platform.get_service_dir(init_system) @@ -246,7 +246,7 @@ def install_service(service_file, default_file = nil, service_name = @component. # @param owner [String] owner of the file # @param group [String] group owner of the file # # @param sudo [Boolean] whether to use sudo to create the file and set mode - def install_file(source, target, mode: nil, owner: nil, group: nil, sudo: false) # rubocop:disable Metrics/AbcSize + def install_file(source, target, mode: nil, owner: nil, group: nil, sudo: false) sudo_check = sudo ? "#{sudo_bin} " : "" @component.install << "#{sudo_check}#{@component.platform.install} -d '#{File.dirname(target)}'" @@ -421,7 +421,7 @@ def add_source(uri, options = {}) # @param mode [String] octal mode to apply to the directory # @param owner [String] owner of the directory # @param group [String] group of the directory - def directory(dir, mode: nil, owner: nil, group: nil) # rubocop:disable Metrics/AbcSize + def directory(dir, mode: nil, owner: nil, group: nil) install_flags = ['-d'] if @component.platform.is_windows? unless mode.nil? && owner.nil? && group.nil? diff --git a/lib/vanagon/component/rules.rb b/lib/vanagon/component/rules.rb index 68684bf2..49d2d79f 100644 --- a/lib/vanagon/component/rules.rb +++ b/lib/vanagon/component/rules.rb @@ -47,7 +47,7 @@ def initialize(component, project, platform) # in the returned rules. # # @return [Array] - def rules # rubocop:disable Metrics/AbcSize + def rules list_of_rules = [ component_rule, unpack_rule, diff --git a/lib/vanagon/component/source.rb b/lib/vanagon/component/source.rb index 0e649056..4476059b 100644 --- a/lib/vanagon/component/source.rb +++ b/lib/vanagon/component/source.rb @@ -18,7 +18,7 @@ class << self # @param options [Hash] hash of the options needed for the subtype # @param workdir [String] working directory to fetch the source into # @return [Vanagon::Component::Source] the correct subtype for the given source - def source(uri_instance, **options) # rubocop:disable Metrics/AbcSize + def source(uri_instance, **options) # Sometimes the uri comes in as a string, but sometimes it's already been # coerced into a URI object. The individual source providers will turn # the passed uri into a URI object if needed, but for this method we diff --git a/lib/vanagon/component/source/git.rb b/lib/vanagon/component/source/git.rb index 483e0d84..cfea4d23 100644 --- a/lib/vanagon/component/source/git.rb +++ b/lib/vanagon/component/source/git.rb @@ -94,7 +94,7 @@ def default_options # rubocop:disable Lint/DuplicateMethods # @param url [String] url of git repo to use as source # @param ref [String] ref to checkout from git repo # @param workdir [String] working directory to clone into - def initialize(url, workdir:, **options) # rubocop:disable Metrics/AbcSize + def initialize(url, workdir:, **options) opts = default_options.merge(options.reject { |k, v| v.nil? }) # Ensure that #url returns a URI object diff --git a/lib/vanagon/component/source/http.rb b/lib/vanagon/component/source/http.rb index 69a781c8..22d4f802 100644 --- a/lib/vanagon/component/source/http.rb +++ b/lib/vanagon/component/source/http.rb @@ -118,7 +118,7 @@ def verify # Downloads the file from @url into the @workdir # @param target_url [String, URI, Addressable::URI] url of an http source to retrieve with GET # @raise [RuntimeError, Vanagon::Error] an exception is raised if the URI scheme cannot be handled - def download(target_url, target_file = nil, headers = { "Accept-Encoding" => "identity" }) # rubocop:disable Metrics/AbcSize + def download(target_url, target_file = nil, headers = { "Accept-Encoding" => "identity" }) uri = URI.parse(target_url.to_s) target_file ||= File.basename(uri.path) diff --git a/lib/vanagon/component/source/local.rb b/lib/vanagon/component/source/local.rb index 2fef9d73..aea27f07 100644 --- a/lib/vanagon/component/source/local.rb +++ b/lib/vanagon/component/source/local.rb @@ -77,7 +77,7 @@ def extension # rubocop:disable Lint/DuplicateMethods # @param tar [String] the tar command to use # @return [String, nil] command to extract the source # @raise [RuntimeError] an exception is raised if there is no known extraction method for @extension - def extract(tar = "tar") # rubocop:disable Metrics/AbcSize + def extract(tar = "tar") # Extension does not appear to be an archive, so "extract" is a no-op return ': nothing to extract' unless archive_extensions.include?(extension) diff --git a/lib/vanagon/driver.rb b/lib/vanagon/driver.rb index 2d7401ee..d0f794ca 100644 --- a/lib/vanagon/driver.rb +++ b/lib/vanagon/driver.rb @@ -22,7 +22,7 @@ def retry_count @retry_count ||= @project.retry_count || ENV["VANAGON_RETRY_COUNT"] || 1 end - def initialize(platform, project, options = {}) # rubocop:disable Metrics/AbcSize, Metrics/PerceivedComplexity + def initialize(platform, project, options = {}) @options = options @verbose = options[:verbose] || false @preserve = options[:preserve] || :'on-failure' @@ -114,7 +114,7 @@ def list_build_dependencies @project.components.map(&:build_requires).flatten.uniq - @project.components.map(&:name) end - def install_build_dependencies # rubocop:disable Metrics/AbcSize + def install_build_dependencies unless list_build_dependencies.empty? if @platform.build_dependencies && @platform.build_dependencies.command && !@platform.build_dependencies.command.empty? @engine.dispatch("#{@platform.build_dependencies.command} #{list_build_dependencies.join(' ')} #{@platform.build_dependencies.suffix}") @@ -126,7 +126,7 @@ def install_build_dependencies # rubocop:disable Metrics/AbcSize end end - def run # rubocop:disable Metrics/AbcSize, Metrics/PerceivedComplexity + def run # Simple sanity check for the project if @project.version.nil? or @project.version.empty? raise Vanagon::Error, "Project requires a version set, all is lost." diff --git a/lib/vanagon/engine/always_be_scheduling.rb b/lib/vanagon/engine/always_be_scheduling.rb index 7ef1e9e7..5837e2de 100644 --- a/lib/vanagon/engine/always_be_scheduling.rb +++ b/lib/vanagon/engine/always_be_scheduling.rb @@ -187,7 +187,7 @@ def read_vanagon_token(path = "~/.vanagon-token") # token: '456def789' # # @return [String, nil] the vmfloaty vmpooler token value - def read_vmfloaty_token(path = "~/.vmfloaty.yml") # rubocop:disable Metrics/AbcSize, Metrics/PerceivedComplexity + def read_vmfloaty_token(path = "~/.vmfloaty.yml") absolute_path = File.expand_path(path) return nil unless File.exist?(absolute_path) @@ -235,7 +235,7 @@ def select_target end # Attempt to provision a host from a specific pooler. - def select_target_from(pooler) # rubocop:disable Metrics/AbcSize + def select_target_from(pooler) request_object = build_request_object VanagonLogger.info "Requesting VMs with job_id: #{@saved_job_id}. Will poll for up to an hour." diff --git a/lib/vanagon/engine/docker.rb b/lib/vanagon/engine/docker.rb index 6b07d1b0..7c817f6f 100644 --- a/lib/vanagon/engine/docker.rb +++ b/lib/vanagon/engine/docker.rb @@ -42,7 +42,7 @@ def build_host_name # This method is used to obtain a vm to build upon using # a docker container. # @raise [Vanagon::Error] if a target cannot be obtained - def select_target # rubocop:disable Metrics/AbcSize + def select_target ssh_args = @platform.use_docker_exec ? '' : "-p #{@target_port}:22" extra_args = @platform.docker_run_args.nil? ? [] : @platform.docker_run_args diff --git a/lib/vanagon/patch.rb b/lib/vanagon/patch.rb index 1d5f08a2..6392c3cd 100644 --- a/lib/vanagon/patch.rb +++ b/lib/vanagon/patch.rb @@ -31,7 +31,7 @@ class Patch # @return [String] What step should this patch be applied to, one of ["unpack", "install"] attr_reader :after - def initialize(origin_path, component, options) # rubocop:disable Metrics/AbcSize,Metrics/PerceivedComplexity + def initialize(origin_path, component, options) valid_keys = %i[namespace destination strip fuzz after] bad_keys = options.each_key.reject { |k| valid_keys.include? k } diff --git a/lib/vanagon/platform.rb b/lib/vanagon/platform.rb index 1cea5dc3..8ba0780f 100644 --- a/lib/vanagon/platform.rb +++ b/lib/vanagon/platform.rb @@ -204,7 +204,7 @@ def add_group(user) # # @param user [Vanagon::Common::User] the user to create # @return [String] the commands required to add a user to the system - def add_user(user) # rubocop:disable Metrics/AbcSize + def add_user(user) cmd_args = ["'#{user.name}'"] cmd_args.unshift "--home '#{user.homedir}'" if user.homedir if user.shell @@ -239,7 +239,7 @@ def add_user(user) # rubocop:disable Metrics/AbcSize # # @param name [String] name of the platform # @return [Vanagon::Platform] the platform with the given name - def initialize(name) # rubocop:disable Metrics/PerceivedComplexity, Metrics/AbcSize + def initialize(name) @name = name @settings = {} @os_name = os_name diff --git a/lib/vanagon/platform/deb.rb b/lib/vanagon/platform/deb.rb index 54309ab9..4ec0c36f 100644 --- a/lib/vanagon/platform/deb.rb +++ b/lib/vanagon/platform/deb.rb @@ -5,7 +5,7 @@ class DEB < Vanagon::Platform # # @param project [Vanagon::Project] project to build a debian package of # @return [Array] list of commands required to build a debian package for the given project from a tarball - def generate_package(project) # rubocop:disable Metrics/AbcSize + def generate_package(project) target_dir = project.repo ? output_dir(project.repo) : output_dir copy_extensions = if project.source_artifacts '*.{deb,buildinfo,tar.gz,changes,dsc}' @@ -32,7 +32,7 @@ def generate_package(project) # rubocop:disable Metrics/AbcSize # @param name [String] name of the project # @param binding [Binding] binding to use in evaluating the packaging templates # @param project [Vanagon::Project] Vanagon::Project we are building for - def generate_packaging_artifacts(workdir, name, binding, project) # rubocop:disable Metrics/AbcSize + def generate_packaging_artifacts(workdir, name, binding, project) deb_dir = File.join(workdir, "debian") FileUtils.mkdir_p(deb_dir) diff --git a/lib/vanagon/platform/macos.rb b/lib/vanagon/platform/macos.rb index 3f53fff1..e9e07fbf 100644 --- a/lib/vanagon/platform/macos.rb +++ b/lib/vanagon/platform/macos.rb @@ -14,7 +14,7 @@ def install_build_dependencies(list_build_dependencies) # # @param project [Vanagon::Project] project to build a MacOS package of # @return [Array] list of commands required to build a MacOS package for the given project from a tarball - def generate_package(project) # rubocop:disable Metrics/AbcSize,Metrics/PerceivedComplexity + def generate_package(project) target_dir = project.repo ? output_dir(project.repo) : output_dir # Here we maintain backward compatibility with older vanagon versions @@ -163,7 +163,7 @@ def generate_package(project) # rubocop:disable Metrics/AbcSize,Metrics/Perceive # @param name [String] name of the project # @param binding [Binding] binding to use in evaluating the packaging templates # @param project [Vanagon::Project] Vanagon::Project we are building for - def generate_packaging_artifacts(workdir, name, binding, project) # rubocop:disable Metrics/AbcSize + def generate_packaging_artifacts(workdir, name, binding, project) resources_dir = File.join(workdir, "resources", "macos") FileUtils.mkdir_p(resources_dir) script_dir = File.join(workdir, "scripts") diff --git a/lib/vanagon/platform/solaris_10.rb b/lib/vanagon/platform/solaris_10.rb index 17bc5b62..16b33648 100644 --- a/lib/vanagon/platform/solaris_10.rb +++ b/lib/vanagon/platform/solaris_10.rb @@ -5,7 +5,7 @@ class Solaris10 < Vanagon::Platform # # @param project [Vanagon::Project] project to build a solaris package of # @return [Array] list of commands required to build a solaris package for the given project from a tarball - def generate_package(project) # rubocop:disable Metrics/AbcSize, Metrics/PerceivedComplexity + def generate_package(project) target_dir = project.repo ? output_dir(project.repo) : output_dir name_and_version = "#{project.name}-#{project.version}" pkg_name = package_name(project) @@ -146,7 +146,7 @@ def package_name(project) # # @param build_dependencies [Array] list of all build dependencies to install # @return [String] a command to install all of the build dependencies - def install_build_dependencies(build_dependencies) # rubocop:disable Metrics/AbcSize + def install_build_dependencies(build_dependencies) http = [] pkgutil = [] noasks = ["instance=overwrite", "partial=nocheck", "runlevel=nocheck", "idepend=nocheck", "rdepend=nocheck", "space=nocheck", "setuid=nocheck", "conflict=nocheck", "action=nocheck", "basedir=default"] diff --git a/lib/vanagon/platform/solaris_11.rb b/lib/vanagon/platform/solaris_11.rb index 52fd433d..51bc6720 100644 --- a/lib/vanagon/platform/solaris_11.rb +++ b/lib/vanagon/platform/solaris_11.rb @@ -5,7 +5,7 @@ class Solaris11 < Vanagon::Platform # # @param project [Vanagon::Project] project to build a solaris package of # @return [Array] list of commands required to build a solaris package for the given project from a tarball - def generate_package(project) # rubocop:disable Metrics/AbcSize + def generate_package(project) target_dir = project.repo ? output_dir(project.repo) : output_dir name_and_version = "#{project.name}-#{project.version}" pkg_name = package_name(project) diff --git a/lib/vanagon/platform/windows.rb b/lib/vanagon/platform/windows.rb index c11b159e..28bf2ee1 100644 --- a/lib/vanagon/platform/windows.rb +++ b/lib/vanagon/platform/windows.rb @@ -91,7 +91,7 @@ def copy_from_project(proj_resources, destination, verbose: false) # @param vanagon_root [String] Vanagon wix resources directory # @param destination [String] Destination directory # @param verbose [String] True or false - def merge_defaults_from_vanagon(vanagon_root, destination, verbose: false) # rubocop:disable Metrics/AbcSize + def merge_defaults_from_vanagon(vanagon_root, destination, verbose: false) # Will use this Pathname object for relative path calculations in loop below. vanagon_path = Pathname.new(vanagon_root) files = Dir.glob(File.join(vanagon_root, "**/*.*")) @@ -157,7 +157,7 @@ def generate_nuget_packaging_artifacts(workdir, name, binding) # @param project [Vanagon::Project] project to build a nuget package of # @return [Array] list of commands required to build a nuget package for # the given project from a tarball - def generate_nuget_package(project) # rubocop:disable Metrics/AbcSize + def generate_nuget_package(project) target_dir = project.repo ? output_dir(project.repo) : output_dir ["mkdir -p output/#{target_dir}", "mkdir -p $(tempdir)/#{project.name}/tools", @@ -180,7 +180,7 @@ def generate_nuget_package(project) # rubocop:disable Metrics/AbcSize # # @param project [Vanagon::Project] project to build a msi package of # @return [Array] list of commands required to build an msi package for the given project from a tarball - def generate_msi_package(project) # rubocop:disable Metrics/AbcSize, Metrics/PerceivedComplexity + def generate_msi_package(project) target_dir = project.repo ? output_dir(project.repo) : output_dir wix_extensions = "-ext WiXUtilExtension -ext WixUIExtension" # Heat command documentation at: http://wixtoolset.org/documentation/manual/v3/overview/heat.html diff --git a/lib/vanagon/project.rb b/lib/vanagon/project.rb index 5c50e462..3b1331a5 100644 --- a/lib/vanagon/project.rb +++ b/lib/vanagon/project.rb @@ -146,7 +146,7 @@ def self.load_project(name, configdir, platform, include_components = []) # @param name [String] name of the project # @param platform [Vanagon::Platform] platform for the project to be built for # @return [Vanagon::Project] the project with the given name and platform - def initialize(name, platform) # rubocop:disable Metrics/AbcSize + def initialize(name, platform) @name = name @components = [] @requires = [] @@ -562,7 +562,7 @@ def get_directories # Gets the highest level directories declared by the project # # @return [Array] the highest level directories that have been declared by the project - def get_root_directories # rubocop:disable Metrics/AbcSize + def get_root_directories dirs = get_directories.map { |dir| dir.path.split('/') } dirs.sort! { |dir1, dir2| dir1.length <=> dir2.length } ret_dirs = [] @@ -751,7 +751,7 @@ def metadata_merge(original, upstream) # @param platform [String] platform we're writing metadata for # @param additional_directories [String|Array[String]] additional # directories to write build_metadata...json to - def save_manifest_json(platform, additional_directories = nil) # rubocop:disable Metrics/AbcSize + def save_manifest_json(platform, additional_directories = nil) manifest = build_manifest_json metadata = metadata_merge(manifest, @upstream_metadata) @@ -821,7 +821,7 @@ def publish_yaml_settings(platform) # @param upstream_project_name [String] The name of the vanagon project to load # @param upstream_git_url [URI] The URL to clone this vanagon project from # @param upstream_git_branch [String] The branch of the vanagon project to clone from - def load_upstream_settings(upstream_project_name, upstream_git_url, upstream_git_branch) # rubocop:disable Metrics/AbcSize + def load_upstream_settings(upstream_project_name, upstream_git_url, upstream_git_branch) Dir.mktmpdir do |working_directory| upstream_source = Vanagon::Component::Source::Git.new(upstream_git_url, workdir: working_directory, ref: upstream_git_branch) upstream_source.fetch diff --git a/lib/vanagon/utilities.rb b/lib/vanagon/utilities.rb index bd3a7687..70724e66 100644 --- a/lib/vanagon/utilities.rb +++ b/lib/vanagon/utilities.rb @@ -50,7 +50,7 @@ def get_sum(file, type = 'md5') # @return [Net::HTTPAccepted] The response object # @raise [RuntimeError, Vanagon::Error] an exception is raised if the # action is not supported, or if there is a problem with the http request - def http_request_generic(url, type, payload = {}.to_json, header = nil) # rubocop:disable Metrics/AbcSize + def http_request_generic(url, type, payload = {}.to_json, header = nil) uri = URI.parse(url) http = Net::HTTP.new(uri.host, uri.port) http.use_ssl = true if uri.scheme == 'https' diff --git a/lib/vanagon/utilities/extra_files_signer.rb b/lib/vanagon/utilities/extra_files_signer.rb index 98006fed..31882b5b 100644 --- a/lib/vanagon/utilities/extra_files_signer.rb +++ b/lib/vanagon/utilities/extra_files_signer.rb @@ -47,7 +47,7 @@ def local_commands(project, mktmp, source_dir) [] end - def commands(project, mktemp, source_dir) # rubocop:disable Metrics/AbcSize + def commands(project, mktemp, source_dir) tempdir = nil commands = [] # Skip signing extra files if logging into the signing_host fails From b19533efc34836b7e5b139737a5a6d4ad745bed3 Mon Sep 17 00:00:00 2001 From: nmburgan <13688219+nmburgan@users.noreply.github.com> Date: Fri, 19 Sep 2025 11:50:54 -0700 Subject: [PATCH 2/2] Add --cachedir This caches downloaded files and git repos so when we are working on changes to packaging, we don't have to download them every time we run the build again. --- lib/vanagon/cli/build.rb | 2 ++ lib/vanagon/component.rb | 14 ++++++++------ lib/vanagon/component/source.rb | 3 +++ lib/vanagon/component/source/git.rb | 22 ++++++++++++++++++---- lib/vanagon/component/source/http.rb | 19 +++++++++++++++---- lib/vanagon/driver.rb | 5 +++-- lib/vanagon/project.rb | 7 ++++--- 7 files changed, 53 insertions(+), 19 deletions(-) diff --git a/lib/vanagon/cli/build.rb b/lib/vanagon/cli/build.rb index 7ad57a2a..76fede09 100644 --- a/lib/vanagon/cli/build.rb +++ b/lib/vanagon/cli/build.rb @@ -22,6 +22,7 @@ class Build < Vanagon::CLI managed automatically based on `keepwork` option -k, --keepwork RULE Rule for preserving local `workdir`: [Default: never] always, on-success, on-failure, never + --cachedir DIRECTORY Directory to cache source files and git repos -v, --verbose Only here for backwards compatibility. Does nothing. Engines: @@ -64,6 +65,7 @@ def options_translate(docopt_options) translations = { '--verbose' => :verbose, '--workdir' => :workdir, + '--cachedir' => :cachedir, '--remote-workdir' => :'remote-workdir', '--configdir' => :configdir, '--engine' => :engine, diff --git a/lib/vanagon/component.rb b/lib/vanagon/component.rb index c063047e..23361b1d 100644 --- a/lib/vanagon/component.rb +++ b/lib/vanagon/component.rb @@ -261,7 +261,7 @@ def mirrors # rubocop:disable Lint/DuplicateMethods # if #fetch is successful. def fetch_mirrors(options) mirrors.to_a.shuffle.each do |mirror| - VanagonLogger.info %(Attempting to fetch from mirror URL "#{mirror}") + VanagonLogger.info %(Attempting to fetch from mirror URL "#{mirror}"#{' unless already cached' if options[:cachedir]}) @source = Vanagon::Component::Source.source(mirror, **options) return true if source.fetch rescue Vanagon::InvalidSource @@ -290,7 +290,7 @@ def fetch_mirrors(options) # @return [Boolean] return True if the source can be retrieved, # or False otherwise def fetch_url(options) - VanagonLogger.info %(Attempting to fetch from canonical URL "#{url}") + VanagonLogger.info %(Attempting to fetch from canonical URL "#{url}"#{' unless already cached' if options[:cachedir]}) @source = Vanagon::Component::Source.source(url, **options) # Explicitly coerce the return value of #source.fetch, # because each subclass of Vanagon::Component::Source returns @@ -313,8 +313,9 @@ def deprecated_rewrite_url # makefile template # # @param workdir [String] working directory to put the source into - def get_source(workdir) - opts = options.merge({ workdir: workdir, dirname: dirname }) + # @param cachedir [String] directory to cache downloaded sources + def get_source(workdir, cachedir = nil) + opts = options.merge({ workdir: workdir, cachedir: cachedir, dirname: dirname }) if url || !mirrors.empty? if %w[y yes true 1].include? ENV.fetch('VANAGON_USE_MIRRORS', 'n').downcase fetch_mirrors(opts) || fetch_url(opts) @@ -358,10 +359,11 @@ def get_dependency_hash # Fetches secondary sources for the component. These are just dumped into the workdir currently. # # @param workdir [String] working directory to put the source into - def get_sources(workdir) # rubocop:disable Metrics/AbcSize + # @param cachedir [String] directory to cache downloaded sources + def get_sources(workdir, cachedir = nil) sources.each do |source| src = Vanagon::Component::Source.source( - source.url, workdir: workdir, ref: source.ref, sum: source.sum + source.url, workdir: workdir, cachedir: cachedir, ref: source.ref, sum: source.sum ) src.fetch src.verify diff --git a/lib/vanagon/component/source.rb b/lib/vanagon/component/source.rb index 4476059b..58d5cbb1 100644 --- a/lib/vanagon/component/source.rb +++ b/lib/vanagon/component/source.rb @@ -17,6 +17,7 @@ class << self # @param uri_instance [#to_s] URI of the source file (includes git@... style links) # @param options [Hash] hash of the options needed for the subtype # @param workdir [String] working directory to fetch the source into + # @param cacheedir [String] directory to cache source files and git repos # @return [Vanagon::Component::Source] the correct subtype for the given source def source(uri_instance, **options) # Sometimes the uri comes in as a string, but sometimes it's already been @@ -39,6 +40,7 @@ def source(uri_instance, **options) sum: options[:sum], ref: options[:ref], workdir: options[:workdir], + cachedir: options[:cachedir], dirname: options[:dirname], clone_options: options[:clone_options] end @@ -47,6 +49,7 @@ def source(uri_instance, **options) return Vanagon::Component::Source::Http.new uri, sum: options[:sum], workdir: options[:workdir], + cachedir: options[:cachedir], # Default sum_type is md5 if unspecified: sum_type: options[:sum_type] || "md5" end diff --git a/lib/vanagon/component/source/git.rb b/lib/vanagon/component/source/git.rb index cfea4d23..3475b176 100644 --- a/lib/vanagon/component/source/git.rb +++ b/lib/vanagon/component/source/git.rb @@ -13,7 +13,7 @@ class Vanagon class Component class Source class Git - attr_accessor :url, :log_url, :ref, :workdir, :clone_options + attr_accessor :url, :log_url, :ref, :workdir, :cachedir, :clone_options attr_reader :version, :default_options, :repo class << self @@ -94,7 +94,8 @@ def default_options # rubocop:disable Lint/DuplicateMethods # @param url [String] url of git repo to use as source # @param ref [String] ref to checkout from git repo # @param workdir [String] working directory to clone into - def initialize(url, workdir:, **options) + # @param cachedir [String] directory to cache cloned repos + def initialize(url, workdir:, cachedir: nil, **options) opts = default_options.merge(options.reject { |k, v| v.nil? }) # Ensure that #url returns a URI object @@ -103,6 +104,7 @@ def initialize(url, workdir:, **options) @ref = opts[:ref] @dirname = opts[:dirname] @workdir = File.realpath(workdir) + @cachedir = cachedir ? File.realpath(cachedir) : nil @clone_options = opts[:clone_options] ||= {} # We can test for Repo existence without cloning @@ -114,6 +116,11 @@ def initialize(url, workdir:, **options) # a side effect. def fetch begin + if @cachedir && File.directory?("#{@cachedir}/#{dirname}") + VanagonLogger.info "Using cached copy of Git repo '#{dirname}' in #{cachedir}" + FileUtils.cp_r("#{@cachedir}/#{dirname}", workdir) + @wascached = true + end @clone ||= ::Git.open(File.join(workdir, dirname)) @clone.fetch rescue ::Git::Error, ArgumentError @@ -156,6 +163,11 @@ def clone else @clone ||= ::Git.clone(url, dirname, path: workdir, **clone_options) end + if @cachedir && !@wascached + FileUtils.cp_r(File.join(workdir, dirname), cachedir) + @wascached = true + end + @clone end # Attempt to connect to whatever URL is provided and @@ -183,8 +195,10 @@ def refs # Clone a remote repo, make noise about it, and fail entirely # if we're unable to retrieve the remote repo def clone! - VanagonLogger.info "Cloning Git repo '#{log_url}'" - VanagonLogger.info "Successfully cloned '#{dirname}'" if clone + unless @wascached + VanagonLogger.info "Cloning Git repo '#{log_url}'" + VanagonLogger.info "Successfully cloned '#{dirname}'" if clone + end rescue ::Git::Error raise Vanagon::InvalidRepo, "Unable to clone from '#{log_url}'" end diff --git a/lib/vanagon/component/source/http.rb b/lib/vanagon/component/source/http.rb index 22d4f802..ae8ac2ba 100644 --- a/lib/vanagon/component/source/http.rb +++ b/lib/vanagon/component/source/http.rb @@ -12,7 +12,7 @@ class Http < Vanagon::Component::Source::Local include Vanagon::Utilities # Accessors :url, :file, :extension, :workdir, :cleanup are inherited from Local - attr_accessor :sum, :sum_type + attr_accessor :sum, :sum_type, :cachedir # Allowed checksum algorithms to use when validating files CHECKSUM_TYPES = %w[md5 sha1 sha256 sha512].freeze @@ -48,8 +48,9 @@ def valid_url?(target_url) # sum from # @param workdir [String] working directory to download into # @param sum_type [String] type of sum we are verifying + # @param cachedir [String] directory to cache downloaded sources # @raise [RuntimeError] an exception is raised is sum is nil - def initialize(url, sum:, workdir:, sum_type:, **options) + def initialize(url, sum:, workdir:, sum_type:, cachedir: nil, **options) unless sum fail "sum is required to validate the http source" end @@ -63,6 +64,7 @@ def initialize(url, sum:, workdir:, sum_type:, **options) @url = url @sum = sum @workdir = workdir + @cachedir = cachedir ? File.realpath(cachedir) : nil @sum_type = sum_type if Vanagon::Component::Source::Http.valid_url?(@sum) @@ -81,9 +83,14 @@ def initialize(url, sum:, workdir:, sum_type:, **options) end # Download the source from the url specified. Sets the full path to the - # file as @file and the @extension for the file as a side effect. + # file as @file and the @extension for the file as a side effect. Use + # the cached copy if it exists and is valid. def fetch @file = File.basename(URI.parse(@url).path) + if @cachedir && File.exist?(File.join(@cachedir, @file)) + VanagonLogger.info "Using cached copy of #{@file} in #{@cachedir}" + FileUtils.cp(File.join(@cachedir, @file), workdir) + end if File.exist?(File.join(workdir, file)) begin return if verify @@ -138,9 +145,13 @@ def download(target_url, target_file = nil, headers = { "Accept-Encoding" => "id location = URI.parse(response.header['location']) download(uri + location, target_file) when Net::HTTPSuccess - File.open(File.join(@workdir, target_file), 'w') do |io| + dir = @cachedir || @workdir + File.open(File.join(dir, target_file), 'w') do |io| response.read_body { |chunk| io.write(chunk) } end + if @cachedir + FileUtils.cp(File.join(@cachedir, target_file), File.join(@workdir, target_file)) + end else fail "Error: #{response.code.to_s}. Unable to get source from #{target_url}" end diff --git a/lib/vanagon/driver.rb b/lib/vanagon/driver.rb index d0f794ca..a00606cd 100644 --- a/lib/vanagon/driver.rb +++ b/lib/vanagon/driver.rb @@ -12,7 +12,7 @@ class Vanagon class Driver include Vanagon::Utilities - attr_accessor :platform, :project, :target, :workdir, :remote_workdir, :verbose, :preserve, :keepwork + attr_accessor :platform, :project, :target, :workdir, :cachedir, :remote_workdir, :verbose, :preserve, :keepwork def timeout @timeout ||= @project.timeout || ENV["VANAGON_TIMEOUT"] || 7200 @@ -27,6 +27,7 @@ def initialize(platform, project, options = {}) @verbose = options[:verbose] || false @preserve = options[:preserve] || :'on-failure' @workdir = options[:workdir] || Dir.mktmpdir + @cachedir = options[:cachedir] @keepwork = options[:keepwork] || :never @@configdir = options[:configdir] || File.join(Dir.pwd, "configs") @@ -144,7 +145,7 @@ def run Vanagon::Utilities.retry_with_timeout(retry_count, timeout) do install_build_dependencies end - @project.fetch_sources(workdir, retry_count, timeout) + @project.fetch_sources(workdir, retry_count, timeout, cachedir) @project.make_makefile(workdir) @project.make_bill_of_materials(workdir) diff --git a/lib/vanagon/project.rb b/lib/vanagon/project.rb index 3b1331a5..1b73e92a 100644 --- a/lib/vanagon/project.rb +++ b/lib/vanagon/project.rb @@ -227,17 +227,18 @@ def vendor_email_only # Collects all sources and patches into the provided workdir # # @param workdir [String] directory to stage sources into + # @param cachedir [String] directory to cache source files and git repos # @param retry_count [Integer] number of times to retry each fetch # @param timeout [Integer] How long to wait (in seconds) for each # fetch before aborting - def fetch_sources(workdir, retry_count = 1, timeout = 7200) + def fetch_sources(workdir, retry_count = 1, timeout = 7200, cachedir = nil) @components.each do |component| Vanagon::Utilities.retry_with_timeout(retry_count, timeout) do - component.get_source(workdir) + component.get_source(workdir, cachedir) end # Fetch secondary sources Vanagon::Utilities.retry_with_timeout(retry_count, timeout) do - component.get_sources(workdir) + component.get_sources(workdir, cachedir) end Vanagon::Utilities.retry_with_timeout(retry_count, timeout) do component.get_patches(workdir)