Add --cachedir

.rubocop.yml

@@ -336,9 +336,6 @@ Metrics/ModuleLength:
 Metrics/ParameterLists:
   Enabled: false
 
-Metrics/PerceivedComplexity:
-  Enabled: true
-
 Naming/AccessorMethodName:
   Enabled: false
 
@@ -815,3 +812,9 @@ Style/FormatStringToken:
 
 Style/MultilineTernaryOperator:
   Enabled: false
+
+Metrics/AbcSize:
+  Enabled: false
+
+Metrics/PerceivedComplexity:
+  Enabled: false

lib/vanagon/cli.rb

@@ -42,7 +42,7 @@ class CLI
           help                print this help
     DOCOPT
 
-    def parse(argv) # rubocop:disable Metrics/AbcSize
+    def parse(argv)
       parsed_options = parse_options(argv)
       sub_command = parsed_options['<command>']
       sub_argv = parsed_options['<args>']

lib/vanagon/cli/build.rb

@@ -22,6 +22,7 @@ class Build < Vanagon::CLI
                                              managed automatically based on `keepwork` option
           -k, --keepwork RULE              Rule for preserving local `workdir`: [Default: never]
                                              always, on-success, on-failure, never
+          --cachedir DIRECTORY             Directory to cache source files and git repos
           -v, --verbose                    Only here for backwards compatibility. Does nothing.
 
         Engines:
@@ -40,7 +41,7 @@ def parse(argv)
         exit 1
       end
 
-      def run(options) # rubocop:disable Metrics/AbcSize
+      def run(options)
         project = options[:project_name]
         platform_list = options[:platforms].split(',')
         target_list = []
@@ -64,6 +65,7 @@ def options_translate(docopt_options)
         translations = {
           '--verbose' => :verbose,
           '--workdir' => :workdir,
+          '--cachedir' => :cachedir,
           '--remote-workdir' => :'remote-workdir',
           '--configdir' => :configdir,
           '--engine' => :engine,

lib/vanagon/cli/build_requirements.rb

@@ -32,7 +32,7 @@ def parse(argv)
         exit 1
       end
 
-      def run(options) # rubocop:disable Metrics/AbcSize
+      def run(options)
         platform = options[:platform]
         project = options[:project_name]
         driver = Vanagon::Driver.new(platform, project)

lib/vanagon/cli/list.rb

@@ -29,7 +29,7 @@ def output(list, use_spaces)
         return list
       end
 
-      def run(options) # rubocop:disable Metrics/AbcSize
+      def run(options)
         check_directories(options)
 
         default_list = topic_list(File.dirname(__FILE__), '..', 'platform', 'defaults')

lib/vanagon/component.rb

@@ -156,7 +156,7 @@ def self.load_component(name, configdir, settings, platform)
     # @param settings [Hash] the settings to be used in the component
     # @param platform [Vanagon::Platform] the platform to build the component for
     # @return [Vanagon::Component] the component with the given settings and platform
-    def initialize(name, settings, platform) # rubocop:disable Metrics/AbcSize
+    def initialize(name, settings, platform)
       @name = name
       @settings = settings
       @platform = platform
@@ -261,7 +261,7 @@ def mirrors # rubocop:disable Lint/DuplicateMethods
     #   if #fetch is successful.
     def fetch_mirrors(options)
       mirrors.to_a.shuffle.each do |mirror|
-        VanagonLogger.info %(Attempting to fetch from mirror URL "#{mirror}")
+        VanagonLogger.info %(Attempting to fetch from mirror URL "#{mirror}"#{' unless already cached' if options[:cachedir]})
         @source = Vanagon::Component::Source.source(mirror, **options)
         return true if source.fetch
       rescue Vanagon::InvalidSource
@@ -290,7 +290,7 @@ def fetch_mirrors(options)
     # @return [Boolean] return True if the source can be retrieved,
     #   or False otherwise
     def fetch_url(options)
-      VanagonLogger.info %(Attempting to fetch from canonical URL "#{url}")
+      VanagonLogger.info %(Attempting to fetch from canonical URL "#{url}"#{' unless already cached' if options[:cachedir]})
       @source = Vanagon::Component::Source.source(url, **options)
       # Explicitly coerce the return value of #source.fetch,
       # because each subclass of Vanagon::Component::Source returns
@@ -313,8 +313,9 @@ def deprecated_rewrite_url
     # makefile template
     #
     # @param workdir [String] working directory to put the source into
-    def get_source(workdir) # rubocop:disable Metrics/AbcSize, Metrics/PerceivedComplexity
-      opts = options.merge({ workdir: workdir, dirname: dirname })
+    # @param cachedir [String] directory to cache downloaded sources
+    def get_source(workdir, cachedir = nil)
+      opts = options.merge({ workdir: workdir, cachedir: cachedir, dirname: dirname })
       if url || !mirrors.empty?
         if %w[y yes true 1].include? ENV.fetch('VANAGON_USE_MIRRORS', 'n').downcase
           fetch_mirrors(opts) || fetch_url(opts)
@@ -358,10 +359,11 @@ def get_dependency_hash
     # Fetches secondary sources for the component. These are just dumped into the workdir currently.
     #
     # @param workdir [String] working directory to put the source into
-    def get_sources(workdir) # rubocop:disable Metrics/AbcSize
+    # @param cachedir [String] directory to cache downloaded sources
+    def get_sources(workdir, cachedir = nil)
       sources.each do |source|
         src = Vanagon::Component::Source.source(
-          source.url, workdir: workdir, ref: source.ref, sum: source.sum
+          source.url, workdir: workdir, cachedir: cachedir, ref: source.ref, sum: source.sum
         )
         src.fetch
         src.verify

lib/vanagon/component/dsl.rb

@@ -176,7 +176,7 @@ def conflicts(pkgname, version = nil)
       #        service_type [String] type of the service (network, application, system, etc)
       #        init_system [String] the init system on which to install service (sysv, systemd)
       #        link_target [String] executable service file should be linked to
-      def install_service(service_file, default_file = nil, service_name = @component.name, **options) # rubocop:disable Metrics/AbcSize
+      def install_service(service_file, default_file = nil, service_name = @component.name, **options)
         init_system = options[:init_system] || @component.platform.servicetype
         servicedir = @component.platform.get_service_dir(init_system)
 
@@ -246,7 +246,7 @@ def install_service(service_file, default_file = nil, service_name = @component.
       # @param owner  [String] owner of the file
       # @param group  [String] group owner of the file
       # # @param sudo   [Boolean] whether to use sudo to create the file and set mode
-      def install_file(source, target, mode: nil, owner: nil, group: nil, sudo: false) # rubocop:disable Metrics/AbcSize
+      def install_file(source, target, mode: nil, owner: nil, group: nil, sudo: false)
         sudo_check = sudo ? "#{sudo_bin} " : ""
 
         @component.install << "#{sudo_check}#{@component.platform.install} -d '#{File.dirname(target)}'"
@@ -421,7 +421,7 @@ def add_source(uri, options = {})
       # @param mode [String] octal mode to apply to the directory
       # @param owner [String] owner of the directory
       # @param group [String] group of the directory
-      def directory(dir, mode: nil, owner: nil, group: nil) # rubocop:disable Metrics/AbcSize
+      def directory(dir, mode: nil, owner: nil, group: nil)
         install_flags = ['-d']
         if @component.platform.is_windows?
           unless mode.nil? && owner.nil? && group.nil?

lib/vanagon/component/rules.rb

@@ -47,7 +47,7 @@ def initialize(component, project, platform)
       # in the returned rules.
       #
       # @return [Array<Makefile::Rule>]
-      def rules # rubocop:disable Metrics/AbcSize
+      def rules
         list_of_rules = [
           component_rule,
           unpack_rule,

lib/vanagon/component/source.rb

@@ -17,8 +17,9 @@ class << self
         # @param uri_instance [#to_s] URI of the source file (includes git@... style links)
         # @param options [Hash] hash of the options needed for the subtype
         # @param workdir [String] working directory to fetch the source into
+        # @param cacheedir [String] directory to cache source files and git repos
         # @return [Vanagon::Component::Source] the correct subtype for the given source
-        def source(uri_instance, **options) # rubocop:disable Metrics/AbcSize
+        def source(uri_instance, **options)
           # Sometimes the uri comes in as a string, but sometimes it's already been
           # coerced into a URI object. The individual source providers will turn
           # the passed uri into a URI object if needed, but for this method we
@@ -39,6 +40,7 @@ def source(uri_instance, **options) # rubocop:disable Metrics/AbcSize
               sum: options[:sum],
               ref: options[:ref],
               workdir: options[:workdir],
+              cachedir: options[:cachedir],
               dirname: options[:dirname],
               clone_options: options[:clone_options]
           end
@@ -47,6 +49,7 @@ def source(uri_instance, **options) # rubocop:disable Metrics/AbcSize
             return Vanagon::Component::Source::Http.new uri,
               sum: options[:sum],
               workdir: options[:workdir],
+              cachedir: options[:cachedir],
               # Default sum_type is md5 if unspecified:
               sum_type: options[:sum_type] || "md5"
           end

lib/vanagon/component/source/git.rb

@@ -13,7 +13,7 @@ class Vanagon
   class Component
     class Source
       class Git
-        attr_accessor :url, :log_url, :ref, :workdir, :clone_options
+        attr_accessor :url, :log_url, :ref, :workdir, :cachedir, :clone_options
         attr_reader :version, :default_options, :repo
 
         class << self
@@ -94,7 +94,8 @@ def default_options # rubocop:disable Lint/DuplicateMethods
         # @param url [String] url of git repo to use as source
         # @param ref [String] ref to checkout from git repo
         # @param workdir [String] working directory to clone into
-        def initialize(url, workdir:, **options) # rubocop:disable Metrics/AbcSize
+        # @param cachedir [String] directory to cache cloned repos
+        def initialize(url, workdir:, cachedir: nil, **options)
           opts = default_options.merge(options.reject { |k, v| v.nil? })
 
           # Ensure that #url returns a URI object
@@ -103,6 +104,7 @@ def initialize(url, workdir:, **options) # rubocop:disable Metrics/AbcSize
           @ref = opts[:ref]
           @dirname = opts[:dirname]
           @workdir = File.realpath(workdir)
+          @cachedir = cachedir ? File.realpath(cachedir) : nil
           @clone_options = opts[:clone_options] ||= {}
 
           # We can test for Repo existence without cloning
@@ -114,6 +116,11 @@ def initialize(url, workdir:, **options) # rubocop:disable Metrics/AbcSize
         # a side effect.
         def fetch
           begin
+            if @cachedir && File.directory?("#{@cachedir}/#{dirname}")
+              VanagonLogger.info "Using cached copy of Git repo '#{dirname}' in #{cachedir}"
+              FileUtils.cp_r("#{@cachedir}/#{dirname}", workdir)
+              @wascached = true
+            end
             @clone ||= ::Git.open(File.join(workdir, dirname))
             @clone.fetch
           rescue ::Git::Error, ArgumentError
@@ -156,6 +163,11 @@ def clone
           else
             @clone ||= ::Git.clone(url, dirname, path: workdir, **clone_options)
           end
+          if @cachedir && !@wascached
+            FileUtils.cp_r(File.join(workdir, dirname), cachedir)
+            @wascached = true
+          end
+          @clone
         end
 
         # Attempt to connect to whatever URL is provided and
@@ -183,8 +195,10 @@ def refs
         # Clone a remote repo, make noise about it, and fail entirely
         # if we're unable to retrieve the remote repo
         def clone!
-          VanagonLogger.info "Cloning Git repo '#{log_url}'"
-          VanagonLogger.info "Successfully cloned '#{dirname}'" if clone
+          unless @wascached
+            VanagonLogger.info "Cloning Git repo '#{log_url}'"
+            VanagonLogger.info "Successfully cloned '#{dirname}'" if clone
+          end
         rescue ::Git::Error
           raise Vanagon::InvalidRepo, "Unable to clone from '#{log_url}'"
         end

lib/vanagon/component/source/http.rb

@@ -12,7 +12,7 @@ class Http < Vanagon::Component::Source::Local
         include Vanagon::Utilities
 
         # Accessors :url, :file, :extension, :workdir, :cleanup are inherited from Local
-        attr_accessor :sum, :sum_type
+        attr_accessor :sum, :sum_type, :cachedir
 
         # Allowed checksum algorithms to use when validating files
         CHECKSUM_TYPES = %w[md5 sha1 sha256 sha512].freeze
@@ -48,8 +48,9 @@ def valid_url?(target_url)
         #                     sum from
         # @param workdir [String] working directory to download into
         # @param sum_type [String] type of sum we are verifying
+        # @param cachedir [String] directory to cache downloaded sources
         # @raise [RuntimeError] an exception is raised is sum is nil
-        def initialize(url, sum:, workdir:, sum_type:, **options)
+        def initialize(url, sum:, workdir:, sum_type:, cachedir: nil, **options)
           unless sum
             fail "sum is required to validate the http source"
           end
@@ -63,6 +64,7 @@ def initialize(url, sum:, workdir:, sum_type:, **options)
           @url = url
           @sum = sum
           @workdir = workdir
+          @cachedir = cachedir ? File.realpath(cachedir) : nil
           @sum_type = sum_type
 
           if Vanagon::Component::Source::Http.valid_url?(@sum)
@@ -81,9 +83,14 @@ def initialize(url, sum:, workdir:, sum_type:, **options)
         end
 
         # Download the source from the url specified. Sets the full path to the
-        # file as @file and the @extension for the file as a side effect.
+        # file as @file and the @extension for the file as a side effect. Use
+        # the cached copy if it exists and is valid.
         def fetch
           @file = File.basename(URI.parse(@url).path)
+          if @cachedir && File.exist?(File.join(@cachedir, @file))
+            VanagonLogger.info "Using cached copy of #{@file} in #{@cachedir}"
+            FileUtils.cp(File.join(@cachedir, @file), workdir)
+          end
           if File.exist?(File.join(workdir, file))
             begin
               return if verify
@@ -118,7 +125,7 @@ def verify
         # Downloads the file from @url into the @workdir
         # @param target_url [String, URI, Addressable::URI] url of an http source to retrieve with GET
         # @raise [RuntimeError, Vanagon::Error] an exception is raised if the URI scheme cannot be handled
-        def download(target_url, target_file = nil, headers = { "Accept-Encoding" => "identity" }) # rubocop:disable Metrics/AbcSize
+        def download(target_url, target_file = nil, headers = { "Accept-Encoding" => "identity" })
           uri = URI.parse(target_url.to_s)
           target_file ||= File.basename(uri.path)
 
@@ -138,9 +145,13 @@ def download(target_url, target_file = nil, headers = { "Accept-Encoding" => "id
                 location = URI.parse(response.header['location'])
                 download(uri + location, target_file)
               when Net::HTTPSuccess
-                File.open(File.join(@workdir, target_file), 'w') do |io|
+                dir = @cachedir || @workdir
+                File.open(File.join(dir, target_file), 'w') do |io|
                   response.read_body { |chunk| io.write(chunk) }
                 end
+                if @cachedir
+                  FileUtils.cp(File.join(@cachedir, target_file), File.join(@workdir, target_file))
+                end
               else
                 fail "Error: #{response.code.to_s}. Unable to get source from #{target_url}"
               end

lib/vanagon/component/source/local.rb

@@ -77,7 +77,7 @@ def extension # rubocop:disable Lint/DuplicateMethods
         # @param tar [String] the tar command to use
         # @return [String, nil] command to extract the source
         # @raise [RuntimeError] an exception is raised if there is no known extraction method for @extension
-        def extract(tar = "tar") # rubocop:disable Metrics/AbcSize
+        def extract(tar = "tar")
           # Extension does not appear to be an archive, so "extract" is a no-op
           return ': nothing to extract' unless archive_extensions.include?(extension)
 

lib/vanagon/driver.rb

@@ -12,7 +12,7 @@ class Vanagon
   class Driver
     include Vanagon::Utilities
 
-    attr_accessor :platform, :project, :target, :workdir, :remote_workdir, :verbose, :preserve, :keepwork
+    attr_accessor :platform, :project, :target, :workdir, :cachedir, :remote_workdir, :verbose, :preserve, :keepwork
 
     def timeout
       @timeout ||= @project.timeout || ENV["VANAGON_TIMEOUT"] || 7200
@@ -22,11 +22,12 @@ def retry_count
       @retry_count ||= @project.retry_count || ENV["VANAGON_RETRY_COUNT"] || 1
     end
 
-    def initialize(platform, project, options = {}) # rubocop:disable Metrics/AbcSize, Metrics/PerceivedComplexity
+    def initialize(platform, project, options = {})
       @options = options
       @verbose = options[:verbose] || false
       @preserve = options[:preserve] || :'on-failure'
       @workdir = options[:workdir] || Dir.mktmpdir
+      @cachedir = options[:cachedir]
       @keepwork = options[:keepwork] || :never
 
       @@configdir = options[:configdir] || File.join(Dir.pwd, "configs")
@@ -114,7 +115,7 @@ def list_build_dependencies
       @project.components.map(&:build_requires).flatten.uniq - @project.components.map(&:name)
     end
 
-    def install_build_dependencies # rubocop:disable Metrics/AbcSize
+    def install_build_dependencies
       unless list_build_dependencies.empty?
         if @platform.build_dependencies && @platform.build_dependencies.command && [email protected]_dependencies.command.empty?
           @engine.dispatch("#{@platform.build_dependencies.command} #{list_build_dependencies.join(' ')} #{@platform.build_dependencies.suffix}")
@@ -126,7 +127,7 @@ def install_build_dependencies # rubocop:disable Metrics/AbcSize
       end
     end
 
-    def run # rubocop:disable Metrics/AbcSize, Metrics/PerceivedComplexity
+    def run
       # Simple sanity check for the project
       if @project.version.nil? or @project.version.empty?
         raise Vanagon::Error, "Project requires a version set, all is lost."
@@ -144,7 +145,7 @@ def run # rubocop:disable Metrics/AbcSize, Metrics/PerceivedComplexity
       Vanagon::Utilities.retry_with_timeout(retry_count, timeout) do
         install_build_dependencies
       end
-      @project.fetch_sources(workdir, retry_count, timeout)
+      @project.fetch_sources(workdir, retry_count, timeout, cachedir)
 
       @project.make_makefile(workdir)
       @project.make_bill_of_materials(workdir)

lib/vanagon/engine/always_be_scheduling.rb

@@ -187,7 +187,7 @@ def read_vanagon_token(path = "~/.vanagon-token")
       #     token: '456def789'
       #
       # @return [String, nil] the vmfloaty vmpooler token value
-      def read_vmfloaty_token(path = "~/.vmfloaty.yml") # rubocop:disable Metrics/AbcSize, Metrics/PerceivedComplexity
+      def read_vmfloaty_token(path = "~/.vmfloaty.yml")
         absolute_path = File.expand_path(path)
         return nil unless File.exist?(absolute_path)
 
@@ -235,7 +235,7 @@ def select_target
       end
 
       # Attempt to provision a host from a specific pooler.
-      def select_target_from(pooler) # rubocop:disable Metrics/AbcSize
+      def select_target_from(pooler)
         request_object = build_request_object
 
         VanagonLogger.info "Requesting VMs with job_id: #{@saved_job_id}.  Will poll for up to an hour."