archzfs-plus-kernel-update-repo

#! /usr/bin/env ruby

# First off, force the Gemfile's path and setup bundler:
ENV[ "BUNDLE_GEMFILE" ] = "#{__dir__}/Gemfile"
require "bundler/setup"



require "fileutils"
require "open-uri"
require "optparse"
require "pathname"
require "ruby-progressbar"
require "timeout"
require "yaml"
require_relative "lib/pacman_db_parser"
require_relative "lib/pacman_package"



###############################################################################
### Constants                                                               ###
###############################################################################

# Main repo url:
ARCHZFS = "https://github.com/archzfs/archzfs/releases/download/experimental"
  .freeze



###############################################################################
### Utility functions                                                       ###
###############################################################################

# Exits removing the lock file.
def clean_exit( outcome )
  @lock.delete if !@lock.nil? and @lock.exist?
  exit outcome
end



# Downloader with progress bar.
def download_with_progressbar( url )
  file = url.split( "/" ).last

  puts "Downloading '#{file}'..."

  bar = ProgressBar.create starting_at: 0,
    total: nil,
    format: "  > %a%e %B %p%% %r KB/sec",
    rate_scale: lambda { |rate| rate / 1024 }

  return URI.open url,
    content_length_proc: Proc.new { |content_length|
      bar.total = content_length
    },
    progress_proc: Proc.new { |bytes_transferred|
      if bar.total and bar.total < bytes_transferred then
        bar.total = nil
      end
      bar.progress = bytes_transferred
    }
end



# Retrieves the configuration, handling eventual initial config creation and
# migration from older versions.
# Returns a hash.
def get_config
  # Config file name:
  config = Pathname.new "#{__dir__}/config.yml"

  # If the file exists, simply return its data as a hash:
  return YAML.load_file config if config.file?

  # If we get here we'll need the config sample:
  sample_file = "#{config}.sample"
  sample_config = YAML.load_file sample_file

  # If the older config file is present, migrate it:
  if File.exist? "kernels.conf" then
    old_config = File.open( "kernels.conf" ).read.split.grep /^linux/
    sample_config[ :packages ] = old_config
    FileUtils.rm "kernels.conf"
  end

  # Write the new config file:
  config.open "w" do |f|
    f.puts \
      "###",
      "# For a complete reference see: #{sample_file}",
      "###",
      "",
      sample_config.to_yaml
  end

  # Finally, return the config data:
  return sample_config
end



###############################################################################
### Main logic                                                              ###
###############################################################################

### Parse eventual parameters
###############################################################################
options = {
  force: false
}

# Parameters parser:
parser = OptionParser.new do |opts|
  opts.banner = "Usage: #{$0.split( /[\\\/]/ ).last} <switches>"

  opts.on "-f", "--force-refresh", "Forces a database refresh." do
    options[ :force ] = true
  end

  opts.on "-h", "--help", "Prints the help reference." do
    puts opts
    clean_exit 0
  end
end

parser.parse!



### Current versions retrieval
###############################################################################

# Repository's path:
repo = "#{__dir__}/repo/archzfs-plus-kernel"

# Ensure it does exist:
FileUtils.mkpath repo

# Lock file:
@lock = Pathname.new "#{repo}/lock"

# If the lock file already exists:
if @lock.exist? then
  # If the file has been created less than 5 minutes ago, it's valid and we
  # should quit:
  if Time.now - @lock.ctime  <= (60 * 5) then
    puts "Lock file #{@lock} present! Is someone else running this script?"
    exit -1
  # Otherwise it's a stale lock file which should be deleted:
  else
    @lock.delete
  end
end

# If we got here, create the lock file:
@lock.write "lock"

# Ensure signal interrupt won't break stuff:
[ "INT", "TERM" ].each do |s|
  Signal.trap s do
    clean_exit 0
  end
end

# Let's ensure any kind of error won't leave a lock file behind:
begin
  # Database name:
  FileUtils.mkpath repo
  database = Pathname.new "#{repo}/archzfs-plus-kernel.db.tar.xz"

  # Last chcek timestamp:
  last_check = Pathname.new "#{repo}/.last_check"

  # Fetch the configuration:
  config = get_config
  puts "Config is set to: #{config}"

  # We start off with no current packages and no extra kernel deps:
  current_pkgs = Hash.new
  extra_kernel_deps = Array.new

  # If the local database exists, read the current packages from it:
  if database.exist? then
    database_data = PacmanDbParser.new database.read

    # This is needed to avoid removing kernels needed by certain configurations
    # (eg. linux-rc, linux-git):
    extra_kernel_deps = database_data
      .kernel_deps(
        config[ :packages ]
        .map { |p| [ p, "zfs-linux-#{p.split( "-" ).last}" ] }
        .flatten
      ).map { |d| d.name }

    current_pkgs = database_data
      .pkgs
      .map { |p|
        [ p.name, p ]
      }.to_h

    puts "Current packages:"
    current_pkgs.each do |k,v|
      puts "  #{k} => #{v.version}"
    end
  end

  # Regex which matches all packages which should be kept:
  pkgs_to_keep = (config[ :packages ] + extra_kernel_deps).uniq.sort
  if pkgs_to_keep.empty? then
    pkgs_to_keep = /WILLNOTMATCH/
  else
    pkgs_to_keep =
      /^(((zfs-)?(#{pkgs_to_keep.join "|"})(-headers)?)$|zfs-utils.*)/
  end

  # Delete any current package which has been removed from the config and
  # remove it from the database:
  current_pkgs
    .keys
    .filter { |c| !c.match pkgs_to_keep }
    .each \
  do |pkg|
    puts "Removing package '#{pkg}'..."
    `repo-remove #{database} #{pkg}`
    Dir[ "#{repo}/#{pkg}-[0-9]*" ].map { |f| FileUtils.rm_rf f }
  end

  # This tells us if the configuration changed enough we are missing some
  # required packages:
  config_updated = (
    current_pkgs
      .keys
      .filter { |c| c.match pkgs_to_keep }.grep_v( /zfs/ ).empty? and
    !config[ :packages ].empty?
  )

  ### Available versions retrieval
  #############################################################################

  # If the file is new enough, just quit:
  if !config_updated and
     last_check.exist? and
     (age = Time.now - last_check.mtime) <
       config[ :seconds_between_updates ] and
     !options[ :force ] \
  then
    puts "Last check happened #{age.to_i} seconds ago, nothing to do."
    clean_exit 0
  end

  # Fetch archzfs' repo db:
  archzfs_db = PacmanDbParser.new(
    download_with_progressbar( "#{ARCHZFS}/archzfs.db" ).read
  )

  # Process the found packages:
  archzfs_pkgs = Array.new
  puts "Packages on archzfs (and kernel dependencies) which match config:"
  archzfs_db.data.each do |k,v|
    if k.match? pkgs_to_keep then
      puts "  #{v[ :pkg ].name} => #{v[ :pkg ].version}"
      archzfs_pkgs << v[ :pkg ]
      unless v[ :k_depends ].nil? then
        puts "    #{v[ :k_depends ].name} => #{v[ :k_depends ].version}"
        archzfs_pkgs << v[ :k_depends ]
      end
    end
  end
  archzfs_pkgs = archzfs_pkgs.sort.uniq

  ### Find packages to download and to delete
  #############################################################################

  to_delete = Array.new
  to_download = Array.new
  to_be_kept = Array.new

  archzfs_pkgs.each do |p|
    # If this package is not present in the current ones, it has to be
    # downloaded:
    if !current_pkgs.key? p.name then
      to_download << p

    # If the available version is different current one, it has to be
    # downloaded and the old one has to be deleted:
    elsif p.version != current_pkgs[ p.name ].version then
      to_download << p
      to_delete << current_pkgs[ p.name ]

    # Everything else should be kept as it is:
    else
      to_be_kept << current_pkgs[ p.name ]
    end
  end

  ### Update the repository
  #############################################################################

  # If there's nothing to download or delete:
  if to_delete.empty? and to_download.empty? then
    # Update the file's mtime, just to avoid issues:
    FileUtils.touch last_check

    # Show a message and quit:
    puts "Already up to date, nothing to do."

  # Else, it's update time:
  else
    # Delete the outdated packages:
    to_delete.map { |d| d.package_files.values.flatten }.flatten
      .map { |d| FileUtils.rm_rf "#{repo}/#{d}" }

    # Container of files to add to the repository:
    to_add = Array.new

    # Update the repository:
    to_download.each do |d|
      d.package_files.each do |name, file_names|
        file_names.each do |file_name|
          # Gotta reset this:
          current_attempt = 0

          # Let's be sure no accidental errors won't stop us:
          begin
            # Linux kernels download url:
            base_url = "https://archive.archlinux.org/packages/l/#{name}"
            # archzfs download url:
            unless d.name.match /^linux/ then
              base_url = "#{ARCHZFS}"
            end

            # Download the package:
            File.open "#{repo}/#{file_name}", "wb" do |f|
              f.write(
                download_with_progressbar( "#{base_url}/#{file_name}" ).read
              )

              # Mark it as to be added to the repo if needed:
              to_add << file_name unless file_name =~ /.sig$/
            end

          # Ensure we'll try again a few times:
          rescue => e
            # If we're within the threshold, sleep a while and retry:
            if current_attempt <= 15 then
              sleep 2 * current_attempt += 1
              retry

            # Otherwise we gotta stop:
            else
              # Clean up all downloaded files:
              file_names.map { |f| FileUtils.rm_rf "#{repo}/#{f}" }

              # Clean these up, just to be 100% sure:
              to_add = Array.new

              # Return this error:
              raise "#{e.class} => Failed to retrieve 'l/#{name}/#{file_name}'"
            end
          end
        end
      end
    end

    # Once we get here, we can safely add all packages to the repository:
    to_add.each do |package|
      puts "Adding '#{package}' to 'archzfs-plus-kernel'..."
      `repo-add #{database} #{repo}/#{package}`
    end

    # Once everything's ready, update the last_check file:
    FileUtils.touch last_check
  end

  # Fix repo permissions:
  `find "#{repo}" -type d -exec chmod 755 '{}' \\;`
  `find "#{repo}" -type f -exec chmod 644 '{}' \\;`

# If something bad shuld happen, let's log it:
rescue => e
  puts "Error during update: #{e.message}", e.backtrace
end

# Finally, remove the lock file:
clean_exit 0