diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 996a3ec5e..02485f503 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -15,29 +15,6 @@ on: pull_request: jobs: - check-docs: - name: Docs up to date - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - show-progress: false - - name: terraform-docs - env: - TERRAFORM_DOCS_VERSION: '0.16.0' - run: | - echo "Downloading terraform-docs-v${TERRAFORM_DOCS_VERSION} binary." - curl -Lsf \ - "https://github.com/terraform-docs/terraform-docs/releases/download/v${TERRAFORM_DOCS_VERSION}/terraform-docs-v${TERRAFORM_DOCS_VERSION}-linux-amd64.tar.gz" | tar -zxf - terraform-docs - - echo "Checking whether auto-generated README files are up to date." - PATH="$PATH:$PWD" tools/update-docs.sh - - if ! git diff --exit-code; then - echo "The documentation isn't up to date. Run tools/update-docs.sh and commit the results." - exit 1 - fi - shellcheck: name: Shellcheck runs-on: ubuntu-latest @@ -61,6 +38,8 @@ jobs: with: path: ~/tfenv key: tfenv-${{ runner.os }} + - name: Validate JSON + run: find . -type -f -name '*.json' | xargs tools/json-check.sh - name: terraform install env: TFENV_VERSION: '3.0.0' @@ -89,27 +68,3 @@ jobs: echo "Run terraform fmt -recursive . to fix formatting." exit 1 fi - - other-linters: - name: Other linters - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - show-progress: false - - uses: ruby/setup-ruby@v1 - - uses: actions/cache@v3 - with: - path: vendor/bundle - key: bundle-${{ hashFiles('**/Gemfile.lock') }} - restore-keys: bundle - - name: bundle install - run: bundle install -j8 --deployment --without development - - name: RSpec - run: bundle exec rspec spec/validate_resources_spec.rb - - name: Lint resource names - run: bundle exec lib/resource_name_lint.rb - - name: Validate JSON - run: find . -type -f -name '*.json' | xargs tools/json-check.sh - - name: Check ADRs - run: tools/adr-check.sh diff --git a/.ruby-version b/.ruby-version deleted file mode 100644 index 944880fa1..000000000 --- a/.ruby-version +++ /dev/null @@ -1 +0,0 @@ -3.2.0 diff --git a/Gemfile b/Gemfile deleted file mode 100644 index bdb03d68b..000000000 --- a/Gemfile +++ /dev/null @@ -1,8 +0,0 @@ -source 'https://rubygems.org' - -gem 'rspec' -gem 'colorize' - -group :development do - gem 'pry' -end diff --git a/Gemfile.lock b/Gemfile.lock deleted file mode 100644 index f12327301..000000000 --- a/Gemfile.lock +++ /dev/null @@ -1,31 +0,0 @@ -GEM - remote: https://rubygems.org/ - specs: - coderay (1.1.3) - colorize (1.1.0) - diff-lcs (1.5.0) - method_source (1.0.0) - pry (0.14.2) - coderay (~> 1.1) - method_source (~> 1.0) - rspec (3.12.0) - rspec-core (~> 3.12.0) - rspec-expectations (~> 3.12.0) - rspec-mocks (~> 3.12.0) - rspec-core (3.12.2) - rspec-support (~> 3.12.0) - rspec-expectations (3.12.3) - diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.12.0) - rspec-mocks (3.12.6) - diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.12.0) - rspec-support (3.12.1) - -PLATFORMS - ruby - -DEPENDENCIES - colorize - pry - rspec diff --git a/jenkins.sh b/jenkins.sh deleted file mode 100755 index 0462daefc..000000000 --- a/jenkins.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/bash -# -# This script invokes the build-terraform-project.sh tool to deploy the code. -# -set -e - -if [[ ! $(command -v sops) ]]; then - echo "sops not installed, exiting" - exit 1 -fi - -# Set the Terraform version to enable testing new versions. -if [[ $TERRAFORM_VERSION != '' ]]; then - BIN='tmp-bin' - - echo "Creating temporary bin directory" - rm -rf $BIN && mkdir $BIN && cd $BIN - - echo "Downloading Terraform ${TERRAFORM_VERSION}" - - wget https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip - wget https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_SHA256SUMS - - SHASUM256=$(shasum -a 256 terraform_${TERRAFORM_VERSION}_linux_amd64.zip |cut -d ' ' -f1) - - echo "Checking integrity of file" - grep -q $SHASUM256 terraform_${TERRAFORM_VERSION}_SHA256SUMS || (echo "SHASUM256 does not match, exiting"; exit 1) - - echo "Checked, unpacking" - unzip terraform_${TERRAFORM_VERSION}_linux_amd64.zip - - cd .. - - echo "Setting path:" - PATH=$(pwd)/$BIN:$PATH - echo $PATH - - echo "Terraform binary: $(command -v terraform)" -fi - -rm -rf govuk-aws-data - -if [[ "$GOVUK_AWS_DATA_BRANCH" == "" ]]; then - GOVUK_AWS_DATA_BRANCH="main" -fi - -if [[ $COMMAND == "plan" && $GOVUK_AWS_BRANCH != "main" ]]; then -echo -e "\e[31m===============================================================\e[0m" -echo -e "\e[31mWARNING: you have run a 'plan' of a branch of govuk-aws that may be a number of commits behind 'main'.\nApplying this branch may lead to unintended infrastructural changes.\nYou should rebase your branch before proceeding.\e[0m" -echo -e "\e[31m===============================================================\e[0m" -fi - -if [[ $COMMAND == "plan" && $GOVUK_AWS_DATA_BRANCH != "main" ]]; then -echo -e "\e[31m===============================================================\e[0m" -echo -e "\e[31mWARNING: you have run a 'plan' of a branch of govuk-aws-data that may be a number of commits behind 'main'.\nApplying this branch may lead to unintended infrastructural changes.\nYou should rebase your branch before proceeding.\e[0m" -echo -e "\e[31m===============================================================\e[0m" -fi - -echo "Cloning govuk-aws-data $GOVUK_AWS_DATA_BRANCH" -git clone --single-branch --branch "$GOVUK_AWS_DATA_BRANCH" git@github.com:alphagov/govuk-aws-data.git - -case $COMMAND in - 'apply') EXTRA='-auto-approve';; - 'plan (destroy)') COMMAND='plan'; EXTRA='-detailed-exitcode -destroy';; - # This flag must be -auto-approve for terraform v1.0+ - # TODO: either also support -force for terraform v0.x, or update remaining - # projects that require terraform v0.x - 'destroy') EXTRA='-auto-approve';; - 'plan') EXTRA='-detailed-exitcode';; -esac - -tools/build-terraform-project.sh -d './govuk-aws-data/data' \ - -c $COMMAND \ - -p $PROJECT \ - -s $STACKNAME \ - -e $ENVIRONMENT \ - -- $EXTRA diff --git a/lib/resource_name_lint.rb b/lib/resource_name_lint.rb deleted file mode 100755 index b59bdcda8..000000000 --- a/lib/resource_name_lint.rb +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env ruby - -require_relative 'validate_resources' -require 'yaml' - -resources = YAML.load_file('lint.yaml')['resources'] - -options = { -# :verbose => true, -# :debug => true, -# :matched => true, -# :colours => true, -} - -$errors = [] - -resources.each do |key, val| - val.each do |v| - $errors << ValidateResources.new.run(File.expand_path('terraform'), key, v, options) - end -end - -exit 1 if $errors.include?(false) diff --git a/lib/validate_resources.rb b/lib/validate_resources.rb deleted file mode 100755 index 3035ff46d..000000000 --- a/lib/validate_resources.rb +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env ruby - -require 'colorize' - -class ValidateResources - def validate_resource_name(file, resource_name, rule, options = {}) - abort("Cannot find #{file}") unless File.exist?(file) - - @failed_resources = [] - File.open(file).readlines.each do |line| - line.chomp! - - puts " -- processing [#{line}]" if options[:debug] - next unless line.match(/\s*resource\s+"#{resource_name}"/) - - m = line.match(/\s*resource\s+"#{resource_name}"\s+"#{rule}"/) - resource = line.match(/\s*resource\s+"#{resource_name}"\s+"(?.+)"/) - - if m.nil? - @failed_resources << resource[:name] - else - puts "Matched #{line}" if options[:matched] - end - end - - if @failed_resources.empty? - return {} - else - { resource_name => @failed_resources } - end - end - - def run(directory, resource, rule, options = {}) - abort("Cannot find #{directory}") unless Dir.exist?(directory) - $errors = [] - - Dir["#{directory}/**/*.tf"].each do |file| - puts "Working on [#{file}]" if options[:verbose] - - file_status = validate_resource_name(file, resource, rule, options) - - unless file_status.empty? - puts "#{file} failed on #{rule}:" - file_status.each do |key, val| - if options[:colours] - puts " Resource #{key.colorize(:red)}" - else - puts " Resource #{key}" - end - val.each do |r| - if options[:colours] - puts " #{r.colorize(:yellow)}" - else - puts " #{r}" - end - end - end - $errors << false - end - end - - $errors.include?(false) ? 'false' : 'true' - end -end diff --git a/lint.yaml b/lint.yaml deleted file mode 100644 index 5559f967c..000000000 --- a/lint.yaml +++ /dev/null @@ -1,4 +0,0 @@ -resources: - aws_security_group_rule: - # source_ingress_destination_service - - '(?(?[^_]+)_(?(ingress|egress))_(?.+)_(?.+))' diff --git a/spec/aws_security_group_rule.tf b/spec/aws_security_group_rule.tf deleted file mode 100644 index 3a528c379..000000000 --- a/spec/aws_security_group_rule.tf +++ /dev/null @@ -1,25 +0,0 @@ -resource "aws_security_group_rule" "foo_ingress_bar_http" { - type = "ingress" - from_port = 80 - to_port = 80 - protocol = "tcp" - - # Which security group is the rule assigned to - security_group_id = "foo" - - # Which security group can use this rule - source_security_group_id = "bar" -} - -resource "aws_security_group_rule" "allow_yellow_from_blue" { - type = "ingress" - from_port = 80 - to_port = 80 - protocol = "tcp" - - # Which security group is the rule assigned to - security_group_id = "foo" - - # Which security group can use this rule - source_security_group_id = "bar" -} diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb deleted file mode 100644 index c6c1c1b3c..000000000 --- a/spec/spec_helper.rb +++ /dev/null @@ -1,8 +0,0 @@ -RSpec.configure do |config| - # Enable flags like --only-failures and --next-failure - config.example_status_persistence_file_path = '.rspec_status' - - config.expect_with :rspec do |c| - c.syntax = :expect - end -end diff --git a/spec/validate_resources_spec.rb b/spec/validate_resources_spec.rb deleted file mode 100644 index 9d9de0b60..000000000 --- a/spec/validate_resources_spec.rb +++ /dev/null @@ -1,18 +0,0 @@ -require 'spec_helper' -require_relative '../lib/validate_resources' - -RSpec.describe ValidateResources do - describe '::validate_resource_name' do - context 'aws_security_group_rule' do - it 'should be in format of source_ingress_dest_service' do - resource = 'aws_security_group_rule' - rule = /(?(?[^_]+)_(?(ingress|egress))_(?.+)_(?.+))/ - expect(ValidateResources.new.validate_resource_name('spec/aws_security_group_rule.tf', resource, rule)).to eq({ - 'aws_security_group_rule' => [ - 'allow_yellow_from_blue' - ] - }) - end - end - end -end diff --git a/tools/adr-check.sh b/tools/adr-check.sh deleted file mode 100755 index c52bf99d3..000000000 --- a/tools/adr-check.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -for i in $(find docs/architecture/decisions/*.md |cut -d"-" -f1 |cut -d"/" -f4); do - if [[ $(find docs/architecture/decisions/ -name "*.md" |grep -c ${i}) -gt 1 ]]; then - echo "There is a conflict in ADR naming:" - find docs/architecture/decisions -name "*.md" |grep $i - exit 1 - fi -done diff --git a/tools/bulk-plan.rb b/tools/bulk-plan.rb deleted file mode 100755 index e7b1421d3..000000000 --- a/tools/bulk-plan.rb +++ /dev/null @@ -1,131 +0,0 @@ -#!/usr/bin/env ruby - -require 'optparse' - -USAGE_BANNER = 'Usage: bulk-plan.rb ENVIRONMENT GLOBS ...' - -def parse_options - options = {} - - @option_parser = OptionParser.new do |opts| - opts.banner = USAGE_BANNER - - opts.on('-v', '--verbose', 'Enable more detailed logging') do - $verbose = true - end - opts.on('-d', '--dry-run', 'Just print what would be done') do - options[:dry_run] = true - end - - opts.on('-h', '--help', 'Prints usage information and examples') do - # TODO - exit - end - end - - @option_parser.parse! - - options -end - -def bulk_plan(environment, globs, dry_run: false) - files = globs.flat_map do |glob| - Dir.glob(glob, base: 'terraform/projects') - end - - projects_and_stacks = {} - - files.sort.each do |project| - backend_filename_parts = Dir.glob( - "*.backend", - base: "terraform/projects/#{project}" - ).map { |backend_filename| backend_filename.split(".") } - - - relevant_backend_files = backend_filename_parts.select do |parts| - parts[0] == environment - end - - stacks = relevant_backend_files.map do |parts| - parts[1] - end - - projects_and_stacks[project] = { stacks: stacks } - end - - puts "Matched projects, with no relevant stacks:" - projects_and_stacks.each do |project, data| - next unless data[:stacks].empty? - - puts " - #{project}" - end - puts - - puts "Matched projects, with stacks:" - projects_and_stacks.each do |project, data| - next if data[:stacks].empty? - - if data[:stacks].length == 1 - puts " - #{project.ljust(40)}#{data[:stacks][0]}" - else - puts " - #{project}" - data[:stacks].each do |stack| - puts " - #{stack}" - end - end - end - puts - - projects_and_stacks.each do |project, data| - next if data[:stacks].empty? - - data[:stacks].each do |stack| - command = [ - "ruby", - "tools/deploy.rb", - stack, - project, - environment, - "plan", - ] - - if dry_run - puts "would run: #{command.join(' ')}" - else - puts "running: #{command.join(' ')}" - - command_succeeded = system(*command) - - unless command_succeeded - puts "command failed" - - exit 1 - end - end - end - end -end - -def main - options = parse_options - - environment, *globs = *ARGV - - unless environment - puts "error: you must specify the environment" - puts - puts @option_parser.help - exit 1 - end - - unless globs - puts "error: you must specify the projects glob" - puts - puts @option_parser.help - exit 1 - end - - bulk_plan(environment, globs, dry_run: options[:dry_run]) -end - -main diff --git a/tools/deploy.rb b/tools/deploy.rb deleted file mode 100755 index f965d37ee..000000000 --- a/tools/deploy.rb +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env ruby - -require 'net/http' -require 'net/https' -require 'json' -require 'uri' - -# Fields from the command line -stack, project, environment, command, *rest= ARGV - -abort("too many arguments: #{rest}") unless rest.empty? - -# Valid values for each field -valid_stacks = %w(blue green govuk).freeze -valid_projects = Dir.chdir("#{Dir.home}/govuk/govuk-aws/terraform/projects") { Dir.glob('*').select { |f| File.directory? f } }.freeze -valid_environments = %w(integration staging production test tools).freeze -valid_commands = %w(plan apply plan-destroy destroy).freeze - -usage = 'Usage: GITHUB_USERNAME=... GITHUB_TOKEN=... ruby deploy.rb ' - -abort("GITHUB_USERNAME environment variable must be set\n#{usage}") unless ENV.has_key?('GITHUB_USERNAME') -abort("GITHUB_TOKEN environment variable must be set\n#{usage}") unless ENV.has_key?('GITHUB_TOKEN') -abort("stack must be one of #{valid_stacks.join(', ')}\n#{usage}") unless valid_stacks.include?(stack) -abort("project must be one of #{valid_projects.join(', ')}\n#{usage}") unless valid_projects.include?(project) -abort("environment must be one of #{valid_environments.join(', ')}\n#{usage}") unless valid_environments.include?(environment) -abort("command must be one of #{valid_commands.join(', ')}\n#{usage}") unless valid_commands.include?(command) - -# Make sure the user is happy to go ahead -puts "You're about to #{command} the #{stack}/#{project} project in #{environment}" -print 'Do you want to go ahead? [y/N] ' -continue = STDIN.gets.chomp -abort('Build aborted') unless continue.downcase == 'y' - -# Jenkins details -jenkins_url = 'https://deploy.integration.publishing.service.gov.uk'.freeze -jenkins_crumb_issuer_path = '/crumbIssuer/api/json'.freeze -jenkins_job_path = '/job/Deploy_Terraform_GOVUK_AWS/buildWithParameters'.freeze -jenkins_crumb_issuer_uri = URI.parse("#{jenkins_url}#{jenkins_crumb_issuer_path}") -jenkins_job_uri = URI.parse("#{jenkins_url}#{jenkins_job_path}") - -puts 'Using AWS credentials from the environment as set by gds-cli...' - -# Get a Jenkins "crumb" to authenticate the next request -puts 'Requesting Jenkins crumb...' -jenkins_crumb_http = Net::HTTP.new(jenkins_crumb_issuer_uri.host, jenkins_crumb_issuer_uri.port) -jenkins_crumb_http.use_ssl = true -jenkins_crumb_request = Net::HTTP::Get.new(jenkins_crumb_issuer_uri.path) -jenkins_crumb_request.basic_auth(ENV['GITHUB_USERNAME'], ENV['GITHUB_TOKEN']) -jenkins_crumb_response = jenkins_crumb_http.request(jenkins_crumb_request) -abort('Could not get crumb from Jenkins') unless jenkins_crumb_response.code == '200' -jenkins_crumb = JSON.parse(jenkins_crumb_response.body) - -if command == 'plan-destroy' - # The Jenkins job uses a slightly different command - command = 'plan (destroy)' -end - -# Make a request to the Jenkins API to queue the build -puts 'Queuing Jenkins job...' -jenkins_job_http = Net::HTTP.new(jenkins_job_uri.host, jenkins_job_uri.port) -jenkins_job_http.use_ssl = true -jenkins_job_request = Net::HTTP::Post.new(jenkins_job_uri.path) -jenkins_job_request.basic_auth(ENV['GITHUB_USERNAME'], ENV['GITHUB_TOKEN']) -jenkins_job_request.set_form_data({ - 'AWS_ACCESS_KEY_ID' => ENV['AWS_ACCESS_KEY_ID'], - 'AWS_SECRET_ACCESS_KEY' => ENV['AWS_SECRET_ACCESS_KEY'], - 'AWS_SESSION_TOKEN' => ENV['AWS_SESSION_TOKEN'], - 'COMMAND' => command, - 'ENVIRONMENT' => environment, - 'STACKNAME' => stack, - 'PROJECT' => project -}) -jenkins_job_request[jenkins_crumb['crumbRequestField']] = jenkins_crumb['crumb'] -jenkins_job_response = jenkins_job_http.request(jenkins_job_request) - -abort('Could not queue Jenkins job') unless jenkins_job_response.code == '201' - -puts 'Jenkins job queued. View it at https://deploy.integration.publishing.service.gov.uk/job/Deploy_Terraform_GOVUK_AWS/' diff --git a/tools/govuk-puppetmaster-integration-bootstrap.sh b/tools/govuk-puppetmaster-integration-bootstrap.sh deleted file mode 100755 index 1c5de20ab..000000000 --- a/tools/govuk-puppetmaster-integration-bootstrap.sh +++ /dev/null @@ -1,124 +0,0 @@ -#!/bin/bash -# This snippet replaces aws-push-puppet and aws-copy-puppet-config.sh -# as part of a new automated bootstrap process based on Terraform -# SSM secrets store. It is called as 30-puppetmaster-bootstrap from -# terraform user_data as defined in govuk-aws-data -set -x -set -u - -GOVUK_ENVIRONMENT='integration' -GOVUK_STACKNAME='blue' - -GOVUK_WORKDIR='/var/govuk' -GOVUK_LOGDIR='/var/log/govuk' - -GOVUK_GIT_URL='git@github.com:alphagov' - -GOVUK_SECRETS_REPO='govuk-secrets' -GOVUK_PUPPET_REPO='govuk-puppet' - -AWS_REGION='eu-west-1' - -GPG_KEYSTORE='/root/.gnupg' -GPG_KEYNAME='gpgkey' - -SSH_KEYSTORE='/root/.ssh' -SSH_KEYNAME='id_rsa' - -# Librarian needs a HOME -[[ -v HOME ]] || export HOME=/root - -# Install required packages -apt-get -y install postgresql-9.3 -apt-get -y install bundler -apt-get -y install git -apt-get -y install python3-pip - -# Create required directories -mkdir -p "${GPG_KEYSTORE}" -mkdir -p "${SSH_KEYSTORE}" -mkdir -p "${GOVUK_LOGDIR}" - -cd "${GOVUK_WORKDIR}" || exit - -# Function to access the AWS SSM parameter store and extract SecureString values from returned JSON -function get_ssm_parameter () -{ -set +x - SSM_PARAMETER_NAME=$1 - SSM_PARAMETER=$(aws --region=${AWS_REGION} ssm get-parameter --name "${SSM_PARAMETER_NAME}" --with-decryption | jq .Parameter.Value | sed -e "s/^\"//;s/\"$//") - echo "${SSM_PARAMETER}" -set -x -} - -# Get github.com publish SSH hostkey and write to known_hosts -set +x -get_ssm_parameter 'govuk_base64_github.com_hostkey' | base64 -d >> ${SSH_KEYSTORE}/known_hosts -set -x - -# Get SSH key with ro access to all of alphagov (for govuk-secrets) -set +x -get_ssm_parameter 'govuk_base64_github.com_ssh_readonly' | base64 -d > ${SSH_KEYSTORE}/${SSH_KEYNAME} -chmod 600 ${SSH_KEYSTORE}/${SSH_KEYNAME} -set -x - -# Get GPG key to decrypt -set +x -echo -n "$(get_ssm_parameter 'govuk_base64_gpg_1_of_3')$(get_ssm_parameter 'govuk_base64_gpg_2_of_3')$(get_ssm_parameter 'govuk_base64_gpg_3_of_3')" | base64 -d > ${GPG_KEYSTORE}/${GPG_KEYNAME} -chmod 600 ${GPG_KEYSTORE}/${GPG_KEYNAME} -set -x - -# Clone Puppet repo -git clone ${GOVUK_GIT_URL}/${GOVUK_PUPPET_REPO} - -# Clone secrets repo -git clone --depth=1 ${GOVUK_GIT_URL}/${GOVUK_SECRETS_REPO} - -# Add secrets to puppet repository -cp -r ${GOVUK_WORKDIR}/${GOVUK_SECRETS_REPO}/puppet_aws/hieradata/* ${GOVUK_WORKDIR}/${GOVUK_PUPPET_REPO}/hieradata_aws/ - -RELEASENAME=$(date +%Y%m%d%H%M%S) - -# If not in production and/or on a (different) stack, shift around respective config yaml -if [[ "${GOVUK_ENVIRONMENT}" != "production" ]] -then - cp ${GOVUK_WORKDIR}/${GOVUK_PUPPET_REPO}/hieradata_aws/${GOVUK_ENVIRONMENT}.yaml ${GOVUK_WORKDIR}/${GOVUK_PUPPET_REPO}/hieradata_aws/production.yaml - cp ${GOVUK_WORKDIR}/${GOVUK_PUPPET_REPO}/hieradata_aws/${GOVUK_ENVIRONMENT}_credentials.yaml ${GOVUK_WORKDIR}/${GOVUK_PUPPET_REPO}/hieradata_aws/production_credentials.yaml - - if [[ -d "${GOVUK_WORKDIR}/${GOVUK_PUPPET_REPO}/hieradata_aws/${GOVUK_STACKNAME}" ]] - then - cp ${GOVUK_WORKDIR}/${GOVUK_PUPPET_REPO}/hieradata_aws/${GOVUK_STACKNAME}/${GOVUK_ENVIRONMENT}_credentials.yaml ${GOVUK_WORKDIR}/${GOVUK_PUPPET_REPO}/hieradata_aws/${GOVUK_STACKNAME}/production_credentials.yaml - fi - - if [[ -f "${GOVUK_WORKDIR}/${GOVUK_PUPPET_REPO}/hieradata_aws/apps/${GOVUK_ENVIRONMENT}_credentials.yaml" ]] - then - cp ${GOVUK_WORKDIR}/${GOVUK_PUPPET_REPO}/hieradata_aws/apps/${GOVUK_ENVIRONMENT}_credentials.yaml ${GOVUK_WORKDIR}/${GOVUK_PUPPET_REPO}/hieradata_aws/apps/production_credentials.yaml - fi -fi - -# Move puppet release to the expected location -mkdir -p /usr/share/puppet/production/releases -mv "${GOVUK_WORKDIR}/${GOVUK_PUPPET_REPO}" "/usr/share/puppet/production/releases/${RELEASENAME}" -rm -f "/usr/share/puppet/production/current" -ln -s "/usr/share/puppet/production/releases/${RELEASENAME}" "/usr/share/puppet/production/current" -# We only want the permissions applied to the deepest directory, so is correct -# behaviour. -# shellcheck disable=SC2174 -mkdir -p -m 0700 /etc/puppet/gpg -gpg --homedir /etc/puppet/gpg --allow-secret-key-import --import ${GPG_KEYSTORE}/${GPG_KEYNAME} -chown -R puppet: /etc/puppet/gpg - -# Install Ruby dependencies for first puppet apply -gem install --no-ri --no-rdoc hiera-eyaml-gpg gpgme - -cd "/usr/share/puppet/production/current/" || exit - -# Installing Puppet dependencies -bundle install --full-index -bundle exec rake librarian:install - -cd "${GOVUK_WORKDIR}" || exit - -# Self-configure puppet -puppet apply --verbose --trusted_node_data --hiera_config=/usr/share/puppet/production/current/hiera_aws.yml --modulepath=/usr/share/puppet/production/current/modules:/usr/share/puppet/production/current/vendor/modules/ --manifestdir=/usr/share/puppet/production/current/manifests /usr/share/puppet/production/current/manifests/site.pp >> ${GOVUK_LOGDIR}/govuk_puppet_apply.log 2>&1 -chown -R deploy:deploy "/usr/share/puppet/production/releases/${RELEASENAME}" diff --git a/tools/govuk-puppetmaster-production-bootstrap.sh b/tools/govuk-puppetmaster-production-bootstrap.sh deleted file mode 100755 index 6eeadc746..000000000 --- a/tools/govuk-puppetmaster-production-bootstrap.sh +++ /dev/null @@ -1,124 +0,0 @@ -#!/bin/bash -# This snippet replaces aws-push-puppet and aws-copy-puppet-config.sh -# as part of a new automated bootstrap process based on Terraform -# SSM secrets store. It is called as 30-puppetmaster-bootstrap from -# terraform user_data as defined in govuk-aws-data -set -x -set -u - -GOVUK_ENVIRONMENT='production' -GOVUK_STACKNAME='blue' - -GOVUK_WORKDIR='/var/govuk' -GOVUK_LOGDIR='/var/log/govuk' - -GOVUK_GIT_URL='git@github.com:alphagov' - -GOVUK_SECRETS_REPO='govuk-secrets' -GOVUK_PUPPET_REPO='govuk-puppet' - -AWS_REGION='eu-west-1' - -GPG_KEYSTORE='/root/.gnupg' -GPG_KEYNAME='gpgkey' - -SSH_KEYSTORE='/root/.ssh' -SSH_KEYNAME='id_rsa' - -# Librarian needs a HOME -[[ -v HOME ]] || export HOME=/root - -# Install required packages -apt-get -y install postgresql-9.3 -apt-get -y install bundler -apt-get -y install git -apt-get -y install python3-pip - -# Create required directories -mkdir -p "${GPG_KEYSTORE}" -mkdir -p "${SSH_KEYSTORE}" -mkdir -p "${GOVUK_LOGDIR}" - -cd "${GOVUK_WORKDIR}" || exit - -# Function to access the AWS SSM parameter store and extract SecureString values from returned JSON -function get_ssm_parameter () -{ -set +x - SSM_PARAMETER_NAME=$1 - SSM_PARAMETER=$(aws --region=${AWS_REGION} ssm get-parameter --name "${SSM_PARAMETER_NAME}" --with-decryption | jq .Parameter.Value | sed -e "s/^\"//;s/\"$//") - echo "${SSM_PARAMETER}" -set -x -} - -# Get github.com publish SSH hostkey and write to known_hosts -set +x -get_ssm_parameter 'govuk_base64_github.com_hostkey' | base64 -d >> ${SSH_KEYSTORE}/known_hosts -set -x - -# Get SSH key with ro access to all of alphagov (for govuk-secrets) -set +x -get_ssm_parameter 'govuk_base64_github.com_ssh_readonly' | base64 -d > ${SSH_KEYSTORE}/${SSH_KEYNAME} -chmod 600 ${SSH_KEYSTORE}/${SSH_KEYNAME} -set -x - -# Get GPG key to decrypt -set +x -echo -n "$(get_ssm_parameter 'govuk_base64_gpg_1_of_3')$(get_ssm_parameter 'govuk_base64_gpg_2_of_3')$(get_ssm_parameter 'govuk_base64_gpg_3_of_3')" | base64 -d > ${GPG_KEYSTORE}/${GPG_KEYNAME} -chmod 600 ${GPG_KEYSTORE}/${GPG_KEYNAME} -set -x - -# Clone Puppet repo -git clone ${GOVUK_GIT_URL}/${GOVUK_PUPPET_REPO} - -# Clone secrets repo -git clone --depth=1 ${GOVUK_GIT_URL}/${GOVUK_SECRETS_REPO} - -# Add secrets to puppet repository -cp -r ${GOVUK_WORKDIR}/${GOVUK_SECRETS_REPO}/puppet_aws/hieradata/* ${GOVUK_WORKDIR}/${GOVUK_PUPPET_REPO}/hieradata_aws/ - -RELEASENAME=$(date +%Y%m%d%H%M%S) - -# If not in production and/or on a (different) stack, shift around respective config yaml -if [[ "${GOVUK_ENVIRONMENT}" != "production" ]] -then - cp ${GOVUK_WORKDIR}/${GOVUK_PUPPET_REPO}/hieradata_aws/${GOVUK_ENVIRONMENT}.yaml ${GOVUK_WORKDIR}/${GOVUK_PUPPET_REPO}/hieradata_aws/production.yaml - cp ${GOVUK_WORKDIR}/${GOVUK_PUPPET_REPO}/hieradata_aws/${GOVUK_ENVIRONMENT}_credentials.yaml ${GOVUK_WORKDIR}/${GOVUK_PUPPET_REPO}/hieradata_aws/production_credentials.yaml - - if [[ -d "${GOVUK_WORKDIR}/${GOVUK_PUPPET_REPO}/hieradata_aws/${GOVUK_STACKNAME}" ]] - then - cp ${GOVUK_WORKDIR}/${GOVUK_PUPPET_REPO}/hieradata_aws/${GOVUK_STACKNAME}/${GOVUK_ENVIRONMENT}_credentials.yaml ${GOVUK_WORKDIR}/${GOVUK_PUPPET_REPO}/hieradata_aws/${GOVUK_STACKNAME}/production_credentials.yaml - fi - - if [[ -f "${GOVUK_WORKDIR}/${GOVUK_PUPPET_REPO}/hieradata_aws/apps/${GOVUK_ENVIRONMENT}_credentials.yaml" ]] - then - cp ${GOVUK_WORKDIR}/${GOVUK_PUPPET_REPO}/hieradata_aws/apps/${GOVUK_ENVIRONMENT}_credentials.yaml ${GOVUK_WORKDIR}/${GOVUK_PUPPET_REPO}/hieradata_aws/apps/production_credentials.yaml - fi -fi - -# Move puppet release to the expected location -mkdir -p /usr/share/puppet/production/releases -mv "${GOVUK_WORKDIR}/${GOVUK_PUPPET_REPO}" "/usr/share/puppet/production/releases/${RELEASENAME}" -rm -f "/usr/share/puppet/production/current" -ln -s "/usr/share/puppet/production/releases/${RELEASENAME}" "/usr/share/puppet/production/current" -# We only want the permissions applied to the deepest directory, so is correct -# behaviour. -# shellcheck disable=SC2174 -mkdir -p -m 0700 /etc/puppet/gpg -gpg --homedir /etc/puppet/gpg --allow-secret-key-import --import ${GPG_KEYSTORE}/${GPG_KEYNAME} -chown -R puppet: /etc/puppet/gpg - -# Install Ruby dependencies for first puppet apply -gem install --no-ri --no-rdoc hiera-eyaml-gpg gpgme - -cd "/usr/share/puppet/production/current/" || exit - -# Installing Puppet dependencies -bundle install --full-index -bundle exec rake librarian:install - -cd "${GOVUK_WORKDIR}" || exit - -# Self-configure puppet -puppet apply --verbose --trusted_node_data --hiera_config=/usr/share/puppet/production/current/hiera_aws.yml --modulepath=/usr/share/puppet/production/current/modules:/usr/share/puppet/production/current/vendor/modules/ --manifestdir=/usr/share/puppet/production/current/manifests /usr/share/puppet/production/current/manifests/site.pp >> ${GOVUK_LOGDIR}/govuk_puppet_apply.log 2>&1 -chown -R deploy:deploy "/usr/share/puppet/production/releases/${RELEASENAME}" diff --git a/tools/govuk-puppetmaster-staging-bootstrap.sh b/tools/govuk-puppetmaster-staging-bootstrap.sh deleted file mode 100755 index 62116169c..000000000 --- a/tools/govuk-puppetmaster-staging-bootstrap.sh +++ /dev/null @@ -1,124 +0,0 @@ -#!/bin/bash -# This snippet replaces aws-push-puppet and aws-copy-puppet-config.sh -# as part of a new automated bootstrap process based on Terraform -# SSM secrets store. It is called as 30-puppetmaster-bootstrap from -# terraform user_data as defined in govuk-aws-data -set -x -set -u - -GOVUK_ENVIRONMENT='staging' -GOVUK_STACKNAME='blue' - -GOVUK_WORKDIR='/var/govuk' -GOVUK_LOGDIR='/var/log/govuk' - -GOVUK_GIT_URL='git@github.com:alphagov' - -GOVUK_SECRETS_REPO='govuk-secrets' -GOVUK_PUPPET_REPO='govuk-puppet' - -AWS_REGION='eu-west-1' - -GPG_KEYSTORE='/root/.gnupg' -GPG_KEYNAME='gpgkey' - -SSH_KEYSTORE='/root/.ssh' -SSH_KEYNAME='id_rsa' - -# Librarian needs a HOME -[[ -v HOME ]] || export HOME=/root - -# Install required packages -apt-get -y install postgresql-9.3 -apt-get -y install bundler -apt-get -y install git -apt-get -y install python3-pip - -# Create required directories -mkdir -p "${GPG_KEYSTORE}" -mkdir -p "${SSH_KEYSTORE}" -mkdir -p "${GOVUK_LOGDIR}" - -cd "${GOVUK_WORKDIR}" || exit - -# Function to access the AWS SSM parameter store and extract SecureString values from returned JSON -function get_ssm_parameter () -{ -set +x - SSM_PARAMETER_NAME=$1 - SSM_PARAMETER=$(aws --region=${AWS_REGION} ssm get-parameter --name "${SSM_PARAMETER_NAME}" --with-decryption | jq .Parameter.Value | sed -e "s/^\"//;s/\"$//") - echo "${SSM_PARAMETER}" -set -x -} - -# Get github.com publish SSH hostkey and write to known_hosts -set +x -get_ssm_parameter 'govuk_base64_github.com_hostkey' | base64 -d >> ${SSH_KEYSTORE}/known_hosts -set -x - -# Get SSH key with ro access to all of alphagov (for govuk-secrets) -set +x -get_ssm_parameter 'govuk_base64_github.com_ssh_readonly' | base64 -d > ${SSH_KEYSTORE}/${SSH_KEYNAME} -chmod 600 ${SSH_KEYSTORE}/${SSH_KEYNAME} -set -x - -# Get GPG key to decrypt -set +x -echo -n "$(get_ssm_parameter 'govuk_base64_gpg_1_of_3')$(get_ssm_parameter 'govuk_base64_gpg_2_of_3')$(get_ssm_parameter 'govuk_base64_gpg_3_of_3')" | base64 -d > ${GPG_KEYSTORE}/${GPG_KEYNAME} -chmod 600 ${GPG_KEYSTORE}/${GPG_KEYNAME} -set -x - -# Clone Puppet repo -git clone ${GOVUK_GIT_URL}/${GOVUK_PUPPET_REPO} - -# Clone secrets repo -git clone --depth=1 ${GOVUK_GIT_URL}/${GOVUK_SECRETS_REPO} - -# Add secrets to puppet repository -cp -r ${GOVUK_WORKDIR}/${GOVUK_SECRETS_REPO}/puppet_aws/hieradata/* ${GOVUK_WORKDIR}/${GOVUK_PUPPET_REPO}/hieradata_aws/ - -RELEASENAME=$(date +%Y%m%d%H%M%S) - -# If not in production and/or on a (different) stack, shift around respective config yaml -if [[ "${GOVUK_ENVIRONMENT}" != "production" ]] -then - cp ${GOVUK_WORKDIR}/${GOVUK_PUPPET_REPO}/hieradata_aws/${GOVUK_ENVIRONMENT}.yaml ${GOVUK_WORKDIR}/${GOVUK_PUPPET_REPO}/hieradata_aws/production.yaml - cp ${GOVUK_WORKDIR}/${GOVUK_PUPPET_REPO}/hieradata_aws/${GOVUK_ENVIRONMENT}_credentials.yaml ${GOVUK_WORKDIR}/${GOVUK_PUPPET_REPO}/hieradata_aws/production_credentials.yaml - - if [[ -d "${GOVUK_WORKDIR}/${GOVUK_PUPPET_REPO}/hieradata_aws/${GOVUK_STACKNAME}" ]] - then - cp ${GOVUK_WORKDIR}/${GOVUK_PUPPET_REPO}/hieradata_aws/${GOVUK_STACKNAME}/${GOVUK_ENVIRONMENT}_credentials.yaml ${GOVUK_WORKDIR}/${GOVUK_PUPPET_REPO}/hieradata_aws/${GOVUK_STACKNAME}/production_credentials.yaml - fi - - if [[ -f "${GOVUK_WORKDIR}/${GOVUK_PUPPET_REPO}/hieradata_aws/apps/${GOVUK_ENVIRONMENT}_credentials.yaml" ]] - then - cp ${GOVUK_WORKDIR}/${GOVUK_PUPPET_REPO}/hieradata_aws/apps/${GOVUK_ENVIRONMENT}_credentials.yaml ${GOVUK_WORKDIR}/${GOVUK_PUPPET_REPO}/hieradata_aws/apps/production_credentials.yaml - fi -fi - -# Move puppet release to the expected location -mkdir -p /usr/share/puppet/production/releases -mv "${GOVUK_WORKDIR}/${GOVUK_PUPPET_REPO}" "/usr/share/puppet/production/releases/${RELEASENAME}" -rm -f "/usr/share/puppet/production/current" -ln -s "/usr/share/puppet/production/releases/${RELEASENAME}" "/usr/share/puppet/production/current" -# We only want the permissions applied to the deepest directory, so is correct -# behaviour. -# shellcheck disable=SC2174 -mkdir -p -m 0700 /etc/puppet/gpg -gpg --homedir /etc/puppet/gpg --allow-secret-key-import --import ${GPG_KEYSTORE}/${GPG_KEYNAME} -chown -R puppet: /etc/puppet/gpg - -# Install Ruby dependencies for first puppet apply -gem install --no-ri --no-rdoc hiera-eyaml-gpg gpgme - -cd "/usr/share/puppet/production/current/" || exit - -# Installing Puppet dependencies -bundle install --full-index -bundle exec rake librarian:install - -cd "${GOVUK_WORKDIR}" || exit - -# Self-configure puppet -puppet apply --verbose --trusted_node_data --hiera_config=/usr/share/puppet/production/current/hiera_aws.yml --modulepath=/usr/share/puppet/production/current/modules:/usr/share/puppet/production/current/vendor/modules/ --manifestdir=/usr/share/puppet/production/current/manifests /usr/share/puppet/production/current/manifests/site.pp >> ${GOVUK_LOGDIR}/govuk_puppet_apply.log 2>&1 -chown -R deploy:deploy "/usr/share/puppet/production/releases/${RELEASENAME}" diff --git a/tools/update-puppet.sh b/tools/update-puppet.sh deleted file mode 100755 index 2f650ba0a..000000000 --- a/tools/update-puppet.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash - -PUPPET_REPO=$1 -PUPPETMASTER_ELB=$2 - -if [[ ! -d "$PUPPET_REPO" ]]; then - echo "Couldn't find puppet directory: ${PUPPET_REPO}" - exit 1 -fi - -if [[ -z "$PUPPETMASTER_ELB" ]]; then - echo "Specify a puppetmaster ELB" - exit 1 -fi - -rsync -avz \ - --exclude='.git/' \ - "${PUPPET_REPO%/}" "$PUPPETMASTER_ELB": - -cat << EOF -WARNING: This is intended for testing purposes only. -It will break any local changes to the puppetmaster and always sets itself -in the 'puppet/production/current' directory. -EOF - -ssh "$PUPPETMASTER_ELB"<