From f560becc102ca2792e6bc14d676ffaeb2c82b621 Mon Sep 17 00:00:00 2001 From: Rachele Morino Date: Fri, 23 Feb 2024 15:15:25 -0500 Subject: [PATCH 1/7] Update boto to boto3 in deployment scripts --- .../cloudformation/privatehostedzone.py | 34 ++++++++++++------- deployment/cloudformation/utils/cfn.py | 6 ++-- .../deployment_requirements.txt | 3 +- 3 files changed, 27 insertions(+), 16 deletions(-) diff --git a/deployment/cloudformation/privatehostedzone.py b/deployment/cloudformation/privatehostedzone.py index b851137f5..003279e3c 100644 --- a/deployment/cloudformation/privatehostedzone.py +++ b/deployment/cloudformation/privatehostedzone.py @@ -1,9 +1,8 @@ -from boto import route53 - from majorkirby import CustomActionNode import json - +import boto3 +import uuid class R53PrivateHostedZone(CustomActionNode): """Represents a Route53 private hosted zone""" @@ -19,21 +18,32 @@ class R53PrivateHostedZone(CustomActionNode): def action(self): """Creates the zone""" - conn = route53.connect_to_region(self.REGION) + client = boto3.client('route53', region_name=self.REGION) comment = json.dumps(self.get_raw_tags()) - hosted_zones = conn.get_all_hosted_zones() + hosted_zones = client.list_hosted_zones() - for hosted_zone in hosted_zones['ListHostedZonesResponse']['HostedZones']: + for hosted_zone in hosted_zones['HostedZones']: if ('Comment' in hosted_zone['Config'] and hosted_zone['Config']['Comment'] == comment): self.stack_outputs = {'PrivateHostedZoneId': hosted_zone['Id'].split('/')[-1]} return - - hosted_zone = conn.create_hosted_zone(self.get_input('PrivateHostedZoneName'), - comment=comment, - private_zone=True, - vpc_id=self.get_input('VpcId'), - vpc_region=self.REGION) + + vpc_config = { + 'VPCRegion': self.REGION, + 'VPCId': self.get_input("VpcId") + } + hosted_zone_config = { + 'Comment': comment, + 'PrivateZone': True + } + + hosted_zone = client.create_hosted_zone( + Name=self.get_input("PrivateHostedZoneName"), + CallerReference=str(uuid.uuid4()), + HostedZoneConfig=hosted_zone_config, + VPC=vpc_config, + ) hosted_zone_id = hosted_zone['CreateHostedZoneResponse']['HostedZone']['Id'] self.stack_outputs = {'PrivateHostedZoneId': hosted_zone_id.split('/')[-1]} + diff --git a/deployment/cloudformation/utils/cfn.py b/deployment/cloudformation/utils/cfn.py index 579b7700b..da5033483 100644 --- a/deployment/cloudformation/utils/cfn.py +++ b/deployment/cloudformation/utils/cfn.py @@ -1,5 +1,5 @@ """Helper functions and classes for dealing with cloudformation""" -import boto +import boto3 class AvailabilityZone(object): @@ -38,5 +38,5 @@ def get_availability_zones(): Returns: (list of AvailabilityZone): List of availability zones for a given EC2 region """ - conn = boto.connect_ec2() - return [AvailabilityZone(az) for az in conn.get_all_zones()] + client = boto3.client('ec2', region_name='us-east-1') + return [AvailabilityZone(az) for az in client.describe_availability_zones()] diff --git a/python/cac_tripplanner/deployment_requirements.txt b/python/cac_tripplanner/deployment_requirements.txt index f872d100c..91924b9f5 100644 --- a/python/cac_tripplanner/deployment_requirements.txt +++ b/python/cac_tripplanner/deployment_requirements.txt @@ -1,5 +1,6 @@ -boto==2.49.0 +boto3==1.15.9 PyYAML==5.3.1 majorkirby==1.0.0 requests==2.24.0 troposphere==3.2.2 +uuid==1.30.0 From 0cbf1ead00b6567ce1e886ff3890ea9ea6d0f70c Mon Sep 17 00:00:00 2001 From: Rachele Morino Date: Thu, 29 Feb 2024 12:50:18 -0500 Subject: [PATCH 2/7] Update AvailabilityZone to account for new JSON response --- deployment/cloudformation/utils/cfn.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/deployment/cloudformation/utils/cfn.py b/deployment/cloudformation/utils/cfn.py index da5033483..8bcd0f889 100644 --- a/deployment/cloudformation/utils/cfn.py +++ b/deployment/cloudformation/utils/cfn.py @@ -15,7 +15,6 @@ def __init__(self, availability_zone): Args: availability_zone (AvailabilityZone): boto object """ - self.availability_zone = availability_zone @property @@ -24,12 +23,12 @@ def cfn_name(self): Utility method to return a string appropriate for CloudFormation name of a resource (e.g. UsEast1a) """ - return self.availability_zone.name.title().replace('-', '') + return self.availability_zone['ZoneName'].title().replace('-', '') @property def name(self): """Utility function to return the name of an availability zone (e.g. us-east-1a)""" - return self.availability_zone.name + return self.availability_zone['ZoneName'] def get_availability_zones(): @@ -39,4 +38,4 @@ def get_availability_zones(): (list of AvailabilityZone): List of availability zones for a given EC2 region """ client = boto3.client('ec2', region_name='us-east-1') - return [AvailabilityZone(az) for az in client.describe_availability_zones()] + return [AvailabilityZone(az) for az in client.describe_availability_zones()['AvailabilityZones']] From a3377e05f446f1e1d6fd49b23bd9a6439a704999 Mon Sep 17 00:00:00 2001 From: Rachele Morino Date: Wed, 28 Feb 2024 16:24:53 -0500 Subject: [PATCH 3/7] Upgrade to Pythonv3.8.10 and psycopg2 --- Vagrantfile | 2 +- deployment/ansible/group_vars/all | 2 +- python/cac_tripplanner/requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index d3418240a..4f54b1d51 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -71,7 +71,7 @@ VAGRANT_PROXYCONF_ENDPOINT = ENV["VAGRANT_PROXYCONF_ENDPOINT"] VAGRANTFILE_API_VERSION = "2" Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - config.vm.box = "bento/ubuntu-18.04" + config.vm.box = "bento/ubuntu-20.04" # Wire up the proxy if: # diff --git a/deployment/ansible/group_vars/all b/deployment/ansible/group_vars/all index edda281ce..f7964c08d 100644 --- a/deployment/ansible/group_vars/all +++ b/deployment/ansible/group_vars/all @@ -9,7 +9,7 @@ postgres_user: "cac_tripplanner" postgres_password: "cac_tripplanner" postgres_host: "192.168.56.25" -postgresql_support_psycopg2_version: "2.8.*" +postgresql_support_psycopg2_version: "2.9.*" 'postgis_version': [3, 0, 1] packer_version: "1.5.4" diff --git a/python/cac_tripplanner/requirements.txt b/python/cac_tripplanner/requirements.txt index 6c580b483..f870faef1 100644 --- a/python/cac_tripplanner/requirements.txt +++ b/python/cac_tripplanner/requirements.txt @@ -10,7 +10,7 @@ easy-thumbnails==2.8.1 gunicorn==20.0.4 lxml==4.9.2 Pillow==7.2.0 -psycopg2-binary==2.8.6 +psycopg2-binary==2.9.0 pytz==2020.1 PyYAML==5.3.1 requests==2.24.0 From d909581c30650c3eb50a042b632f664543c5d3af Mon Sep 17 00:00:00 2001 From: Rachele Morino Date: Fri, 1 Mar 2024 12:40:47 -0500 Subject: [PATCH 4/7] Update base box to 22.04 for upgraded Python default --- Vagrantfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index 4f54b1d51..d0eae035b 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -71,7 +71,7 @@ VAGRANT_PROXYCONF_ENDPOINT = ENV["VAGRANT_PROXYCONF_ENDPOINT"] VAGRANTFILE_API_VERSION = "2" Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - config.vm.box = "bento/ubuntu-20.04" + config.vm.box = "bento/ubuntu-22.04" # Wire up the proxy if: # @@ -86,6 +86,9 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| end config.vm.define "database" do |database| + # SSH issues bringing up DB VM with base box 22.04 + # Explicitly define downgraded version for use by database until fix + database.vm.box = "bento/ubuntu-20.04" database.vm.hostname = "database" database.vm.network "private_network", ip: "192.168.56.25" From 122967df78d41844973402dfc2b06de541f2811a Mon Sep 17 00:00:00 2001 From: Rachele Morino Date: Fri, 1 Mar 2024 13:36:56 -0500 Subject: [PATCH 5/7] Copy azavea.nginx role over to customize --- deployment/ansible/roles.yml | 5 --- .../roles/cac-tripplanner.app/meta/main.yml | 2 +- .../cac-tripplanner.nginx/defaults/main.yml | 3 ++ .../cac-tripplanner.nginx/handlers/main.yml | 3 ++ .../cac-tripplanner.nginx/tasks/main.yml | 30 ++++++++++++++++ .../templates/logrotate_nginx.j2 | 18 ++++++++++ .../templates/nginx.conf.j2 | 34 +++++++++++++++++++ 7 files changed, 89 insertions(+), 6 deletions(-) create mode 100644 deployment/ansible/roles/cac-tripplanner.nginx/defaults/main.yml create mode 100644 deployment/ansible/roles/cac-tripplanner.nginx/handlers/main.yml create mode 100644 deployment/ansible/roles/cac-tripplanner.nginx/tasks/main.yml create mode 100644 deployment/ansible/roles/cac-tripplanner.nginx/templates/logrotate_nginx.j2 create mode 100644 deployment/ansible/roles/cac-tripplanner.nginx/templates/nginx.conf.j2 diff --git a/deployment/ansible/roles.yml b/deployment/ansible/roles.yml index 04cfe7286..68c03534b 100644 --- a/deployment/ansible/roles.yml +++ b/deployment/ansible/roles.yml @@ -1,9 +1,6 @@ - src: azavea.opentripplanner version: 2.0.1 -- src: azavea.nginx - version: 1.0.0 - - src: azavea.nodejs version: 0.4.0 @@ -12,5 +9,3 @@ - src: azavea.papertrail version: 2.0.0 - - diff --git a/deployment/ansible/roles/cac-tripplanner.app/meta/main.yml b/deployment/ansible/roles/cac-tripplanner.app/meta/main.yml index 04c53cc7e..cc206e8ca 100644 --- a/deployment/ansible/roles/cac-tripplanner.app/meta/main.yml +++ b/deployment/ansible/roles/cac-tripplanner.app/meta/main.yml @@ -1,8 +1,8 @@ --- dependencies: - { role: "azavea.git" } - - { role: "azavea.nginx" } - { role: "azavea.nodejs" } - { role: "azavea.packer" } + - { role: "cac-tripplanner.nginx" } - { role: "cac-tripplanner.papertrail", when: production } - { role: "cac-tripplanner.python3" } diff --git a/deployment/ansible/roles/cac-tripplanner.nginx/defaults/main.yml b/deployment/ansible/roles/cac-tripplanner.nginx/defaults/main.yml new file mode 100644 index 000000000..76614cc54 --- /dev/null +++ b/deployment/ansible/roles/cac-tripplanner.nginx/defaults/main.yml @@ -0,0 +1,3 @@ +--- +nginx_version: "stable" +nginx_delete_default_site: False diff --git a/deployment/ansible/roles/cac-tripplanner.nginx/handlers/main.yml b/deployment/ansible/roles/cac-tripplanner.nginx/handlers/main.yml new file mode 100644 index 000000000..159596ed1 --- /dev/null +++ b/deployment/ansible/roles/cac-tripplanner.nginx/handlers/main.yml @@ -0,0 +1,3 @@ +--- +- name: Restart Nginx + service: name=nginx state=restarted diff --git a/deployment/ansible/roles/cac-tripplanner.nginx/tasks/main.yml b/deployment/ansible/roles/cac-tripplanner.nginx/tasks/main.yml new file mode 100644 index 000000000..17a05c5ae --- /dev/null +++ b/deployment/ansible/roles/cac-tripplanner.nginx/tasks/main.yml @@ -0,0 +1,30 @@ +--- +- name: Configure the Nginx PPA + apt_repository: repo=ppa:nginx/{{ nginx_version }} state=present + +- name: Install Nginx + apt: pkg=nginx-full state=present + +- name: Delete default site + file: path=/etc/nginx/sites-enabled/default state=absent + register: delete_default_site + when: nginx_delete_default_site | bool + notify: + - Restart Nginx + +- name: Delete default web root + file: path=/var/www/html state=absent + when: nginx_delete_default_site | bool and delete_default_site is changed + +- name: Check Nginx Upstart service definition exists + stat: path=/etc/init/nginx.conf + register: nginx_upstart + +- name: Configure Nginx log rotation + template: src=logrotate_nginx.j2 dest=/etc/logrotate.d/nginx + when: nginx_upstart.stat.exists + +- name: Configure Nginx + template: src=nginx.conf.j2 dest=/etc/nginx/nginx.conf + notify: + - Restart Nginx diff --git a/deployment/ansible/roles/cac-tripplanner.nginx/templates/logrotate_nginx.j2 b/deployment/ansible/roles/cac-tripplanner.nginx/templates/logrotate_nginx.j2 new file mode 100644 index 000000000..92e628ec3 --- /dev/null +++ b/deployment/ansible/roles/cac-tripplanner.nginx/templates/logrotate_nginx.j2 @@ -0,0 +1,18 @@ +/var/log/nginx/*.log { + weekly + missingok + rotate 52 + compress + delaycompress + notifempty + create 0640 www-data adm + sharedscripts + prerotate + if [ -d /etc/logrotate.d/httpd-prerotate ]; then \ + run-parts /etc/logrotate.d/httpd-prerotate; \ + fi \ + endscript + postrotate + service nginx rotate >/dev/null 2>&1 + endscript +} diff --git a/deployment/ansible/roles/cac-tripplanner.nginx/templates/nginx.conf.j2 b/deployment/ansible/roles/cac-tripplanner.nginx/templates/nginx.conf.j2 new file mode 100644 index 000000000..541317b03 --- /dev/null +++ b/deployment/ansible/roles/cac-tripplanner.nginx/templates/nginx.conf.j2 @@ -0,0 +1,34 @@ +user www-data; +worker_processes {{ ansible_processor_count }}; +pid /run/nginx.pid; + +events { + worker_connections 768; +} + +http { + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + + server_tokens off; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/error.log; + + gzip on; + gzip_vary on; + gzip_proxied any; + gzip_comp_level 6; + gzip_buffers 16 8k; + gzip_http_version 1.1; + gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript application/javascript; + + include /etc/nginx/conf.d/*.conf; + include /etc/nginx/sites-enabled/*; +} From 912102cd291fb7821ebca2366cb78b54d2e1f7fb Mon Sep 17 00:00:00 2001 From: Rachele Morino Date: Fri, 1 Mar 2024 14:16:17 -0500 Subject: [PATCH 6/7] Update nginx to Ubuntu 22.04-compatible repo --- .../ansible/roles/cac-tripplanner.nginx/defaults/main.yml | 1 - deployment/ansible/roles/cac-tripplanner.nginx/tasks/main.yml | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/deployment/ansible/roles/cac-tripplanner.nginx/defaults/main.yml b/deployment/ansible/roles/cac-tripplanner.nginx/defaults/main.yml index 76614cc54..bcb9c0025 100644 --- a/deployment/ansible/roles/cac-tripplanner.nginx/defaults/main.yml +++ b/deployment/ansible/roles/cac-tripplanner.nginx/defaults/main.yml @@ -1,3 +1,2 @@ --- -nginx_version: "stable" nginx_delete_default_site: False diff --git a/deployment/ansible/roles/cac-tripplanner.nginx/tasks/main.yml b/deployment/ansible/roles/cac-tripplanner.nginx/tasks/main.yml index 17a05c5ae..6086c06aa 100644 --- a/deployment/ansible/roles/cac-tripplanner.nginx/tasks/main.yml +++ b/deployment/ansible/roles/cac-tripplanner.nginx/tasks/main.yml @@ -1,6 +1,8 @@ --- +# Tasks and related artifacts copied from azavea.nginx role +# Customized to use Nginx Stable PPA that works with Ubuntu 22.04 - name: Configure the Nginx PPA - apt_repository: repo=ppa:nginx/{{ nginx_version }} state=present + apt_repository: repo=ppa:ondrej/nginx state=present - name: Install Nginx apt: pkg=nginx-full state=present From 778fe611117b4948a33e02e97fce149a043a094f Mon Sep 17 00:00:00 2001 From: Rachele Morino Date: Mon, 4 Mar 2024 14:27:22 -0500 Subject: [PATCH 7/7] Add migrations note to README for spin-up clarity --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 8e8005c0c..dae09a7b7 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,11 @@ CAC_APP_SHARED_FOLDER_TYPE=virtualbox vagrant up Note that if there is an existing build Graph.obj in `otp_data`, vagrant provisioning in development mode will not attempt to rebuild the graph, but will use the one already present. +Django migrations are run as part of app provisioning, [here](https://github.com/azavea/cac-tripplanner/blob/develop/deployment/ansible/roles/cac-tripplanner.app/tasks/main.yml#L67-L72), but there may be instances where you need to manually run migrations outside of provisioning, in which case use the command: +``` +vagrant ssh app -c 'cd /opt/app/python/cac_tripplanner && python3 manage.py migrate' +``` + Building AMIs ------------------------