diff --git a/Makefile b/Makefile index bcd1bc2f..3a1837c8 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,7 @@ help: .PHONY: virtualenv virtualenv: ## create virtual environment typically used for development purposes - virtualenv -p python3.9 --always-copy --prompt='(twindb_backup)' ~/.virtualenvs/twindb-backup/ + virtualenv -p python3 --always-copy --prompt='(twindb_backup)' ~/.virtualenvs/twindb-backup/ @printf "To activate run:\n\n. ~/.virtualenvs/twindb-backup/bin/activate\n\n" .PHONY: pip @@ -94,9 +94,6 @@ clean-test: ## remove test and coverage artifacts rm -fr htmlcov/ rm -rf .pytest_cache -clean-docker: - @sudo docker rm twindb-backup-build-${PLATFORM} - clean-docs: rm -rf docs/_build @@ -129,9 +126,6 @@ test-including-azure-blob: ## Like 'make test' but includes tests for azure blob test-integration: ## Run integration tests. Must be run in vagrant py.test -xsv tests/integration/ -coverage: ## check code coverage quickly with the default Python - py.test --cov-report term-missing --cov=twindb_backup tests/unit - docs: ## generate Sphinx HTML documentation, including API docs rm -f docs/twindb_backup.rst # rm -f docs/modules.rst @@ -155,25 +149,6 @@ install: clean ## install the package to the active Python's site-packages install -m 600 -o root support/twindb-backup.cfg "${DESTDIR}/etc/twindb" ; \ fi -docker-test: ## Test twindb-backup in a docker container - @sudo docker run \ - -v `pwd`:/twindb-backup:rw \ - -e "AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}" \ - -e "AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION}" \ - -e "AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}" \ - -e "GC_CREDENTIALS_FILE=${GC_CREDENTIALS_FILE}" \ - -e "OS_VERSION"=${OS_VERSION} \ - -e "CI"=${CI} \ - -e "TRAVIS"=${TRAVIS} \ - -e "TRAVIS_BRANCH"=${TRAVIS_BRANCH} \ - -e "TRAVIS_COMMIT"=${TRAVIS_COMMIT} \ - -e "TRAVIS_JOB_NUMBER"=${TRAVIS_JOB_NUMBER} \ - -e "TRAVIS_PULL_REQUEST"=${TRAVIS_PULL_REQUEST} \ - -e "TRAVIS_JOB_ID"=${TRAVIS_JOB_ID} \ - -e "TRAVIS_REPO_SLUG"=${TRAVIS_REPO_SLUG} \ - -e "TRAVIS_TAG"=${TRAVIS_TAG} \ - ${DOCKER_IMAGE} /bin/bash /twindb-backup/support/docker-test-${PLATFORM}.sh - .PHONY: docker-start docker-start: ## Start a container with /twindb-backup for debugging packaging, etc. @docker run \ @@ -182,8 +157,6 @@ docker-start: ## Start a container with /twindb-backup for debugging packaging, -it \ --name builder_xtrabackup \ --rm \ - --dns 8.8.8.8 \ - --dns 208.67.222.222 \ --env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} \ --env AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} \ --env "GC_CREDENTIALS_FILE=${GC_CREDENTIALS_FILE}" \ @@ -203,8 +176,6 @@ package: ## Build package - OS_VERSION must be one of: jammy, focal. -v ${pwd}:/twindb-backup \ --name builder_xtrabackup \ --rm \ - --dns 8.8.8.8 \ - --dns 208.67.222.222 \ --env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} \ --env AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} \ --env PLATFORM=${PLATFORM} \ diff --git a/omnibus/config/projects/twindb-backup.rb b/omnibus/config/projects/twindb-backup.rb index 2a5bab44..0618e805 100644 --- a/omnibus/config/projects/twindb-backup.rb +++ b/omnibus/config/projects/twindb-backup.rb @@ -23,7 +23,7 @@ # and /opt/twindb-backup on all other platforms install_dir '/opt/twindb-backup' -build_version '3.2.0' +build_version '3.2.1' build_iteration 1 diff --git a/setup.cfg b/setup.cfg index 50918ec5..a1459e08 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.2.0 +current_version = 3.2.1 commit = True tag = False diff --git a/setup.py b/setup.py index 8d9abf5e..d6995e8e 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ setup( name="twindb-backup", - version="3.2.0", + version="3.2.1", description="TwinDB Backup tool for files, MySQL et al.", long_description=readme + "\n\n" + history, author="TwinDB Development Team", diff --git a/support/bootstrap/master/centos/master1.sh b/support/bootstrap/master/centos/master1.sh deleted file mode 100644 index ad5ae55d..00000000 --- a/support/bootstrap/master/centos/master1.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash - -set -exu - -TB_VERSION=$(PYTHONPATH=/twindb-backup python -c "from twindb_backup import __version__; print __version__") - -wait_time=2 -for _ in $(seq 5) -do - yum clean all - yum -y install /twindb-backup/omnibus/pkg/twindb-backup-"${TB_VERSION}"-1.x86_64.rpm && break - echo "Waiting ${wait_time} seconds before retry" - sleep ${wait_time} - wait_time=$((wait_time * 2)) -done - -set +u -if ! test -z "${DEV}"; then - /bin/cp -R /twindb-backup/twindb_backup /opt/twindb-backup/embedded/lib/python3.9/site-packages -fi - -# MySQL sets random root password. Reset to empty -systemctl stop mysqld -rm -rf /var/lib/mysql -mkdir /var/lib/mysql -mysqld --initialize-insecure -chown -R mysql:mysql /var/lib/mysql -systemctl start mysqld diff --git a/support/bootstrap/master/centos/slave.sh b/support/bootstrap/master/centos/slave.sh deleted file mode 100644 index 2b73a86a..00000000 --- a/support/bootstrap/master/centos/slave.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash - -set -exu - -TB_VERSION=$(PYTHONPATH=/twindb-backup python -c "from twindb_backup import __version__; print __version__") - -wait_time=2 -for _ in $(seq 5) -do - yum clean all - yum -y install /twindb-backup/omnibus/pkg/twindb-backup-"${TB_VERSION}"-1.x86_64.rpm && break - echo "Waiting ${wait_time} seconds before retry" - sleep ${wait_time} - wait_time=$((wait_time * 2)) -done - -set +u -if ! test -z "${DEV}"; then - /bin/cp -R /twindb-backup/twindb_backup /opt/twindb-backup/embedded/lib/python3.9/site-packages -fi - -# MySQL sets random root password. Reset to empty -systemctl stop mysqld -rm -rf /var/lib/mysql -mkdir /var/lib/mysql diff --git a/support/bootstrap/master/debian/master1.sh b/support/bootstrap/master/debian/master1.sh deleted file mode 100644 index d946add4..00000000 --- a/support/bootstrap/master/debian/master1.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash - -set -exu - -wait_time=2 -for _ in $(seq 5) -do - apt-get -qq update && break - echo "Waiting ${wait_time} seconds before retry" - sleep ${wait_time} - wait_time=$((wait_time * 2)) -done - -apt-get -qqq -y install apt-utils - -TB_VERSION=$(PYTHONPATH=/twindb-backup python -c "from twindb_backup import __version__; print(__version__)") - -package="/twindb-backup/omnibus/pkg/twindb-backup_${TB_VERSION}-1_amd64.deb" - -apt -y install "$package" - -set +u -if ! test -z "${DEV}"; then - /bin/cp -R /twindb-backup/twindb_backup /opt/twindb-backup/embedded/lib/python3.9/site-packages -fi diff --git a/support/bootstrap/master/debian/slave.sh b/support/bootstrap/master/debian/slave.sh deleted file mode 100644 index 8dce4d06..00000000 --- a/support/bootstrap/master/debian/slave.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash - -set -exu - -wait_time=2 -for _ in $(seq 5) -do - apt-get -qq update && break - echo "Waiting ${wait_time} seconds before retry" - sleep ${wait_time} - wait_time=$((wait_time * 2)) -done - -apt-get -qqq -y install xtrabackup - -TB_VERSION=$(PYTHONPATH=/twindb-backup python -c "from twindb_backup import __version__; print __version__") - -package="/twindb-backup/omnibus/pkg/twindb-backup_${TB_VERSION}-1_amd64.deb" - -dpkg -I "${package}" | grep Depends: | sed -e 's/Depends://' -e 's/,//g' | xargs apt-get -y install -dpkg -i "${package}" - -set +u -if ! test -z "${DEV}"; then - /bin/cp -R /twindb-backup/twindb_backup /opt/twindb-backup/embedded/lib/python3.9/site-packages -fi diff --git a/support/bootstrap/master/ubuntu/master1.sh b/support/bootstrap/master/ubuntu/master1.sh index d946add4..caf1e65d 100644 --- a/support/bootstrap/master/ubuntu/master1.sh +++ b/support/bootstrap/master/ubuntu/master1.sh @@ -19,6 +19,8 @@ package="/twindb-backup/omnibus/pkg/twindb-backup_${TB_VERSION}-1_amd64.deb" apt -y install "$package" +chmod 644 /etc/mysql/mysql.conf.d/mysqld.cnf + set +u if ! test -z "${DEV}"; then /bin/cp -R /twindb-backup/twindb_backup /opt/twindb-backup/embedded/lib/python3.9/site-packages diff --git a/support/bootstrap/master/ubuntu/slave.sh b/support/bootstrap/master/ubuntu/slave.sh index 6d2522bc..5c6d1157 100644 --- a/support/bootstrap/master/ubuntu/slave.sh +++ b/support/bootstrap/master/ubuntu/slave.sh @@ -22,6 +22,8 @@ apt -y install "$package" rm -rf /var/lib/mysql mkdir /var/lib/mysql +chmod 644 /etc/mysql/mysql.conf.d/mysqld.cnf + set +u if ! test -z "${DEV}"; then /bin/cp -R /twindb-backup/twindb_backup /opt/twindb-backup/embedded/lib/python3.9/site-packages diff --git a/support/docker-test-centos.sh b/support/docker-test-centos.sh deleted file mode 100644 index 0827484a..00000000 --- a/support/docker-test-centos.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash - -set -eux -yum clean all -yum install -y epel-release - -for i in $(seq 5); do - yum install -y \ - gcc \ - zlib-devel \ - bzip2-devel \ - openssl-devel \ - make \ - wget \ - Percona-Server-server-56 \ - Percona-Server-devel-56 \ - percona-xtrabackup-24 \ - libffi-devel \ - git && break - sleep 5 -done - -mysql_install_db && mysqld --user=root & - -timeout=300 -while [ ${timeout} -gt 0 ] ; do mysqladmin ping && break; sleep 1; timeout=$((${timeout} - 1)); done - -if [ "$OS_VERSION" = 6 ] ; then - cd /tmp > /dev/null - wget http://python.org/ftp/python/2.7.14/Python-2.7.14.tgz > /dev/null - tar zxvf Python-2.7.14.tgz > /dev/null - cd Python-2.7.14 - ./configure > /dev/null - make && make install > /dev/null - wget https://bootstrap.pypa.io/get-pip.py > /dev/null - python2.7 get-pip.py > /dev/null - export GPG_TTY=/dev/tty -else - yum install -y python-devel python-setuptools python-pip -fi - -cd /twindb-backup -make bootstrap lint test diff --git a/support/docker-test-debian.sh b/support/docker-test-debian.sh deleted file mode 100644 index a6da90bf..00000000 --- a/support/docker-test-debian.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash - -set -eux - -export DEBIAN_FRONTEND=noninteractive -apt-get update -apt-get install -qq \ - wget \ - lsb-release \ - zlib1g-dev \ - libssl-dev \ - make \ - libpython2.7-dev \ - python-setuptools \ - libffi-dev \ - git \ - gcc - - -easy_install pip - -# install percona repository -wget https://repo.percona.com/apt/percona-release_0.1-4.$(lsb_release -sc)_all.deb -dpkg -i percona-release_0.1-4.$(lsb_release -sc)_all.deb -apt-get update - -apt-get install -qq \ - percona-server-server-5.6 \ - libperconaserverclient18.1-dev \ - percona-xtrabackup-24 - -mysql_install_db && mysqld --user=root & - -timeout=300 -while [ ${timeout} -gt 0 ] ; do mysqladmin ping && break; sleep 1; timeout=$((${timeout} - 1)); done - -cd /twindb-backup -pip install --editable . - -make bootstrap lint test diff --git a/support/docker-test-ubuntu.sh b/support/docker-test-ubuntu.sh deleted file mode 100644 index 10ef82d7..00000000 --- a/support/docker-test-ubuntu.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash - -set -eux - -export DEBIAN_FRONTEND=noninteractive -apt-get update -apt-get install -qq \ - wget \ - lsb-release \ - zlib1g-dev \ - libssl-dev \ - make \ - libpython2.7-dev \ - python-setuptools \ - libffi-dev \ - git \ - gcc - - -easy_install pip - -# install percona repository -wget "https://repo.percona.com/apt/percona-release_0.1-4.$(lsb_release -sc)_all.deb" -dpkg -i "percona-release_0.1-4.$(lsb_release -sc)_all.deb" -apt-get update - -apt-get install -qq \ - percona-server-server-5.6 \ - libperconaserverclient18.1-dev \ - percona-xtrabackup-24 - -mysql_install_db && mysqld --user=root & - -timeout=300 -while [ ${timeout} -gt 0 ] ; do mysqladmin ping && break; sleep 1; timeout=$((timeout - 1)); done - -cd /twindb-backup -pip install --editable . - -make bootstrap lint test diff --git a/tests/integration/backup/s3/test_backup.py b/tests/integration/backup/s3/test_backup.py index 7adfb3cc..34533ff8 100644 --- a/tests/integration/backup/s3/test_backup.py +++ b/tests/integration/backup/s3/test_backup.py @@ -581,7 +581,6 @@ def test_take_mysql_backup_aenc_restores_full( files_to_test = [] mysql_files = [ "ibdata1", - "ib_logfile0", "backup-my.cnf", ] for datadir_file in mysql_files: @@ -596,7 +595,7 @@ def test_take_mysql_backup_aenc_restores_full( docker_client, master1["Id"], dst_dir, - ["ib_logfile1", "xtrabackup_logfile"], + ["xtrabackup_logfile"], ) check_either_file( docker_client, @@ -733,7 +732,6 @@ def test_take_mysql_backup_aenc_restores_inc( for datadir_file in [ "ibdata1", - "ib_logfile0", "backup-my.cnf", ]: cmd = ["bash", "-c", f"test -f {osp.join(dst_dir, datadir_file)}"] @@ -747,7 +745,7 @@ def test_take_mysql_backup_aenc_restores_inc( docker_client, master1["Id"], dst_dir, - ["ib_logfile1", "xtrabackup_logfile"], + ["xtrabackup_logfile"], ) check_either_file( diff --git a/tests/integration/backup/s3/test_restore.py b/tests/integration/backup/s3/test_restore.py index 1dd8f970..96126d7c 100644 --- a/tests/integration/backup/s3/test_restore.py +++ b/tests/integration/backup/s3/test_restore.py @@ -87,15 +87,11 @@ def test__restore_mysql_inc_creates_log_files( ret, cout = docker_execute(docker_client, master1["Id"], cmd) assert_and_pause((ret == 0,), cout) - cmd = ["test", "-f", "/tmp/dst_full_log_files/ib_logfile0"] - ret, cout = docker_execute(docker_client, master1["Id"], cmd) - assert_and_pause((ret == 0,), cout) - check_files_if_xtrabackup( docker_client, master1["Id"], "/tmp/dst_full_log_files", - ["ib_logfile1", "xtrabackup_logfile"], + ["xtrabackup_logfile"], ) check_either_file( diff --git a/tests/integration/backup/ssh/test_restore.py b/tests/integration/backup/ssh/test_restore.py index 5960e105..a9fbf20d 100644 --- a/tests/integration/backup/ssh/test_restore.py +++ b/tests/integration/backup/ssh/test_restore.py @@ -77,7 +77,7 @@ def test_restore(master1, storage_server, config_content_ssh, docker_client, rsa ret, cout = docker_execute(docker_client, master1["Id"], cmd) assert_and_pause((ret == 0,), cout) - for fl in ["backup-my.cnf", "ibdata1", "ib_logfile0"]: + for fl in ["backup-my.cnf", "ibdata1"]: cmd = ["test", "-f", osp.join(dst_dir, fl)] ret, cout = docker_execute(docker_client, master1["Id"], cmd) assert_and_pause((ret == 0,), cout) @@ -86,7 +86,7 @@ def test_restore(master1, storage_server, config_content_ssh, docker_client, rsa docker_client, master1["Id"], dst_dir, - ["ib_logfile1", "xtrabackup_logfile"], + ["xtrabackup_logfile"], ) check_either_file( docker_client, diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index b48ad727..4f1a8064 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -18,10 +18,8 @@ SUPPORTED_IMAGES = [ "twindb/backup-test:jammy", "twindb/backup-test:focal", - "twindb/backup-test:focal-mariadb-10", ] - try: NODE_IMAGE = os.environ["DOCKER_IMAGE"] except KeyError: @@ -252,7 +250,6 @@ def get_container( # noinspection PyShadowingNames @pytest.fixture(scope="module") def master1(docker_client, container_network, tmpdir_factory): - LOG.info("Starting fixture master1 container") platform = ( os.environ["PLATFORM"] if "PLATFORM" in os.environ else get_platform_from_image(os.environ["DOCKER_IMAGE"]) @@ -545,5 +542,4 @@ def config_content_ssh(): def get_container_hostname(client, container) -> str: - return client.api.inspect_container(container)["Config"]["Hostname"] diff --git a/twindb_backup/__init__.py b/twindb_backup/__init__.py index e0b84748..5991621b 100644 --- a/twindb_backup/__init__.py +++ b/twindb_backup/__init__.py @@ -40,7 +40,7 @@ class and saves the backup copy in something defined in a destination class. __author__ = "TwinDB Development Team" __email__ = "dev@twindb.com" -__version__ = "3.2.0" +__version__ = "3.2.1" STATUS_FORMAT_VERSION = 1 LOCK_FILE = "/var/run/twindb-backup.lock" LOG_FILE = "/var/log/twindb-backup-measures.log" diff --git a/twindb_backup/cli.py b/twindb_backup/cli.py index 5e229095..bbe2b478 100644 --- a/twindb_backup/cli.py +++ b/twindb_backup/cli.py @@ -355,7 +355,7 @@ def clone_mysql_backup( ): """ Clone mysql backup on remote server and make it a slave. - By default it will take a slave from a local MySQL on port 3306. + By default, it will take a slave from a local MySQL on port 3306. Source and destinations are strings hostname:port. E.g. 10.10.10.10:3306. diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile index b7df0570..9849bcda 100644 --- a/vagrant/Vagrantfile +++ b/vagrant/Vagrantfile @@ -3,13 +3,13 @@ nodes = [ { :hostname => 'master1', :ip => '192.168.36.250', - :box => 'ubuntu/focal64', - :ram => 3072 + :box => 'ubuntu/jammy64', + :ram => 8192 }, { :hostname => 'master2', :ip => '192.168.36.251', - :box => 'ubuntu/focal64', - :ram => 3072 + :box => 'ubuntu/jammy64', + :ram => 8192 } ] @@ -47,8 +47,8 @@ SCRIPT Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.provider "virtualbox" do |v| - v.memory = 4096 - v.cpus = 4 + v.memory = 8192 + v.cpus = 12 end nodes.each do |node| config.vm.define node[:hostname] do |nodeconfig| diff --git a/vagrant/environment/puppet/modules/profile/files/mysql_grants.sql b/vagrant/environment/puppet/modules/profile/files/mysql_grants.sql index 50093b92..0dc0c09d 100644 --- a/vagrant/environment/puppet/modules/profile/files/mysql_grants.sql +++ b/vagrant/environment/puppet/modules/profile/files/mysql_grants.sql @@ -1,6 +1,6 @@ -CREATE USER 'dba'@'%' IDENTIFIED BY 'qwerty'; -CREATE USER 'dba'@'localhost' IDENTIFIED BY 'qwerty'; -CREATE USER 'repl'@'%' IDENTIFIED BY 'qwerty'; +CREATE USER 'dba'@'%' IDENTIFIED WITH mysql_native_password BY 'qwerty'; +CREATE USER 'dba'@'localhost' IDENTIFIED WITH mysql_native_password BY 'qwerty'; +CREATE USER 'repl'@'%' IDENTIFIED WITH mysql_native_password BY 'qwerty'; GRANT ALL ON *.* TO 'dba'@'%' WITH GRANT OPTION; GRANT ALL ON *.* TO 'dba'@'localhost' WITH GRANT OPTION; diff --git a/vagrant/environment/puppet/modules/profile/manifests/base.pp b/vagrant/environment/puppet/modules/profile/manifests/base.pp index a14fcb91..dc62621e 100644 --- a/vagrant/environment/puppet/modules/profile/manifests/base.pp +++ b/vagrant/environment/puppet/modules/profile/manifests/base.pp @@ -55,13 +55,12 @@ $packages = [ 'vim', 'netcat', - 'percona-server-client-5.7', - 'percona-server-server-5.7', + 'mysql-client', + 'mysql-server', 'percona-toolkit', - 'percona-xtrabackup-24', - 'python3.9-dev', 'python3.9', 'python', 'virtualenv', + 'percona-xtrabackup-80', + 'python3-dev', 'python3', 'virtualenv', 'python-is-python3', 'gcc', - 'zlibc', 'openssl', 'docker.io', 'strace', diff --git a/vagrant/environment/puppet/modules/profile/manifests/master.pp b/vagrant/environment/puppet/modules/profile/manifests/master.pp index ac95fc2e..804f227f 100644 --- a/vagrant/environment/puppet/modules/profile/manifests/master.pp +++ b/vagrant/environment/puppet/modules/profile/manifests/master.pp @@ -8,7 +8,7 @@ owner => 'mysql', source => 'puppet:///modules/profile/my-master.cnf', notify => Service['mysql'], - require => Package['percona-server-server-5.7'] + require => Package['mysql-server'] } file { "/home/${profile::base::user}/mysql_grants.sql": @@ -81,6 +81,6 @@ service { 'mysql': ensure => running, enable => true, - require => Package['percona-server-server-5.7'], + require => Package['mysql-server'], } }