Add caching when downloading percona toolkit

.gitignore

@@ -5,3 +5,4 @@ target
 .settings/
 .idea/
 liquibase-percona.iml
+.cache

pom.xml

@@ -65,7 +65,8 @@
         <mariadb_image>mariadb:10</mariadb_image>
 
         <docker.removeVolumes>true</docker.removeVolumes>
-
+        <percona.toolkit.usecache>true</percona.toolkit.usecache>
+        <percona.toolkit.cachedir>${project.basedir}/.cache</percona.toolkit.cachedir>
         <itPrebuildHookScript>../sharedScripts/setup</itPrebuildHookScript>
     </properties>
 
@@ -128,10 +129,6 @@
                   and maven-javadoc-plugin >= 3.2.0 is required.
                   See https://maven.apache.org/guides/mini/guide-reproducible-builds.html
                  -->
-                <plugin>
-                    <artifactId>maven-antrun-plugin</artifactId>
-                    <version>3.0.0</version>
-                </plugin>
                 <plugin>
                     <artifactId>maven-compiler-plugin</artifactId>
                     <version>3.9.0</version>
@@ -202,6 +199,11 @@
                     <artifactId>build-helper-maven-plugin</artifactId>
                     <version>3.3.0</version>
                 </plugin>
+                <plugin>
+                    <groupId>org.codehaus.mojo</groupId>
+                    <artifactId>exec-maven-plugin</artifactId>
+                    <version>3.0.0</version>
+                </plugin>
                 <plugin>
                     <groupId>org.sonatype.plugins</groupId>
                     <artifactId>nexus-staging-maven-plugin</artifactId>
@@ -402,32 +404,26 @@
             <build>
                 <plugins>
                     <plugin>
-                        <artifactId>maven-antrun-plugin</artifactId>
+                        <groupId>org.codehaus.mojo</groupId>
+                        <artifactId>exec-maven-plugin</artifactId>
                         <executions>
                             <execution>
                                 <id>download-percona-toolkit</id>
                                 <phase>pre-integration-test</phase>
                                 <goals>
-                                    <goal>run</goal>
+                                    <goal>exec</goal>
                                 </goals>
                                 <configuration>
-                                    <target>
-                                        <mkdir dir="${project.build.directory}/percona-toolkit" />
-
-                                        <exec executable="${project.basedir}/src/main/scripts/determine-latest-toolkit.sh" dir="${project.build.directory}/percona-toolkit" outputproperty="LATEST" />
-                                        <echo message="Determined latest version of percona toolkit: ${LATEST}" />
-
-                                        <exec executable="${project.basedir}/src/main/scripts/download-toolkit.sh" dir="${project.build.directory}/percona-toolkit">
-                                            <arg value="${LATEST}" />
-                                        </exec>
-                                        <move file="${project.build.directory}/percona-toolkit/percona-toolkit-${LATEST}" tofile="${project.build.directory}/percona-toolkit/percona-toolkit-LATEST" />
-                                        <echo message="The latest percona toolkit is available in: ${project.build.directory}/percona-toolkit/percona-toolkit-LATEST/" />
-
-                                        <exec executable="${project.basedir}/src/main/scripts/download-toolkit.sh" dir="${project.build.directory}/percona-toolkit">
-                                            <arg value="${percona.toolkit.version}" />
-                                        </exec>
-                                        <echo message="Downloaded percona toolkit in: ${project.build.directory}/percona-toolkit/percona-toolkit-${percona.toolkit.version}/" />
-                                    </target>
+                                    <executable>${project.basedir}/src/main/scripts/download-toolkit.sh</executable>
+                                    <arguments>
+                                        <argument>LATEST</argument>
+                                        <argument>${percona.toolkit.version}</argument>
+                                    </arguments>
+                                    <environmentVariables>
+                                        <USE_CACHE>${percona.toolkit.usecache}</USE_CACHE>
+                                        <CACHE_DIR>${percona.toolkit.cachedir}</CACHE_DIR>
+                                        <TARGET>${project.build.directory}/percona-toolkit</TARGET>
+                                    </environmentVariables>
                                 </configuration>
                             </execution>
                         </executions>

src/main/scripts/download-toolkit.sh

@@ -16,48 +16,182 @@
 
 set -e
 
-VERSION=$1
-
-if [ -z "${VERSION}" ]; then
-    echo "$0 <version>" 2>&1
-    echo 2>&1
-    echo "Please specify the version to download!" 2>&1
-    exit 1
-fi
-
-url=https://downloads.percona.com/downloads/percona-toolkit/${VERSION}/source/tarball/percona-toolkit-${VERSION}.tar.gz
-# the tags on github are unfortunately wrong and can't be used to download the source...
-#url=https://github.com/percona/percona-toolkit/archive/v${VERSION}.tar.gz
-
-filename=percona-toolkit-${VERSION}.tar.gz
-target=percona-toolkit-${VERSION}
-
-if [ -e "${filename}" ]; then
-  echo "Skipping download ${filename}..."
-else
-  echo "Downloading ${filename}..."
-  curl \
-    --location \
-    --output ${filename} \
-    ${url}
-  if [ $? -ne 0 ]; then
-    echo "Download from ${url} failed..."
-    exit 1
-  fi
-fi
-
-echo "Extracting..."
-tar xfz ${filename}
-
-if [ ! -d ${target} ]; then
-  echo "The directory ${target} doesn't exist - something went wrong!"
-  exit 1
-fi
-
-downloaded_version=$(tail -3 ${target}/bin/pt-online-schema-change |head -1)
-if [ "${downloaded_version}" != "pt-online-schema-change ${VERSION}" ]; then
-  echo "Wrong version downloaded: '${downloaded_version}' but expected to be '${VERSION}'"
-  exit 1
-fi
-
-echo "percona toolkit is available at: $(realpath ${target})"
+function main()
+{
+    if [ $# -eq 0 ] || [ "$1" = "-h" ] || [ "$1" = "--help" ]; then
+        (
+            echo "$0 <version> [<version>...]"
+            echo
+            echo "Please specify the versions to download!"
+            echo
+            echo "Environment Variables:"
+            echo "TARGET - base target directory, where percona toolkit versions should be extracted to"
+            echo "USE_CACHE - true or false"
+            echo "CACHE_DIR - caches previously downloaded percona toolkit versions"
+        ) >&2
+        exit 1
+    fi
+
+    if [ -z "${TARGET}" ]; then
+        echo "Environment variable TARGET is missing!" >&2
+        exit 1
+    fi
+
+    echo "Percona Toolkit Downloader"
+    echo "--------------------------"
+    echo "Target Directory: ${TARGET}"
+    if [ "${USE_CACHE,,}" = "true" ]; then
+        USE_CACHE=yes
+        if [ -z "${CACHE_DIR}" ]; then
+            echo "Environment variable CACHE_DIR is missing!" >&1
+            exit 1
+        fi
+        echo "Cache: ${USE_CACHE}"
+        if [ ! -d "${CACHE_DIR}" ]; then
+            mkdir "${CACHE_DIR}"
+            echo "Created cache directory: ${CACHE_DIR}"
+        else
+            echo "Using cache directory: ${CACHE_DIR}"
+        fi
+    else
+        USE_CACHE=no
+        echo "Cache: ${USE_CACHE}"
+    fi
+
+    if [ -f "${TARGET}" ]; then
+        echo "Directory ${TARGET} is a file!" >&2
+        exit 1
+    fi
+
+    if [ ! -d "${TARGET}" ]; then
+        mkdir "${TARGET}"
+        echo "Created directory ${TARGET}"
+    fi
+
+    for version in "$@"; do
+        if [ "${version,,}" = "latest" ]; then
+            real_version=$(determine_latest_toolkit)
+        else
+            real_version="${version}"
+        fi
+
+        echo
+        echo "Downloading ${real_version}..."
+        download_toolkit "${real_version}"
+        extract_toolkit "${real_version}" "${version}"
+    done
+}
+
+function is_cache_enabled() { [ "${USE_CACHE}" = "yes" ]; }
+function determine_latest_toolkit()
+{
+    local cached_version="${CACHE_DIR}/latest-version.txt"
+
+    if is_cache_enabled
+    then
+        if [ -e "${cached_version}" ]
+        then
+            cat "${cached_version}"
+            return
+        fi
+    fi
+
+    # percona way:
+    local latest
+    latest=$(curl --silent https://www.percona.com/downloads/percona-toolkit/LATEST/)
+
+    local version
+    version=$(echo "$latest"|grep 'selected="selected">Percona' | sed 's/.*\([0-9][0-9]*.[0-9][0-9]*.[0-9][0-9]*\).*/\1/')
+
+    # github way:
+    # ask github for the tags, use the first tag - hope, it is the latest
+    #
+    #local version=$(curl \
+    #  --silent \
+    #  --header "Accept: application/vnd.github.v3+json" \
+    #  https://api.github.com/repos/percona/percona-toolkit/tags \
+    #  | jq ".[0].name" --raw-output)
+    #
+    # output version, removing potential "v" prefix from version
+    #version="${version##v}"
+
+    if is_cache_enabled
+    then
+        echo "$version" > "$cached_version"
+    fi
+    echo "$version"
+}
+
+function download_toolkit()
+{
+    local version="$1"
+    local filename="percona-toolkit-${version}.tar.gz"
+    local cached_file="${CACHE_DIR}/${filename}"
+
+    if is_cache_enabled
+    then
+        if [ -e "${cached_file}" ]; then
+            echo "${filename} already exists in ${CACHE_DIR}, not downloading"
+            cp "${cached_file}" "${TARGET}/${filename}"
+            return
+        fi
+    fi
+
+    url=https://downloads.percona.com/downloads/percona-toolkit/${version}/source/tarball/percona-toolkit-${version}.tar.gz
+    # the tags on github are unfortunately wrong and can't be used to download the source...
+    #url=https://github.com/percona/percona-toolkit/archive/v${version}.tar.gz
+
+    echo "Downloading ${filename}..."
+    if ! curl \
+      --location \
+      --output "${TARGET}/${filename}" \
+        "${url}"; then
+        echo "Download from ${url} failed..."
+        exit 1
+    fi
+
+    if is_cache_enabled
+    then
+        echo "Caching ${filename}"
+        cp "${TARGET}/${filename}" "${cached_file}"
+    fi
+}
+
+function extract_toolkit() {
+    local version="$1"
+    local name="$2"
+    local filename="percona-toolkit-${version}.tar.gz"
+    local cached_file="${CACHE_DIR}/${filename}"
+    local extract_target_dir="percona-toolkit-${version}"
+    local extract_target_dir2="percona-toolkit-${name}"
+
+    echo "Extracting..."
+    (
+        cd "${TARGET}"
+        rm -rf "${extract_target_dir}"
+        tar xfz "${filename}"
+
+        if [ ! -d "${extract_target_dir}" ]; then
+            echo "The directory ${extract_target_dir} doesn't exist - something went wrong!"
+            exit 1
+        fi
+
+        local downloaded_version
+        downloaded_version=$(tail -3 "${extract_target_dir}/bin/pt-online-schema-change" |head -1)
+        if [ "${downloaded_version}" != "pt-online-schema-change ${version}" ]; then
+          echo "Wrong version downloaded: '${downloaded_version}' but expected to be '${version}'"
+          exit 1
+        fi
+
+        if [ "${extract_target_dir}" != "${extract_target_dir2}" ]; then
+            echo "Renaming to ${extract_target_dir2}"
+            rm -rf "${extract_target_dir2}"
+            mv "${extract_target_dir}" "${extract_target_dir2}"
+            extract_target_dir="${extract_target_dir2}"
+        fi
+
+        echo "--> Percona Toolkit ${name} is available at: $(realpath "${extract_target_dir}")"
+    )
+}
+
+main "$@"