From dbce63f78318ecb12eabd56dc8576df398cfe6cd Mon Sep 17 00:00:00 2001 From: Davi Koscianski Vidal Date: Tue, 3 Oct 2023 00:24:41 +0200 Subject: [PATCH 1/2] SONAR-20590 - Adds retry mechanism for public and private scan tasks --- .cirrus.yml | 8 ++----- .cirrus/private-scan-task.sh | 43 ++++++++++++++++++++++++++++++++++++ .cirrus/public-scan-task.sh | 42 +++++++++++++++++++++++++++++++++++ 3 files changed, 87 insertions(+), 6 deletions(-) create mode 100755 .cirrus/private-scan-task.sh create mode 100755 .cirrus/public-scan-task.sh diff --git a/.cirrus.yml b/.cirrus.yml index 9c720e5f..61519450 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -79,9 +79,7 @@ private_scan_task: - curl -sSL https://unified-agent.s3.amazonaws.com/wss-unified-agent.jar -o wss-unified-agent.jar - echo "docker.includes=${tag}" >> .cirrus/wss-unified-agent.config scan_script: - - echo "Scan the ${STAGING_IMAGE_NAME}:${tag} image supporting linux/${platform}" - - docker pull --platform linux/${platform} "${STAGING_IMAGE_NAME}:${tag}" - - java -jar wss-unified-agent.jar -c .cirrus/wss-unified-agent.config -apiKey $MEND_API_KEY -product ${WS_PRODUCTNAME} -project ${STAGING_IMAGE_NAME}:${tag} -wss.url ${WS_WSS_URL} -docker.scanImages true + - .cirrus/private-scan-task.sh "${STAGING_IMAGE_NAME}" "${tag}" "${platform}" "${WS_PRODUCTNAME}" "${WS_WSS_URL}" "${MEND_API_KEY}" depends_on: multi_arch_build public_scan_task: @@ -104,9 +102,7 @@ public_scan_task: - curl -sSL https://unified-agent.s3.amazonaws.com/wss-unified-agent.jar -o wss-unified-agent.jar - echo "docker.includes=${tag}" >> .cirrus/wss-unified-agent.config scan_script: - - echo "Scan the ${PUBLIC_IMAGE_NAME}:${tag} image" - - docker pull "${PUBLIC_IMAGE_NAME}:${tag}" - - java -jar wss-unified-agent.jar -c .cirrus/wss-unified-agent.config -apiKey $MEND_API_KEY -product ${WS_PRODUCTNAME} -project ${PUBLIC_IMAGE_NAME}:${tag} -wss.url ${WS_WSS_URL} -docker.scanImages true + - .cirrus/public-scan-task.sh "${PUBLIC_IMAGE_NAME}" "${tag}" "${WS_PRODUCTNAME}" "${WS_WSS_URL}" "${MEND_API_KEY}" multi_arch_test_task: matrix: diff --git a/.cirrus/private-scan-task.sh b/.cirrus/private-scan-task.sh new file mode 100755 index 00000000..6887264f --- /dev/null +++ b/.cirrus/private-scan-task.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +STAGING_IMAGE_NAME=${1} +tag=${2} +platform=${3} +WS_PRODUCTNAME=${4} +WS_WSS_URL=${5} +MEND_API_KEY=${6} + +IMAGE_PULLED=1 +SCANNED=1 + +echo "Scan the ${STAGING_IMAGE_NAME}:${tag} image supporting linux/${platform}" + +for i in $(seq 1 3); do + if docker pull --platform linux/"${platform}" "${STAGING_IMAGE_NAME}:${tag}"; then + IMAGE_PULLED=0 + break + fi + echo "[${i}/3] Retrying to pull image ${STAGING_IMAGE_NAME}:${tag}..." + sleep 5 +done + +if [[ ${IMAGE_PULLED} -ne 0 ]]; then + echo "Failed to pull image ${STAGING_IMAGE_NAME}:${tag}" + exit 1 +fi + +for i in $(seq 1 3); do + if java -jar wss-unified-agent.jar -c .cirrus/wss-unified-agent.config -apiKey "${MEND_API_KEY}" -product "${WS_PRODUCTNAME}" -project "${STAGING_IMAGE_NAME}:${tag}" -wss.url ${WS_WSS_URL} -docker.scanImages true; then + SCANNED=0 + break + fi + echo "[${i}/3] Retrying to scan image ${STAGING_IMAGE_NAME}:${tag}..." + sleep 5 +done + +if [[ ${SCANNED} -ne 0 ]]; then + echo "Failed to scan image ${STAGING_IMAGE_NAME}:${tag}" + exit 2 +fi + +exit 0 diff --git a/.cirrus/public-scan-task.sh b/.cirrus/public-scan-task.sh new file mode 100755 index 00000000..cde4a5bd --- /dev/null +++ b/.cirrus/public-scan-task.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +PUBLIC_IMAGE_NAME=${1} +tag=${2} +WS_PRODUCTNAME=${3} +WS_WSS_URL=${4} +MEND_API_KEY=${5} + +IMAGE_PULLED=1 +SCANNED=1 + +echo "Scan the ${PUBLIC_IMAGE_NAME}:${tag} image" + +for i in $(seq 1 3); do + if docker pull "${PUBLIC_IMAGE_NAME}:${tag}"; then + IMAGE_PULLED=0 + break + fi + echo "[${i}/3] Retrying to pull image ${PUBLIC_IMAGE_NAME}:${tag}..." + sleep 5 +done + +if [[ ${IMAGE_PULLED} -ne 0 ]]; then + echo "Failed to pull image ${PUBLIC_IMAGE_NAME}:${tag}" + exit 1 +fi + +for i in $(seq 1 3); do + if java -jar wss-unified-agent.jar -c .cirrus/wss-unified-agent.config -apiKey $MEND_API_KEY -product ${WS_PRODUCTNAME} -project ${PUBLIC_IMAGE_NAME}:${tag} -wss.url ${WS_WSS_URL} -docker.scanImages true; then + SCANNED=0 + break + fi + echo "[${i}/3] Retrying to scan image ${PUBLIC_IMAGE_NAME}:${tag}..." + sleep 5 +done + +if [[ ${SCANNED} -ne 0 ]]; then + echo "Failed to scan image ${PUBLIC_IMAGE_NAME}:${tag}" + exit 2 +fi + +exit 0 From 16dd63dce092896e50268b5e97b371e38e49dcb4 Mon Sep 17 00:00:00 2001 From: Carmine Vassallo Date: Mon, 9 Oct 2023 11:17:07 +0200 Subject: [PATCH 2/2] Refactor pull script --- .cirrus.yml | 10 +++++++-- .cirrus/private-scan-task.sh | 43 ------------------------------------ .cirrus/public-scan-task.sh | 42 ----------------------------------- .cirrus/pull.sh | 15 +++++++++++++ 4 files changed, 23 insertions(+), 87 deletions(-) delete mode 100755 .cirrus/private-scan-task.sh delete mode 100755 .cirrus/public-scan-task.sh create mode 100755 .cirrus/pull.sh diff --git a/.cirrus.yml b/.cirrus.yml index 61519450..6752de31 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -79,12 +79,15 @@ private_scan_task: - curl -sSL https://unified-agent.s3.amazonaws.com/wss-unified-agent.jar -o wss-unified-agent.jar - echo "docker.includes=${tag}" >> .cirrus/wss-unified-agent.config scan_script: - - .cirrus/private-scan-task.sh "${STAGING_IMAGE_NAME}" "${tag}" "${platform}" "${WS_PRODUCTNAME}" "${WS_WSS_URL}" "${MEND_API_KEY}" + - echo "Scan the ${STAGING_IMAGE_NAME}:${tag} image supporting linux/${platform}" + - .cirrus/pull.sh ${STAGING_IMAGE_NAME} ${tag} ${platform} + - java -jar wss-unified-agent.jar -c .cirrus/wss-unified-agent.config -apiKey $MEND_API_KEY -product ${WS_PRODUCTNAME} -project ${STAGING_IMAGE_NAME}:${tag} -wss.url ${WS_WSS_URL} -docker.scanImages true depends_on: multi_arch_build public_scan_task: only_if: $CIRRUS_CRON == 'nightly-mend-scan' env: + platform: amd64 MEND_API_KEY: VAULT[development/kv/data/mend data.apikey] WS_WSS_URL: https://saas-eu.whitesourcesoftware.com/agent matrix: @@ -102,7 +105,10 @@ public_scan_task: - curl -sSL https://unified-agent.s3.amazonaws.com/wss-unified-agent.jar -o wss-unified-agent.jar - echo "docker.includes=${tag}" >> .cirrus/wss-unified-agent.config scan_script: - - .cirrus/public-scan-task.sh "${PUBLIC_IMAGE_NAME}" "${tag}" "${WS_PRODUCTNAME}" "${WS_WSS_URL}" "${MEND_API_KEY}" + - echo "Scan the ${PUBLIC_IMAGE_NAME}:${tag} image" + - docker pull "${PUBLIC_IMAGE_NAME}:${tag}" + - .cirrus/pull.sh ${PUBLIC_IMAGE_NAME} ${tag} ${platform} + - java -jar wss-unified-agent.jar -c .cirrus/wss-unified-agent.config -apiKey $MEND_API_KEY -product ${WS_PRODUCTNAME} -project ${PUBLIC_IMAGE_NAME}:${tag} -wss.url ${WS_WSS_URL} -docker.scanImages true multi_arch_test_task: matrix: diff --git a/.cirrus/private-scan-task.sh b/.cirrus/private-scan-task.sh deleted file mode 100755 index 6887264f..00000000 --- a/.cirrus/private-scan-task.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash - -STAGING_IMAGE_NAME=${1} -tag=${2} -platform=${3} -WS_PRODUCTNAME=${4} -WS_WSS_URL=${5} -MEND_API_KEY=${6} - -IMAGE_PULLED=1 -SCANNED=1 - -echo "Scan the ${STAGING_IMAGE_NAME}:${tag} image supporting linux/${platform}" - -for i in $(seq 1 3); do - if docker pull --platform linux/"${platform}" "${STAGING_IMAGE_NAME}:${tag}"; then - IMAGE_PULLED=0 - break - fi - echo "[${i}/3] Retrying to pull image ${STAGING_IMAGE_NAME}:${tag}..." - sleep 5 -done - -if [[ ${IMAGE_PULLED} -ne 0 ]]; then - echo "Failed to pull image ${STAGING_IMAGE_NAME}:${tag}" - exit 1 -fi - -for i in $(seq 1 3); do - if java -jar wss-unified-agent.jar -c .cirrus/wss-unified-agent.config -apiKey "${MEND_API_KEY}" -product "${WS_PRODUCTNAME}" -project "${STAGING_IMAGE_NAME}:${tag}" -wss.url ${WS_WSS_URL} -docker.scanImages true; then - SCANNED=0 - break - fi - echo "[${i}/3] Retrying to scan image ${STAGING_IMAGE_NAME}:${tag}..." - sleep 5 -done - -if [[ ${SCANNED} -ne 0 ]]; then - echo "Failed to scan image ${STAGING_IMAGE_NAME}:${tag}" - exit 2 -fi - -exit 0 diff --git a/.cirrus/public-scan-task.sh b/.cirrus/public-scan-task.sh deleted file mode 100755 index cde4a5bd..00000000 --- a/.cirrus/public-scan-task.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -PUBLIC_IMAGE_NAME=${1} -tag=${2} -WS_PRODUCTNAME=${3} -WS_WSS_URL=${4} -MEND_API_KEY=${5} - -IMAGE_PULLED=1 -SCANNED=1 - -echo "Scan the ${PUBLIC_IMAGE_NAME}:${tag} image" - -for i in $(seq 1 3); do - if docker pull "${PUBLIC_IMAGE_NAME}:${tag}"; then - IMAGE_PULLED=0 - break - fi - echo "[${i}/3] Retrying to pull image ${PUBLIC_IMAGE_NAME}:${tag}..." - sleep 5 -done - -if [[ ${IMAGE_PULLED} -ne 0 ]]; then - echo "Failed to pull image ${PUBLIC_IMAGE_NAME}:${tag}" - exit 1 -fi - -for i in $(seq 1 3); do - if java -jar wss-unified-agent.jar -c .cirrus/wss-unified-agent.config -apiKey $MEND_API_KEY -product ${WS_PRODUCTNAME} -project ${PUBLIC_IMAGE_NAME}:${tag} -wss.url ${WS_WSS_URL} -docker.scanImages true; then - SCANNED=0 - break - fi - echo "[${i}/3] Retrying to scan image ${PUBLIC_IMAGE_NAME}:${tag}..." - sleep 5 -done - -if [[ ${SCANNED} -ne 0 ]]; then - echo "Failed to scan image ${PUBLIC_IMAGE_NAME}:${tag}" - exit 2 -fi - -exit 0 diff --git a/.cirrus/pull.sh b/.cirrus/pull.sh new file mode 100755 index 00000000..b38bcbf6 --- /dev/null +++ b/.cirrus/pull.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +IMAGE_NAME=${1} +tag=${2} +platform=${3} + +for i in $(seq 1 3); do + if docker pull --platform linux/"${platform}" "${IMAGE_NAME}:${tag}"; then + exit 0 + fi + echo "[${i}/3] Retrying to pull image ${IMAGE_NAME}:${tag}..." + sleep 5 +done +echo "[Error]: Failed to pull image ${IMAGE_NAME}:${tag}" +exit 1