Merge pull request #281 from aztfmod/agents_august_updates

Agents august updates

.env.terraform

@@ -1,3 +1,3 @@
-1.2.5
+1.2.9
 1.1.9
-1.3.0-alpha20220706
+1.3.0-rc1

Dockerfile

@@ -214,6 +214,7 @@ RUN apt-get update && \
     echo "Installing latest Azure CLI ..." && \
     pip3 install azure-cli  && \
     az extension add --name ${extensionsAzureCli} --system && \
+    az extension add --name containerapp --system && \
     az config set extension.use_dynamic_install=yes_without_prompt && \
     #
     # Install checkov

agents/azure_devops/Dockerfile

@@ -65,5 +65,11 @@ RUN echo "versionRover=${versionRover}" && \
     rm -rf /home/vscode/agent/externals/node && \
     rm -rf /home/vscode/agent/externals/node10 && \
     rm -rf /tmp/*
+    #
+    # install node 16
+    #
+    curl -sL https://deb.nodesource.com/setup_16.x | sudo -E bash -
+    sudo apt-get install -y nodejs
+    node --version
 
 CMD /bin/bash -c ./azdo.sh

agents/github/github.sh

@@ -1,43 +1,68 @@
 #!/bin/bash
 
-set -Ee
+trap_with_arg() {
+    func="$1" ; shift
+    for sig; do
+        trap "$func $sig" "$sig"
+    done
+}
 
-function finally {
+function finally() 
+{
   echo "Trapped: $1"
+  trap - 0
   echo "Un-register the runner"
-  ./config.sh remove --unattended
+  ./config.sh remove --token ${AGENT_TOKEN}
 }
 
-trap finally EXIT SIGTERM SIGKILL TERM
+trap_with_arg finally 0 1 2 3 4 5 6 7 8 9 15
 
-AGENT_NAME=${AGENT_NAME:="agent"}
+AGENT_SUFFIX=$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 5 | head -n 1)
+AGENT_NAME=${AGENT_NAME:="agent-${AGENT_SUFFIX}"}
 
 if [ -n "${AGENT_TOKEN}" ]; then
   echo "Connect to GitHub using AGENT_TOKEN environment variable."
-else
+elif [ -n "${GH_TOKEN}" ]; then
+  echo "Connect to GitHub using GH_TOKEN environment variable to retrieve registration token."
+  AGENT_TOKEN=$(curl -sX POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${GH_TOKEN}" https://api.github.com/repos/${GH_OWNER}/${GH_REPOSITORY}/actions/runners/registration-token | jq -r .token)
+elif [ -n "${KEYVAULT_NAME}" ]; then
   echo "Connect to Azure AD using MSI ${MSI_ID}"
   az login --identity -u ${MSI_ID} --allow-no-subscriptions
   # Get AGENT_TOKEN from KeyVault if not provided from the AGENT_TOKEN environment variable
   AGENT_TOKEN=$(az keyvault secret show -n ${KEYVAULT_SECRET} --vault-name ${KEYVAULT_NAME} -o json | jq -r .value)
+else
+  echo "You need to provide either AGENT_TOKEN, GH_TOKEN or (MSI_ID, KEYVAULT_NAME and KEYVAULT_SECRET) to start the self-hosted agent."
+  exit 1
 fi
 
-LABELS+=",runner-version-$(./run.sh --version),"
-LABELS+=$(cat /tf/rover/version.txt)
+LABELS+=",$(cat /tf/rover/version.txt)"
 
-# Grant access to the docker socket
-sudo chmod 666 /var/run/docker.sock || true
+if [ -d "/var/run/docker.sock" ]; then
+  # Grant access to the docker socket
+  sudo chmod 666 /var/run/docker.sock
+fi
 
 echo "Configuring the agent with:"
 echo " - url: ${URL}"
 echo " - labels: ${LABELS}"
 echo " - name: ${AGENT_NAME}"
 
-./config.sh \
+command="./config.sh \
   --unattended \
+  --disableupdate \
   --replace \
-  --url ${URL} \
+  --url ${URL}/${GH_OWNER}/${GH_REPOSITORY} \
   --token ${AGENT_TOKEN} \
   --labels ${LABELS} \
-  --name ${AGENT_NAME} \
+  $(if [ "${EPHEMERAL}" = "true" ]; then
+    echo "--ephemeral --name ${AGENT_NAME}"
+  else
+    echo "--name ${AGENT_NAME}"
+  fi)"
+
+echo "running command:" && echo $command
+
+
+eval $command
 
 ./run.sh

agents/github/testing/docker-compose.yml

@@ -0,0 +1,10 @@
+---
+version: '3.8'
+
+services:
+  rover-agent:
+    image: rover-agent:1.2.7-2208.170552-local-github
+    build:
+      context: .
+    env_file:
+      - ./variables.env

agents/github/testing/readme.md

@@ -0,0 +1,177 @@
+# Building the rover agent for Github and testing locally
+
+## Clone
+
+Clone the rover repository
+
+## Create a local build
+
+```
+make local
+```
+
+## Docker images
+
+You can see the local images that have been created on your local machine
+
+```
+docker images
+```
+
+Example:
+
+```
+% docker images
+REPOSITORY                         TAG                                            IMAGE ID       CREATED             SIZE
+rover-agent                        1.3.0-alpha20220803-2208.170347-local-github   1e3d4de6a979   36 minutes ago      3.64GB
+rover-agent                        1.3.0-alpha20220803-2208.170347-local-gitlab   4669d7c61fdb   37 minutes ago      3.36GB
+rover-agent                        1.3.0-alpha20220803-2208.170347-local-tfc      9c4fe627e333   37 minutes ago      3.37GB
+rover-agent                        1.1.9-2208.170347-local-gitlab                 83a90f49c291   38 minutes ago      3.37GB
+rover-agent                        1.1.9-2208.170347-local-github                 494cf73b8e29   38 minutes ago      3.64GB
+rover-agent                        1.1.9-2208.170347-local-tfc                    85c2971b0b03   38 minutes ago      3.37GB
+rover-agent                        1.2.7-2208.170347-local-github                 cbb98046d631   39 minutes ago      3.64GB
+rover-agent                        1.2.7-2208.170347-local-gitlab                 281d9383a8b0   39 minutes ago      3.36GB
+rover-agent                        1.2.7-2208.170347-local-tfc                    7c7a0887ba83   39 minutes ago      3.37GB
+localhost:5000/rover-local         1.3.0-alpha20220803-2208.170347                7c24fb68d69b   39 minutes ago      3.31GB
+localhost:5000/rover-local         1.1.9-2208.170347                              01627897a3f2   39 minutes ago      3.32GB
+localhost:5000/rover-local         1.2.7-2208.170347                              d75861cef748   40 minutes ago      3.31GB
+localhost:5000/rover-local         1.3.0-alpha20220803-2208.170329                d8a0a368d9e5   58 minutes ago      3.31GB
+localhost:5000/rover-local         1.1.9-2208.170329                              ba57a6d5fe51   58 minutes ago      3.32GB
+localhost:5000/rover-local         1.2.7-2208.170329                              6bc9c449f8bc   58 minutes ago      3.31GB
+```
+
+## Create a PAT token
+
+Under your Github profile, developer section, create a PAT token and give the following permissions:
+- repo
+- read:org
+
+## Update docker-compose image and variables.env
+
+docker-compose.yml
+```yaml
+---
+version: '3.8'
+
+services:
+  rover-agent:
+    image: rover-agent:1.2.7-2208.170347-local-github
+    build:
+      context: .
+    env_file:
+      - ./variables.env
+```
+
+variables.env
+```yaml
+GH_TOKEN=copy the value from the PAT token
+GH_OWNER_REPOSITORY=owner/repo
+URL=https://github.com/owner/repo
+LABELS=platform
+EPHEMERAL=true
+```
+
+
+Adjust the other variables
+
+## Test the agent is working
+
+```
+cd agents/github/testing
+docker-compose up
+```
+
+output:
+
+```
+% docker-compose up
+[+] Running 1/0
+ ⠿ Container testing-rover-agent-1  Recreated                                                                                                  0.1s
+Attaching to testing-rover-agent-1
+testing-rover-agent-1  | Connect to GitHub using GH_TOKEN environment variable to retrieve registration token.
+testing-rover-agent-1  | Configuring the agent with:
+testing-rover-agent-1  |  - url: https://github.com/LaurentLesle/a15
+testing-rover-agent-1  |  - labels: test,runner-version-2.294.0
+testing-rover-agent-1  | Runner listener exit with 0 return code, stop the service, no retry needed.
+testing-rover-agent-1  | Exiting runner...,localhost:5000/rover-local:1.2.7-2208.170442
+testing-rover-agent-1  |  - name: agent-2kasm
+testing-rover-agent-1  | 
+testing-rover-agent-1  | --------------------------------------------------------------------------------
+testing-rover-agent-1  | |        ____ _ _   _   _       _          _        _   _                      |
+testing-rover-agent-1  | |       / ___(_) |_| | | |_   _| |__      / \   ___| |_(_) ___  _ __  ___      |
+testing-rover-agent-1  | |      | |  _| | __| |_| | | | | '_ \    / _ \ / __| __| |/ _ \| '_ \/ __|     |
+testing-rover-agent-1  | |      | |_| | | |_|  _  | |_| | |_) |  / ___ \ (__| |_| | (_) | | | \__ \     |
+testing-rover-agent-1  | |       \____|_|\__|_| |_|\__,_|_.__/  /_/   \_\___|\__|_|\___/|_| |_|___/     |
+testing-rover-agent-1  | |                                                                              |
+testing-rover-agent-1  | |                       Self-hosted runner registration                        |
+testing-rover-agent-1  | |                                                                              |
+testing-rover-agent-1  | --------------------------------------------------------------------------------
+testing-rover-agent-1  | 
+testing-rover-agent-1  | # Authentication
+testing-rover-agent-1  | 
+testing-rover-agent-1  | 
+testing-rover-agent-1  | √ Connected to GitHub
+testing-rover-agent-1  | 
+testing-rover-agent-1  | # Runner Registration
+testing-rover-agent-1  | 
+testing-rover-agent-1  | 
+testing-rover-agent-1  | 
+testing-rover-agent-1  | 
+testing-rover-agent-1  | √ Runner successfully added
+testing-rover-agent-1  | √ Runner connection is good
+testing-rover-agent-1  | 
+testing-rover-agent-1  | # Runner settings
+testing-rover-agent-1  | 
+testing-rover-agent-1  | 
+testing-rover-agent-1  | √ Settings Saved.
+testing-rover-agent-1  | 
+testing-rover-agent-1  | 
+testing-rover-agent-1  | √ Connected to GitHub
+testing-rover-agent-1  | 
+testing-rover-agent-1  | Current runner version: '2.294.0'
+testing-rover-agent-1  | 2022-08-16 20:55:23Z: Listening for Jobs
+```
+
+You can check the runner is running under the Github project runner section
+
+From another terminal **docker-compose down** to clean up and de-register the container
+
+```
+docker-compose down 
+```
+
+## Test the agent is working in auto-scaling mode
+
+
+```
+docker-compose up --scale rover-agent=3 -d
+```
+
+output:
+
+```
+% docker-compose up --scale rover-agent=3 -d
+[+] Running 3/3
+ ⠿ Container testing-rover-agent-3  Started                                                                                                    0.5s
+ ⠿ Container testing-rover-agent-1  Started                                                                                                    0.8s
+ ⠿ Container testing-rover-agent-2  Started                                                                                                    0.8s
+```
+
+You can adjust the number of runners
+
+```
+docker-compose up --scale rover-agent=1 -d
+```
+
+## Stop all agents
+
+```
+% docker-compose down
+```
+
+```
+% docker-compose down     
+[+] Running 2/2
+ ⠿ Container testing-rover-agent-1  Removed                                                                                                   10.3s
+ ⠿ Network testing_default          Removed                                                                                                    0.3s
+```

agents/github/testing/variables.env

@@ -0,0 +1,5 @@
+GH_TOKEN=replace PAT token
+GH_OWNER_REPOSITORY=owner/repo
+URL=https://github.com/owner/repo
+LABELS=test
+EPHEMERAL=true

docker-bake.override.hcl

@@ -1,16 +1,16 @@
 # Base image tools version
 extensionsAzureCli="resource-graph"
-versionDockerCompose="2.6.1"
+versionDockerCompose="2.9.0"
 versionGolang="1.18.4"
 versionKubectl="1.24.3"
-versionPacker="1.8.2"
+versionPacker="1.8.3"
 versionPowershell="7.2.5"
 versionTerraformDocs="0.16.0"
 versionVault="1.11.1"
 
 # Agents versions
 USERNAME="vscode"
-versionAzdo="2.204.0"
-versionGithubRunner="2.294.0"
+versionAzdo="2.206.1"
+versionGithubRunner="2.295.0"
 versionGitlab="15.2.1"
-versionTfc="1.2.6"
+versionTfc="1.3.0"