development/docker/CentOS7Slim.Dockerfile → Linux/development/docker/CentOS7Slim.Dockerfile

@@ -18,42 +18,52 @@ FROM centos:7
 RUN yum install -y \
   https://repo.ius.io/ius-release-el7.rpm && \
   yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
-  
+
 # Add target user
 RUN useradd --uid 911 --user-group --shell /bin/bash --create-home abc
 
 # Install minimal developer tools
 RUN yum install -y \
   ccache \
   curl \
+  gcc \
   git236 \
   graphviz \
   libXScrnSaver \
-  openssl \
+  make \
   pciutils-libs \
   perl-Digest-MD5 \
   sudo \
+  wget \
   which \
   xorg-x11-server-Xvfb && \
   # Clean up
-  rm -rf /tmp/* /var/tmp/*
+  rm -rf /tmp/* /var/tmp/* 
+
+
+# Install patched version of OpenSSL (v1.1.1t)
+COPY ./install_openssl.sh /tmp/
+RUN bash /tmp/install_openssl.sh && \
+   rm -rf /OpenSSL
 
+# Install latex
 COPY ./install_latex.sh /tmp/
 RUN bash /tmp/install_latex.sh && \
    rm -rf /latex
 
-#Set paths for latex
+# Set paths for latex
 ENV PATH=/usr/local/texlive/2022/bin/x86_64-linux:$PATH
 ENV MANPATH=$MANPATH:/usr/local/texlive/2022/texmf-dist/doc/man
 ENV INFOPATH=$INFOPATH:/usr/local/texlive/2022/texmf-dist/doc/info
 
-#install anyfontsize package
+# install anyfontsize package
 RUN tlmgr install anyfontsize
-  
+
 # Copy in cppcheck
 COPY --from=upstream_cppcheck /usr/bin/cppcheck /usr/local/bin/
 COPY --from=upstream_cppcheck /usr/bin/cppcheck-htmlreport /usr/local/bin/
 COPY --from=upstream_cppcheck /cfg/ /cfg/
+
 # Fix-up for Python3
 RUN sed -e '1 s@python@python3@' -i /usr/local/bin/cppcheck-htmlreport
 

development/docker/build_common_slim.sh → Linux/development/docker/build_common_slim.sh

@@ -3,6 +3,6 @@
 # Use GitHub packages container registry
 REGISTRY="ghcr.io"
 ORG="mantidproject"
-VERSION="0.13"
+VERSION="0.14"
 
 BUILD_LOG_DIR="build_logs"

Linux/development/docker/install_openssl.sh

@@ -0,0 +1,11 @@
+#! /bin/bash
+
+#Install OpenSSL
+mkdir -p /OpenSSL
+cd /OpenSSL
+wget https://www.openssl.org/source/openssl-1.1.1t.tar.gz --no-check-certificate
+zcat openssl-1.1.1t.tar.gz | tar xf -
+cd openssl-1.1.1t
+./config --prefix=/usr --openssldir=/etc/ssl --libdir=lib no-shared
+make
+make install

Linux/jenkins-node/ansible/group_vars/all/users.yml

@@ -0,0 +1,26 @@
+---
+interactive_users:
+  # Conor Finn
+  - username: mkq48465
+    sudoer: true
+    ssh_pubkey: "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC20VCiUGDsTcOjrKuPG6LsoN3Q3pLFcndV0jwfUBR3O0p2EszPSEsZimq3+wU5JR1IxWcX6lnB7LgePoIs/QHkEaaBn7C1W1/hv+GP3GDaj27u/DHGmbWJpbgD7PPpoD18DL0XM2T9H80SNWfWKOb39yC8uEONSYos7vHP+0abpZo/+VTy0RUu/YFoMi8CugsnGeMeyOha4lEjkc0SBD+ZF53awovsIjKFIIx8AnnUFpdPJ3vliUfXusq+dZQac4XDkuZ3hoLnNRLPKezkQRsrGgABcCvddN7zDSQy59y/BX1CU9pSooXXAEAzH8eMeOSpYfLtxQca3au4hRxzkNcQE1JNDPiN7spYzLjMxQNwCaQ7OJnE57v4kPwrkkB4Ep326bsvvucdxY+B6RLmp51dnQGUvie7PsgS7s6Q51k2JcWgLaxRuvfpSOWaHPIbITar8+BfIPF+5DhiELGSIlShtVlrkxrYXO1ggvgaojfAhKV1SxabGeuOVEsRAzOSwBc= conor.finn@NDLT1581MAC"
+
+  # Sarah Foxley
+  - username: ekg26817
+    sudoer: true
+    ssh_pubkey: "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC8YsKwSxAKLEd4RrjC0vTpOzC5vCnxGAAnqpIuwwedkIzG10ThhsiApaZu9y5jGUQ1Umjz1tUeClJqZM5WoBSlbbCbaiwdiQJWruGzmgioV+DgJxQScJPFYs3GHgoNFscu6rna6BqHw2wT/8jF1C+jB8OZoNytHtxpCf5221bNcZdH3W/w5XXzKv2h3RlJzuXyAT/3npZChvJa6IWg4wcjW/iP7mp/V5h6w8kNvLn68PcZoS/kA2nkl+WH6NfD4+EuDtOA2+QCyiX0JM2b9wIYCmaKDdUSjc5aiqjfoMek/rw/OKdR2GE5E8PH3aF6R+1GBClAK/Jvh2FNPKC3ghmNxp9fmPwb+E1qAMDaR2BDAWK7EE1fZCR8VELE5MMiCZgstzHQage45wx/ELG5UEyy+dmn7hsUJz9yQusogEaKzP4hi580tguwsaDppjAUf6CezBVQhNdMwkql6SLZEW+eNgV45G+vI3UGxLoyhgFp8yOmWsQcpOJzBUbQJMSrQDk= CLRC+ekg26817@NDLT1554"
+
+  # Tom Hampson
+  - username: dsc49661
+    sudoer: true
+    ssh_pubkey: "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCe3et6Rw6b3v8WzERRS0DE1cm33Te3WcUOgzU5xV8tZd0YgTMy1VENG4daZNjlY4ZJ+JMRIehqyOWXEbnJ5d0phdoM6qGj+Wp4WGzvqRHzDcfZC+vAZPsLEwGlcCwLUDihgwpiWePIYVUUuWD5uIEVEPA7mLYvVRwnE6sipGimWXkEpNlRlJZht17w7vCkqFkYx9ULrdXfo29AWkVHqkJcVgBqkg9kglxyerIwy0BXahnDW56WSfPK47ZgcktI7+qu2tu6ZoyyUZZ/rZze3BoQ9f6hJkp0jCN33MPAHZxTL9nkYgG6sWO2TzgM49z6zd2av3pN6NxYA6z5sH9yagMd5L6dSLmMjodfCBdrtyaAgXXhySgq71xtzYUgRVxLLJ/MnVb5vqiExUxAppxYa4JXkeXikTNGXjMPgcs6Bo55i/T7rvjjAQAobK7q6ppYUUBu0V7TM/72P/hC44NChci1FC+6rQcuEJ8I1824pcngdRE432gmb3iBQN7XefUI74s= CLRC+dsc49661@NDLT1577"
+
+  # Silke Schomann
+  - username: eoc57742
+    sudoer: true
+    ssh_pubkey: "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDbIUpR0gYUaCmAx+z0j7amPBy/TxOxott90icoMI+W3PNiyA5hjF1Tfo6yRawFXzGxBbrMc+JNUgOVHXV4P7pIcny+lX9YvSGFuFvROLmNeo9yf0Nyu198tutpHKmejy6nKIlm4IM/IeTzBz0jBcCu//1UNWv/b/hwel8ZYyMCesTcV2janOyg8HvQYMVCkGgQ0c74NAFG55MyL2BrbGlVxkGk7xwan2NmELShyc4pQsOknrMW8/tu10SDCDFB+zFBg8HfeW+qFZ4QMKHPhfp3DqmLMCmUs/W1D+8Os+zH4+MDVqIYiiNWLlnOMc0ulPMHLJQO8LVYcuQcgzdLddyLh1iYOX7ThfmqVqSfblOx7+W9K5yU5+96k4RMd6BEjUqOsg7a91n1boL6OdX8yp/LTMZHAuCr3pFeoilJ/d0IUFU1XcHZwELdqTvifdLnUnxL6JXw7sPFkLRRbxguEH6QzkLJ+CVONf3I9Hi4Zcyf7Pfxmq2qgH0oq418LxY0O5k= clrc\eoc57742@NDLT1439"
+
+  # Robert Applin
+  - username: mlc47243
+    sudoer: true
+    ssh_pubkey: "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDD9RMOb7VJBEesEAaxAzY1GBQMYndAmmkBZMFUBbf97EbXVXCiQWba2doAjSblADXnf4+rodGCrJfEkxaUmM00qRzgxI0wred1XuVzbmH5A16Y3vIBlRlKSUGh9cCGHzabahsMBNzifCsxG73nmCnj+pySvSITGBXcg8/ZeaJu9apZ1TvsmJ+L+ffAs5koqT+DJU40QfR79CJP/TxE3RLNiFa35DtOJH6mHo+4/jUQQLCky1ue2eLilhHu2SHHZNakM9fNdbEUZZwTtib9VGjQxveAmSixC+vag1ITbcaKXfaRd/IkRbv3CSf/4wN+3HPbSkQOxee9bXL+ChEPvypvMGzRHRk7CCA95gxGli0qFT9jJoMDNmDu8paltfGGHubV/lOM4mD/koPkHc8YgtFSwCcq//O3Qk6rhal62zEyw+vKeLzsAb5dnf5p7jc74P3puw96pB7zJTnYxzHMiwrqC9V1pU/emnKuJpbW0y9fQjVODKwvyitfjGxEb2baQLk= mlc47243@mlc47243-OptiPlex-7050"

jenkins-node/mantid-builder/docker-images/CentOS7Slim.Dockerfile → Linux/jenkins-node/docker/CentOS7Slim.Dockerfile

@@ -1,6 +1,6 @@
 # Arguments allowed to be used in FROM have to come
 # before the first stage
-ARG DEVELOPMENT_IMAGE_VERSION=0.13
+ARG DEVELOPMENT_IMAGE_VERSION=0.14
 FROM ghcr.io/mantidproject/mantid-development-centos7-slim:${DEVELOPMENT_IMAGE_VERSION}
 
 #Add label for transparency

jenkins-node/mantid-builder/docker-images/build_common_slim.sh → Linux/jenkins-node/docker/build_common_slim.sh

@@ -3,4 +3,4 @@
 # Use GitHub packages container registry
 REGISTRY="ghcr.io"
 ORG="mantidproject"
-VERSION="0.13"
+VERSION="0.14"

Windows/jenkins-node/Win.Dockerfile

@@ -0,0 +1,63 @@
+# escape=`
+# Use JNLP agent as base and install required tools
+FROM mcr.microsoft.com/windows:10.0.17763.3887-amd64 AS full
+
+RUN xcopy /y C:\Windows\System32\opengl32.dll C:\GatheredDlls\ && `
+	xcopy /y C:\Windows\System32\glu32.dll C:\GatheredDlls\ && `
+	xcopy /y C:\Windows\System32\MF.dll C:\GatheredDlls\ && `
+	xcopy /y C:\Windows\System32\MFPlat.dll C:\GatheredDlls\ && `
+	xcopy /y C:\Windows\System32\MFReadWrite.dll C:\GatheredDlls\ && `
+	xcopy /y C:\Windows\System32\dxva2.dll C:\GatheredDlls\
+
+FROM jenkins/inbound-agent:4.13-2-jdk11-windowsservercore-ltsc2019
+COPY --from=full C:\GatheredDlls\ C:\Windows\System32\
+
+# Reset the shell.
+SHELL ["cmd", "/S", "/C"]
+
+# Install Node.js LTS
+ADD https://nodejs.org/dist/v8.11.3/node-v8.11.3-x64.msi C:\TEMP\node-install.msi
+RUN start /wait msiexec.exe /i C:\TEMP\node-install.msi /l*vx "%TEMP%\MSI-node-install.log" /qn ADDLOCAL=ALL
+
+#Enable long paths
+RUN powershell New-ItemProperty `
+                   -Path "HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem" `
+                   -Name "LongPathsEnabled" `
+                   -Value 1 `
+                   -PropertyType DWORD `
+                   -Force
+
+#Install chocolatey
+ENV ChocolateyUseWindowsCompression false 
+RUN powershell Set-ExecutionPolicy -ExecutionPolicy Unrestricted -Force
+RUN powershell -NoProfile -Command "iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))" && SET "PATH=%PATH%;%ALLUSERSPROFILE%\chocolatey\bin"
+RUN powershell Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Force
+
+#Install git
+RUN choco install git.install -y --no-progress
+
+#Create .bashrc file and set BASH_ENV var
+COPY create_bashrc.bat C:\TEMP\
+RUN powershell Start-Process -FilePath 'C:\TEMP\create_bashrc.bat'
+ENV BASH_ENV C:\Users\Jenkins\.bashrc
+
+#Install visual studio build tools
+RUN `
+    # Download the Build Tools bootstrapper.
+    curl -SL --output vs_buildtools.exe https://aka.ms/vs/16/release/vs_buildtools.exe `
+    `
+    # Install Build Tools with the Microsoft.VisualStudio.Workload.VCTools workload, excluding workloads and components with known issues.
+    && (start /w vs_buildtools.exe --quiet --wait --norestart --nocache `
+        --installPath "%ProgramFiles(x86)%\Microsoft Visual Studio\2019\BuildTools" `
+        --add Microsoft.VisualStudio.Workload.VCTools --includeRecommended`
+        --remove Microsoft.VisualStudio.Component.Windows10SDK.10240 `
+        --remove Microsoft.VisualStudio.Component.Windows10SDK.10586 `
+        --remove Microsoft.VisualStudio.Component.Windows10SDK.14393 `
+        --remove Microsoft.VisualStudio.Component.Windows81SDK `
+        || IF "%ERRORLEVEL%"=="3010" EXIT 0) `
+    `
+    # Cleanup
+    && del /q vs_buildtools.exe
+
+# Start the agent process
+ENTRYPOINT ["powershell.exe", "-f", "C:/ProgramData/Jenkins/jenkins-agent.ps1"]

Windows/jenkins-node/create_bashrc.bat

@@ -0,0 +1 @@
+echo export PATH=$PATH:/c/Program\ Files/Git/bin:/c/Program\ Files/Git/cmd:/c/Program\ Files/Git/usr/bin:$(pwd)/mambaforge:$(pwd)/mambaforge/Library/usr/bin:$(pwd)/mambaforge/Library/bin:$(pwd)/mambaforge/condabin:$(pwd)/mambaforge/Scripts; git config --system core.longpaths true > .bashrc; fi

Windows/jenkins-node/readme.md

@@ -0,0 +1,104 @@
+# Cloud Windows Jenkins Node Setup
+
+This document provides instructions for setting up a cloud Windows Jenkins node on a cloud hosted virtual machine running the `Windows Server 2019` OS.
+
+## Permissions
+
+Before setting up a cloud Windows Jenkins node, you will need to have the following permissions. If you do not have any of these permissions, please contact a member of the RAL Mantid Devops team.
+
+- Admin access to the cloud Windows virtual machine to host the node.
+- Admin access to Jenkins.
+- Read access to the `mantidproject/dockerfiles` repository (https://github.com/mantidproject/dockerfiles).
+
+## Connect to the windows cloud virtual machine
+
+1. Press `Windows key` + `R`. Type in `MSTSC` and hit `Enter` to open `Remote Desktop Connection`.
+2. In the `Computer` field type the name of the cloud virtual machine, for example `isiscloudwin1`.
+3. The `Username` field is typically automatically populated. For ISIS users it will be their username in the `CLRC` domain: `CLRC\<username>`.
+4. Upon clicking `Connect` you will be prompted to enter the password typically associated with your username.
+5. If you have successfully established a remote connection to the virtual machine, a windows desktop will be opened in a new window.
+
+## Install git
+
+1. The easiest way to install `git` is via `chocolatey`. To use `chocolatey`, it must first be installed itself.
+2. Open `powershell` in administrator mode. Run `Set-ExecutionPolicy Bypass -Scope Process -Force`.
+3. Run
+   ```sh
+   [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1'))`
+   ```
+
+4. Confirm chocolatey has been downloaded and installed correctly using `choco --version`.
+5. Download and install `git` using `choco install -y git.install`.
+6. Close and reopen `powershell`.
+7. Confirm git has been downloaded and installed correctly using `git --version`.
+
+## Install Docker
+
+1. Open `powershell` in administrator mode. Run `Set-ExecutionPolicy Bypass -Scope Process -Force`.
+1. Run `Install-Module -Name DockerMsftProvider -Repository PSGallery -Force`. This is the provider that will enable access to the docker online package repository.
+3. Check the `DockerMsftProvider` module has been installed by checking the output of `Get-PackageProvider -ListAvailable`.
+4. Run `Install-Package -Name docker -ProviderName DockerMsftProvider` to install `docker`.
+5. Check that the docker package has been installed using `Get-Package -Name Docker -ProviderName DockerMsftProvider`.
+6. Restart the virtual machine using `Restart-Computer -Force`, then remote back in to the machine.
+7. Reopen `powershell` in administrator mode. Start the `docker` service using `Start-service docker`
+8. Check docker is running correctly using `Docker version` (note the lack of `--`). If docker is available, the following output or similar is to be expected:
+```sh
+Client: Mirantis Container Runtime
+ Version:           20.10.9
+ API version:       1.41
+ Go version:        go1.16.12m2
+ Git commit:        591094d
+ Built:             12/21/2021 21:34:30
+ OS/Arch:           windows/amd64
+ Context:           default
+ Experimental:      true
+
+Server: Mirantis Container Runtime
+ Engine:
+  Version:          20.10.9
+  API version:      1.41 (minimum version 1.24)
+  Go version:       go1.16.12m2
+  Git commit:       9b96ce992b
+  Built:            12/21/2021 21:33:06
+  OS/Arch:          windows/amd64
+  Experimental:     false
+```
+
+## Set up Node on Jenkins
+
+1. Using the `jenkins` web UI (https://builds.mantidproject.org/), navigate to `Manage Jenkins` then `Manage nodes and clouds` under `System Configuration`.
+2. On the side bar, select `New Node`. Enter the Node name using the naming convention `<virtual machine name>-<n>`, `<n>` being the node index base 1 to be hosted on that VM.
+3. Select the `Copy Existing Node` radio button. Type `isiscloudwin1-1` into the emergent text box and click `Create`.
+4. The node configuration will appear, in the `Labels` input box append `-test` to `win-64-cloud` then click `Save`.
+5. Take note of the jenkins secret, an encryption key stated after `-secret` in the code box entitled `Run from agent command line`. This key will be needed to enable access to Jenkins.
+
+## Build Image (Only required upon the setting up of the first windows node on a VM, or following a change to the image).
+
+1. Clone the `mantidproject/dockerfiles` repository (https://github.com/mantidproject/dockerfiles).
+2. Open `powershell` in administrator mode.
+3. `cd` into `<dockerfiles root path>\ Windows\jenkins-node`
+4. Run `docker build -t <docker image name> -f Win.Dockerfile .`
+5. Confirm that the image has successfully been built by viewing the output from `docker images`
+
+## Create container
+1. Open `powershell` in administrator mode.
+2. `cd` into `<dockerfiles root path>\Windows\jenkins-node`
+3. Create a container from the image using the command:
+   ```sh
+   docker run -d --name <cloud node name> --storage-opt "size=250GB" <docker image name> -Url https://builds.mantidproject.org -Secret <jenkins secret> -WorkDir C:/jenkins_workdir -Name <cloud node name>
+   ```
+
+4. Confirm that the container has been created and is listed as running using `docker container ps -a`.
+5. To SSH into the container to access the command line, `docker exec -it <cloud node name> cmd` can be used.
+
+## Testing the new node
+
+1. Log in to the `Jenkins` web UI (https://builds.mantidproject.org/), navigate to `Manage Jenkins`, then `Manage nodes and clouds` under `System Configuration`.
+2. On the `Manage nodes and clouds` page of Jenkins, select the new node from the table.
+3. Ensure the node is online. If `Bring this node back online` is available on the right of screen, select it.
+4. If the cloud node is connected to Jenkins, `Agent is connected` will be stated below the agent name. 
+5. In a new tab, navigate to the Jenkins home page, from the pipeline table select `testing-new-windows-builder`. From the menu on the left-hand side of the page, select `Build Now`.
+6. Switch back to the previous tab, refreshing if necessary. Observe that the new node is running the `testing-new-windows-builder` job.
+7. If the node is not running the job, there may be other nodes with the `win-64-cloud-test` label. Either bring these other nodes offline (ensuring they are not undertaking any jobs) or raise the `Preference Score` of the desired node via `Configure` and rerun the job.
+8. From the menu on the left-hand side of the page, select `Script Console`, ensure that the job status is successful upon conclusion, and that all the tests pass (note that the job status cannot solely be relied upon).
+9. Finally, change the label on the node back to `win-64-cloud` via `Configure` (and reset the preference score if applicable). This will enable the node to take part in the CI/CD pipeline.