improve devops as code integration

provision_ansible.sh

@@ -0,0 +1,6 @@
+apt update
+apt install --yes software-properties-common
+apt-add-repository --yes --update ppa:ansible/ansible
+apt install --yes ansible
+ansible-playbook --version
+echo "-- DONE --"

src/main/resources/ansible_step/ansible-playbook.sh.ftl

@@ -10,6 +10,7 @@
 
 -->
 export PYTHONUNBUFFERED=1
+export ANSIBLE_CONFIG=ansible.cfg
 
 <#assign verbose=""/>
 <#if ansibleController.debug>

src/main/resources/devops-as-code/apply.py

@@ -8,11 +8,48 @@
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #
 
-from overtherepy import LocalConnectionOptions, OverthereHost, OverthereHostSession
+from overtherepy import LocalConnectionOptions, OverthereHost, OverthereHostSession, StringUtils, BashScriptBuilder, CommandResponse, SshConnectionOptions
+from com.xebialabs.overtherepy import PyLoggerExecutionOutputHandler
 from com.xebialabs.overthere import OperatingSystemFamily
-from com.xebialabs.overthere.util import OverthereUtils
+from com.xebialabs.overthere.util import OverthereUtils, MultipleOverthereExecutionOutputHandler, CapturingOverthereExecutionOutputHandler
 from java.nio.charset import Charset
 from java.lang import System
+from com.xebialabs.overthere import CmdLine
+import time
+import sys
+
+
+def local_execute(session, cmd):
+    """
+    Executes the command on the remote system and returns the result
+    :param session: checks the return code is 0. On failure the output is printed to stdout and a system exit is performed
+    :param cmd: Command line as an Array of Strings or String.  A String is split by space.
+    :return: CommandResponse
+    """
+    if isinstance(cmd, basestring):
+        cmd = cmd.split()
+
+    cmdline = CmdLine.build(cmd)
+    capture_so_handler = CapturingOverthereExecutionOutputHandler.capturingHandler()
+    capture_se_handler = CapturingOverthereExecutionOutputHandler.capturingHandler()
+
+    console_so_handler = PyLoggerExecutionOutputHandler.sysoutHandler(session.logger)
+    console_se_handler = PyLoggerExecutionOutputHandler.syserrHandler(session.logger)
+    so_handler = MultipleOverthereExecutionOutputHandler.multiHandler([capture_so_handler, console_so_handler])
+    se_handler = MultipleOverthereExecutionOutputHandler.multiHandler([capture_se_handler, console_se_handler])
+
+    rc = session.get_conn().execute(so_handler, se_handler, cmdline)
+    # wait for output to drain
+    time.sleep(1)
+
+    response = CommandResponse(rc=rc, stdout=capture_so_handler.outputLines, stderr=capture_se_handler.outputLines)
+
+    if response.rc != 0:
+        session.logger.error(StringUtils.concat(response.stdout))
+        session.logger.error(StringUtils.concat(response.stderr))
+        session.logger.error("Exit code {0}".format(response.rc))
+        sys.exit(response.rc)
+    return response
 
 
 ansible_controler = repositoryService.read(ansible_controller_id)
@@ -22,15 +59,16 @@
 print("remote_yaml_file {0}".format(remote_yaml_file))
 
 operating_system = System.getProperty("os.name").lower()
-print(operating_system)
+print("running " + operating_system + " OS on localhost")
 if operating_system.startswith("win"):
     os = OperatingSystemFamily.WINDOWS
 else:
     os = OperatingSystemFamily.UNIX
 
 local_opts = LocalConnectionOptions(os=os)
 local_host = OverthereHost(local_opts)
-local_session = OverthereHostSession(local_host)
+local_session = OverthereHostSession(local_host, stream_command_output=True)
+
 local_yaml_file = local_session.work_dir_file(remote_yaml_file.getName())
 print("local_yaml_file {0}".format(local_yaml_file))
 
@@ -47,6 +85,22 @@
            'yaml_file': local_yaml_file.path,
            }
 
-command_line = "/usr/local/bin/xl --xl-deploy-password {devopsAsCodePassword} --xl-deploy-username {devopsAsCodeUsername} --xl-deploy-url {devopsAsCodeUrl} apply  -f {yaml_file}".format(**context)
+command_line = "/usr/local/bin/xl --xl-deploy-password {devopsAsCodePassword} --xl-deploy-username {devopsAsCodeUsername} --xl-deploy-url {devopsAsCodeUrl} apply  -f {yaml_file} ".format(**context)
 print(command_line)
-response = local_session.execute(command_line)
+
+import subprocess
+process = subprocess.Popen(command_line, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+process.wait()
+if process.returncode != 0:
+    for line in process.stdout:
+        local_session.logger.error(line)
+    for line in process.stderr:
+        local_session.logger.error(line)
+    local_session.logger.error("Exit code {0}".format(process.returncode))
+    sys.exit(process.returncode)
+else:
+    for line in process.stdout:
+        local_session.logger.info(line)
+    for line in process.stderr:
+        local_session.logger.info(line)
+

src/main/resources/xl-rules.xml

@@ -164,20 +164,18 @@
             <expression>deployed.includeDevOpsAsCodeTask and deployed.applyRulesOnNOOP</expression>
         </conditions>
         <steps>
-            <os-script>
+            <jython>
                 <description expression="true">"Apply the devops-as-code definition fetched from {1}".format(deployed.container.host.name, deployed.container.name)</description>
-                <!-- as the ansibleController can be assigned dynamically to the host's container, we force XLD to read it again to set the target-host property -->
-                <target-host expression="true">repositoryService.read(deployed.container.host.ansibleController.host.id)</target-host>
-                <script>devops-as-code/apply.sh.ftl</script>
+                <script>devops-as-code/apply.py</script>
                 <order>90</order>
-                <freemarker-context>
+                <jython-context>
+                    <ansible_controller_id expression="true">deployed.container.host.ansibleController.id</ansible_controller_id>
                     <yaml_file expression="true">
                         "{0}/{1}/tmp/digital.ai_xldeploy.yaml".format(deployed.container.host.ansibleController.devopsAsCodeDirectory, deployed.container.host.name)
                     </yaml_file>
-                    <ansibleController expression="true">repositoryService.read(deployed.container.host.ansibleController.host.id)</ansibleController>
-                </freemarker-context>
-            </os-script>
-
+                    <ansibleController expression="true">deployed.container.host.ansibleController</ansibleController>
+                </jython-context>
+            </jython>
             <jython>
                 <description expression="true">"Add new provisioned CI from {0} to the environment".format(deployed.container.name)</description>
                 <order>95</order>

xebialabs/ansible_controler.yaml

@@ -0,0 +1,56 @@
+---
+apiVersion: xl-deploy/v1
+kind: Applications
+spec:
+- name: Applications/ansible-controller
+  type: udm.Application
+  lastVersion: 1.0.1
+  children:
+  - name: 1.0.0
+    type: udm.ProvisioningPackage
+    deployables:
+    - name: ansible-instance
+      type: aws.ec2.InstanceSpec
+      boundTemplates:
+      - Applications/ansible-controller/1.0.0/ansible-host-template
+      provisioners:
+      - name: ansible-setup
+        type: script.provisioner.Script
+        hostTemplate: Applications/ansible-controller/1.0.0/ansible-host-template
+        file: !file "artifacts/Applications/ansible-controller/1.0.0/ansible-instance/ansible-setup/provision_ansible.sh"
+      instanceName: ansible-controller-instance
+      amiId: ami-08c757228751c5335
+      region: eu-west-3
+      securityGroup:
+      - Name:ansible-controller-sg
+      instanceType: t2.micro
+      keyName: adlere-seminaire
+    - name: ansible-controller-sg
+      type: aws.vpc.SecurityGroupSpec
+      securityGroupName: ansible-controller-sg
+      description: ssh-security (HTTP/SSH)
+      region: eu-west-3
+      inboundRules:
+      - name: ssh
+        type: aws.vpc.SecurityGroupInboundRuleSpec
+        protocol: TCP
+        portRange: "22"
+        source: 0.0.0.0/0
+    templates:
+    - name: ansible-host-template
+      type: template.overthere.SshHost
+      instanceName: ansible-controller-{{%instanceId%}}-host
+      childTemplates:
+      - name: ansible-controler-template
+        type: template.ansible.Controller
+        instanceName: defaultAnsibleController
+        ansiblePlaybookPath: /usr/bin/ansible-playbook
+        ansibleGalaxyPath: /usr/bin/ansible-galaxy
+      tags:
+      - ansible_roles_not_applied
+      os: UNIX
+      connectionType: SUDO
+      address: '{{%publicHostname%}}'
+      username: ubuntu
+      privateKeyFile: /Users/bmoussaud/.ssh/adlere-seminaire.pem
+      sudoUsername: root

xebialabs/application_tomcat.yaml

@@ -0,0 +1,28 @@
+---
+apiVersion: xl-deploy/v1
+kind: Applications
+spec:
+  - name: Applications/java-server-application
+    type: udm.Application
+    lastVersion: 0.1.1
+    children:
+      - name: 0.1.1
+        type: udm.DeploymentPackage
+        orchestrator:
+          - parallel-by-container
+        deployables:
+          - name: tomcat
+            type: ansible.RolesSpec
+            includeDevOpsAsCodeTask: True
+            scanPlaceholders: False
+            file: !file "artifacts/tomcat/0.1.1/roles_tomcat"
+            tags:
+              - app_server
+            roles:
+              - tomcat
+            variables:
+              tomcat_ver: '9.0.30'
+              ui_manager_user: 'manager'
+              ui_manager_pass: 'Str0ngManagerP@ssw4rd'
+              ui_admin_username: 'admin'
+              ui_admin_pass: 'Str0#ngManagerP@@@ssword'

xebialabs/artifacts/Applications/ansible-controller/1.0.0/ansible-instance/ansible-setup/provision_ansible.sh

@@ -0,0 +1,6 @@
+apt update
+apt install --yes software-properties-common
+apt-add-repository --yes --update ppa:ansible/ansible
+apt install --yes ansible
+ansible-playbook --version
+echo "-- DONE --"

xebialabs/artifacts/tomcat/0.1.1/roles_tomcat/roles/tomcat/defaults/main.yml

@@ -0,0 +1,3 @@
+---
+tomcat_archive_url: https://archive.apache.org/dist/tomcat/tomcat-9/v{{ tomcat_ver }}/bin/apache-tomcat-{{ tomcat_ver }}.tar.gz
+tomcat_archive_dest: /tmp/apache-tomcat-{{ tomcat_ver }}.tar.gz

xebialabs/artifacts/tomcat/0.1.1/roles_tomcat/roles/tomcat/handlers/main.yml

@@ -0,0 +1,4 @@
+- name: restart tomcat
+  service:
+    name: tomcat
+    state: restarted

xebialabs/artifacts/tomcat/0.1.1/roles_tomcat/roles/tomcat/tasks/main.yaml

@@ -0,0 +1,8 @@
+---
+- name: Add the OS specific variables
+  include_vars: "{{ item }}"
+  with_first_found:
+    - "{{ ansible_distribution }}{{ ansible_distribution_major_version }}.yml"
+    - "{{ ansible_os_family }}.yml"
+
+- include_tasks: "tomcat-setup-{{ ansible_os_family }}.yml"

xebialabs/artifacts/tomcat/0.1.1/roles_tomcat/roles/tomcat/tasks/tomcat-setup-Debian.yml

@@ -0,0 +1,94 @@
+- name: Ensure the system can use the HTTPS transport for APT.
+  stat:
+    path: /usr/lib/apt/methods/https
+  register: apt_https_transport
+
+- name: Install APT HTTPS transport.
+  apt:
+    name: "apt-transport-https"
+    state: present
+    update_cache: yes
+  when: not apt_https_transport.stat.exists
+
+- name: Install basic packages
+  package:
+    name: ['vim','aptitude','bash-completion','tmux','tree','htop','wget','unzip','curl','git']
+    state: present
+    update_cache: yes
+
+- name: Install Default Java (Debian/Ubuntu)
+  apt:
+    name: default-jdk
+    state: present
+
+- name: Add tomcat group
+  group:
+    name: tomcat
+
+- name: Add "tomcat" user
+  user:
+    name: tomcat
+    group: tomcat
+    home: /usr/share/tomcat
+    createhome: no
+    system: yes
+
+- name: Download Tomcat
+  get_url:
+    url: "{{ tomcat_archive_url }}"
+    dest: "{{ tomcat_archive_dest }}"
+
+- name: Create a tomcat directory
+  file:
+    path: /usr/share/tomcat
+    state: directory
+    owner: tomcat
+    group: tomcat
+
+- name: Extract tomcat archive
+  unarchive:
+    src: "{{ tomcat_archive_dest }}"
+    dest: /usr/share/tomcat
+    owner: tomcat
+    group: tomcat
+    remote_src: yes
+    extra_opts: "--strip-components=1"
+    creates: /usr/share/tomcat/bin
+
+- name: Copy tomcat service file
+  template:
+    src: templates/tomcat.service.j2
+    dest: /etc/systemd/system/tomcat.service
+  when: ansible_service_mgr == "systemd"
+
+- name: Start and enable tomcat
+  service:
+    daemon_reload: yes
+    name: tomcat
+    state: started
+    enabled: yes
+  when: ansible_service_mgr == "systemd"
+- name: Set UI access credentials
+  template:
+    src: tomcat-users.xml.j2
+    dest: /usr/share/tomcat/conf/tomcat-users.xml
+  notify: restart tomcat
+
+- name: Allow access to Manager and Host Manager apps from any IP
+  template:
+    src: context.xml.j2 
+    dest: "{{ item }}"
+  with_items:
+    - /usr/share/tomcat/webapps/host-manager/META-INF/context.xml
+    - /usr/share/tomcat/webapps/manager/META-INF/context.xml
+  notify: restart tomcat
+
+- name: Generate the Devops-As-Code
+  template: src=xldeploy.yaml.j2 dest=/tmp/digital.ai_xldeploy.yaml mode=0755
+
+- name: Fetch the Devops-As-Code yaml file  on the master
+  fetch:
+    src: /tmp/digital.ai_xldeploy.yaml
+    dest: "{{ devops_as_code_directory }}"
+    flat: no
+