deploy-heketi-configMap.yaml

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: deploy-heketi
data:
  deploy-heketi: |-
    #!/bin/bash -e

    set -x
    mkdir -p /srv/asset/manifests/glusterfs

    /hyperkube kubectl apply -f /var/heketi/manifests/heketi-storage.json
    cp /var/heketi/manifests/heketi-storage.json /srv/asset/manifests/glusterfs/
    cp /var/heketi/manifests/topology.json /srv/asset/manifests/glusterfs/

    # delete all deploy-heketi resources
    /hyperkube kubectl delete all,service,jobs,deployment,secret --selector="deploy-heketi"
    rm -vf /srv/asset/manifests/glusterfs/deploy-heketi-deployment.yaml

    /hyperkube kubectl apply -f /opt/bin/heketi-deployment.yaml


    set +x

    function create_dynamic_provision {
      # using non-capturing groups
      # posible to use dns?
      heketiIP=`/hyperkube kubectl get svc heketi --no-headers -o yaml| grep -Po '(?<=clusterIP: ).*'`
      [ -n ${heketiIP} ] && cat <<EOF | /hyperkube kubectl create -f -
    ---
    apiVersion: storage.k8s.io/v1beta1
    kind: StorageClass
    metadata:
      name: default
    provisioner: kubernetes.io/glusterfs
    parameters:
      resturl: "http://$heketiIP:8080"
    EOF
    }

    until create_dynamic_provision
    do
      echo "heketi service ip unable to find"
    done

    until /hyperkube kubectl create -f /opt/bin/pvc-test.yaml
    do
      /hyperkube kubectl delete -f /opt/bin/pvc-test.yaml
      sleep 2
    done

    until /hyperkube kubectl get pvc pvc-test --no-headers | grep Bound
    do
      sleep 2
    done
    /hyperkube kubectl delete -f /opt/bin/pvc-test.yaml

  pvc-test.yaml: |-
    ---
    kind: "PersistentVolumeClaim"
    apiVersion: "v1"
    metadata:
      name: "pvc-test"
      annotations:
        volume.beta.kubernetes.io/storage-class: "default"
    spec:
      accessModes:
        - "ReadWriteOnce"
      resources:
        requests:
          storage: "1Gi"
  heketi-deployment.yaml: |-
    ---
    kind: Service
    apiVersion: v1
    metadata:
      name: heketi
      labels:
        glusterfs: heketi-service
        deploy-heketi: support
      annotations:
        description: Exposes Heketi Service
    spec:
      selector:
        name: heketi
      ports:
      - name: heketi
        port: 8080
        targetPort: 8080
    ---
    kind: Deployment
    apiVersion: extensions/v1beta1
    metadata:
      name: heketi
      labels:
        glusterfs: heketi-deployment
      annotations:
        description: Defines how to deploy Heketi
    spec:
      replicas: 1
      template:
        metadata:
          name: heketi
          labels:
            name: heketi
            glusterfs: heketi-pod
        spec:
          serviceAccountName: heketi-service-account
          containers:
          - image: heketi/heketi:dev
            imagePullPolicy: IfNotPresent
            name: heketi
            env:
            - name: HEKETI_EXECUTOR
              value: kubernetes
            - name: HEKETI_KUBE_USE_SECRET
              value: "y"
            - name: HEKETI_FSTAB
              value: "/var/lib/heketi/fstab"
            - name: HEKETI_SNAPSHOT_LIMIT
              value: '14'
            - name: HEKETI_KUBE_GLUSTER_DAEMONSET
              value: "y"
            ports:
            - containerPort: 8080
            volumeMounts:
            - name: db
              mountPath: "/var/lib/heketi"
            readinessProbe:
              timeoutSeconds: 3
              initialDelaySeconds: 3
              httpGet:
                path: "/hello"
                port: 8080
            livenessProbe:
              timeoutSeconds: 3
              initialDelaySeconds: 30
              httpGet:
                path: "/hello"
                port: 8080
          volumes:
          - name: db
            glusterfs:
              endpoints: heketi-storage-endpoints
              path: heketidbstorage
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: heketi-topology
data:
  provision: |-
    #!/bin/bash
    KUBE_TOKEN=$(</var/run/secrets/kubernetes.io/serviceaccount/token)
    NODE_DEVICES="[ `echo $NODE_DEVICES_PATTERN | sed -e "s/ /','/g" -e "s/\(.*\)/'\1'/g"` ]"

    function gluster_readiness {
      curl -sSk -H "Authorization: Bearer $KUBE_TOKEN" https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_PORT_443_TCP_PORT/api/v1/pods?labelSelector=glusterfs-node=pod > /tmp/gluster_pods.json 2>&1
      python3 - <<EOF
    import sys
    import json

    if __name__ == '__main__':
        json_raw = json.loads(open('/tmp/gluster_pods.json').read())
        pods = list((x for x in json_raw['items'] if next((c for c in x['status']['conditions'] if c['type'] == 'Ready' and c['status'] == 'True'), None) is not None))
        sys.exit(int(len(pods) < 3))
    EOF
    }

    until gluster_readiness
    do
      echo -ne "glusterfs needs 3 pods at least .."\\r
      sleep 3
    done
    echo "glusterfs is ready"

    function create_topology {
      curl -sSk -H "Authorization: Bearer $KUBE_TOKEN" https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_PORT_443_TCP_PORT/api/v1/nodes?labelSelector=storagenode=glusterfs > /tmp/nodes.json 2>&1

      python3 - <<END
    import sys
    import json
    import argparse
    import os

    raw = json.loads(open('/tmp/nodes.json','r').read())

    def getAddrWithType(dic, t):
        return next((x for x in dic['status']['addresses'] if x['type'] == t))['address']

    if __name__ == '__main__':
        devices = $NODE_DEVICES
        topo = { 'clusters': [ { 'nodes': [] } ] }

        for item in raw['items']:
          ipv4 = getAddrWithType(item, 'LegacyHostIP')
          hostname = getAddrWithType(item, 'Hostname')
          node = {
            'node': {
              'hostnames': {
          'manage' : [ ipv4 ] ,
          'storage': [ hostname ]
              },
              'zone': 1
            },
           'devices': devices
          }
          topo['clusters'][0]['nodes'].append(node)
        f = open('/topology.json', 'w')
        f.write(json.dumps(topo))
        f.close()
        sys.exit(int(len(raw['items']) < 3))

    END
    }

    until create_topology
    do
      echo "createing topology needs at least 3 nodes .."
    done
    echo "topology.json saved"

    cp /topology.json /var/heketi/manifests/

    heketi-cli topology load --json=/topology.json
    sleep 6

    result=`heketi-cli setup-openshift-heketi-storage 2>&1`
    echo $result
    if echo $result | grep -q "Error.*Volume.*heketidbstorage alreay exists"; then
      exit 0
    fi
    cp -fv /heketi-storage.json /var/heketi/manifests/