-
Notifications
You must be signed in to change notification settings - Fork 0
/
.gitlab-ci.yml
133 lines (119 loc) · 8.49 KB
/
.gitlab-ci.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/docker:latest
stages:
- build
- deploy
services:
- docker:dind
variables:
DOCKER_DRIVER: overlay2
DOCKER_TLS_CERTDIR: ""
DOCKER_HOST: tcp://docker:2375
CI_REGISTRY_IMAGE: ebispot/gwas-sumstats-service
# always build an image tagged with the commit SHA from master
build:
stage: build
script:
- echo "$DOCKER_HUB_PASSWORD" > dhpw.txt
- docker login -u "${DOCKER_HUB_USER}" --password-stdin < dhpw.txt
- docker pull $CI_REGISTRY_IMAGE:latest
- docker build --network=host --cache-from $CI_REGISTRY_IMAGE:latest -t $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA .
- docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
only:
- master
- dev
# if a tag is created create an images with the tag and a latest
build_release:
variables:
GIT_STRATEGY: none
stage: build
script:
- echo "$DOCKER_HUB_PASSWORD" > dhpw.txt
- docker login -u "${DOCKER_HUB_USER}" --password-stdin < dhpw.txt
- docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
- docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:latest
- docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG
- docker push $CI_REGISTRY_IMAGE:latest
- docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG
only:
- tags
deploy_staging_app:
image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/dtzar/helm-kubectl:2.13.1
stage: deploy
script:
- echo "Deploy to staging server"
- mkdir -p /root/.kube
- echo ${PLIVE_KUBECONFIG} | base64 -d > /root/.kube/config
- RABBITPWD=$(kubectl --namespace rabbitmq get secret rabbitmq -o jsonpath="{.data.rabbitmq-password}" | base64 -d)
- helm init --stable-repo-url https://charts.helm.sh/stable
- helm delete --purge gwas-sumstats-service-dev || true
- helm install --name gwas-sumstats-service-dev --set k8Namespace=gwas-dev,image.repository=$CI_REGISTRY_IMAGE,image.tag=$CI_COMMIT_SHA,service.name=gwas-ss-service-dev,worker.name=gwas-ss-worker-dev,image.env.celeryUser=$RABBITUSER,image.env.celeryPwd=$RABBITPWD,image.env.celeryQueue1='prevalDev',image.env.celeryQueue2='postvalDev',image.env.celeryQueue3='metadata-yml-update-sandbox',image.env.gwasEndpointID=$GWAS_ENDPOINT_ID_DEV,image.env.globusSecret=$CLIENT_SECRET,image.env.transferClientID=$TRANSFER_CLIENT_ID_DEV,image.env.clientID=$CLIENT_ID_DEV,image.env.gwasGlobusGroup=$GWAS_GLOBUS_GROUP,image.env.globusHostname=$GLOBUS_HOSTNAME,image.env.mappedCollectionId=$MAPPED_COLLECTION_ID,image.env.storageGatewayId=$STORAGE_GATEWAY_ID,image.env.ftpServer=$FTP_SERVER,image.env.ftpUser=$FTP_USERNAME,image.env.ftpPassword=$FTP_PASSWORD,image.env.mongoURI=$SANDBOX_MONGO_URI,image.env.mongoUser=$SANDBOX_MONGO_USER,image.env.mongoPassword=$SANDBOX_MONGO_PASSWORD,image.env.mongoDB=$SANDBOX_MONGO_DB,image.env.validatedPath=$VALIDATED_PATH_SANDBOX,volume.log.ClaimName=gwas-dev-depo-logs,image.env.singularityCache=$SINGULARITY_CACHEDIR,image.env.storagePath=$STORAGE_PATH_SANDBOX,image.env.computeFarmLogin=$HH_LOGIN_NODE,image.env.computeFarmUser=$CLUSTER_USERNAME,image.env.computeFarmQueueLong=$COMPUTE_FARM_QUEUE_LONG,image.env.httpProxy=$HH_HTTP_PROXY,image.env.httpsProxy=$HH_HTTPS_PROXY,image.env.remoteHttpProxy=$PG_HTTP_PROXY,image.env.remoteHttpsProxy=$PG_HTTPS_PROXY,image.env.stagingPath=$STAGING_PATH_SANDBOX,image.env.swPath=$SW_PATH,replicaCount=1,volume.data.nfsServer=$NFS_SERVER_PROD,volume.data.path=$NFS_PATH_PROD,image.env.uid=$UID,image.env.gid=$GID,image.env.depoAPIToken=$DEPO_API_AUTH_TOKEN,image.env.outputDataPath=$OUTPUT_PATH,image.env.depoPath=$DEPO_PATH ./k8chart/ --wait
environment:
name: sandbox
only:
- master
- dev
deploy_staging_workers:
image: alpine
stage: deploy
before_script:
- apk add openssh-client
- eval $(ssh-agent -s)
- echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
script:
# note the new START_CELERY_WORKERS_SLURM script
- ssh -o StrictHostKeyChecking=no "$WORKER_USER"@"$WORKER_SERVER_SLURM" "sbatch --mem=12000M --time=00:30:00 -D $WORKER_DIR_DEV -o deploy.out -e deploy.err --wrap='./START_CELERY_WORKERS_SLURM.sh $CI_COMMIT_SHA'"
tags:
- gwas
only:
- master
- dev
deploy_prod_workers:
image: alpine
stage: deploy
before_script:
- apk add openssh-client
- eval $(ssh-agent -s)
- echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
script:
# note the new START_CELERY_WORKERS_SLURM script
- ssh -o StrictHostKeyChecking=no "$WORKER_USER"@"$WORKER_SERVER_SLURM" "sbatch --mem=12000M --time=00:30:00 -D $WORKER_DIR -o deploy.out -e deploy.err --wrap='./START_CELERY_WORKERS_SLURM.sh'"
tags:
- gwas
only:
- tags
deploy_fallback_app:
image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/dtzar/helm-kubectl:2.13.1
stage: deploy
script:
- echo "Deploy to production fallback server"
- mkdir -p /root/.kube
- echo ${PFALLBACK_KUBECONFIG} | base64 -d > /root/.kube/config
- RABBITPWD=$(kubectl --namespace rabbitmq get secret rabbitmq -o jsonpath="{.data.rabbitmq-password}" | base64 -d)
- helm init --stable-repo-url https://charts.helm.sh/stable
- helm delete --purge gwas-sumstats-service || true
- helm install --name gwas-sumstats-service --set k8Namespace=gwas,image.repository=$CI_REGISTRY_IMAGE,image.tag=$CI_COMMIT_SHA,image.env.celeryUser=$RABBITUSER,image.env.celeryPwd=$RABBITPWD,image.env.storagePath=$STORAGE_PATH,image.env.computeFarmLogin=$HX_LOGIN_NODE,image.env.computeFarmUser=$CLUSTER_USERNAME,image.env.computeFarmQueueLong=$COMPUTE_FARM_QUEUE_LONG,image.env.httpProxy=$HX_HTTP_PROXY,image.env.httpsProxy=$HX_HTTPS_PROXY,image.env.gwasEndpointID=$GWAS_ENDPOINT_ID,image.env.globusSecret=$CLIENT_SECRET,image.env.transferClientID=$TRANSFER_CLIENT_ID,image.env.clientID=$CLIENT_ID,image.env.ftpServer=$FTP_SERVER,image.env.ftpUser=$FTP_USERNAME,image.env.ftpPassword=$FTP_PASSWORD,image.env.mongoURI=$PROD_MONGO_URI,image.env.mongoUser=$PROD_MONGO_USER,image.env.mongoPassword=$PROD_MONGO_PASSWORD,image.env.mongoDB=$PROD_MONGO_DB,image.env.remoteHttpProxy=$HX_HTTP_PROXY,image.env.remoteHttpsProxy=$HX_HTTPS_PROXY,image.env.stagingPath=$STAGING_PATH,image.env.validatedPath=$VALIDATED_PATH,image.env.swPath=$SW_PATH,image.env.singularityCache=$SINGULARITY_CACHEDIR,image.env.gwasGlobusGroup=$GWAS_GLOBUS_GROUP,replicaCount=3,image.env.globusHostname=$GLOBUS_HOSTNAME,image.env.mappedCollectionId=$MAPPED_COLLECTION_ID,image.env.storageGatewayId=$STORAGE_GATEWAY_ID ./k8chart/ --wait
environment:
name: fallback
when: manual
only:
- tags
deploy_prod_app:
image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/dtzar/helm-kubectl:2.13.1
stage: deploy
script:
- echo "Deploy to production server"
- mkdir -p /root/.kube
- echo ${PLIVE_KUBECONFIG} | base64 -d > /root/.kube/config
- RABBITPWD=$(kubectl --namespace rabbitmq get secret rabbitmq -o jsonpath="{.data.rabbitmq-password}" | base64 -d)
- helm init --stable-repo-url https://charts.helm.sh/stable
- helm delete --purge gwas-sumstats-service || true
- helm install --name gwas-sumstats-service --set k8Namespace=gwas,image.repository=$CI_REGISTRY_IMAGE,image.tag=$CI_COMMIT_SHA,image.env.celeryUser=$RABBITUSER,image.env.celeryPwd=$RABBITPWD,image.env.storagePath=$STORAGE_PATH,image.env.computeFarmLogin=$HH_LOGIN_NODE,image.env.computeFarmUser=$CLUSTER_USERNAME,image.env.computeFarmQueueLong=$COMPUTE_FARM_QUEUE_LONG,image.env.httpProxy=$HH_HTTP_PROX$STORAGE_PATH,image.env.computeFarmLogin=$HH_LOGIN_NODE,image.env.computeFarmUser=$CLUSTER_USERNAME,image.env.computeFarmQueueLong=$COMPUTE_FARM_QUEUE_LONG,image.env.httpProxy=$HH_Y,image.env.httpsProxy=$HH_HTTPS_PROXY,image.env.gwasEndpointID=$GWAS_ENDPOINT_ID,image.env.globusSecret=$CLIENT_SECRET,image.env.transferClientID=$TRANSFER_CLIENT_ID,image.env.clientID=$CLIENT_ID,image.env.ftpServer=$FTP_SERVER,image.env.ftpUser=$FTP_USERNAME,image.env.ftpPassword=$FTP_PASSWORD,image.env.mongoURI=$PROD_MONGO_URI,image.env.mongoUser=$PROD_MONGO_USER,image.env.mongoPassword=$PROD_MONGO_PASSWORD,image.env.mongoDB=$PROD_MONGO_DB,image.env.remoteHttpProxy=$PG_HTTP_PROXY,image.env.remoteHttpsProxy=$PG_HTTPS_PROXY,image.env.stagingPath=$STAGING_PATH,image.env.validatedPath=$VALIDATED_PATH,image.env.swPath=$SW_PATH,image.env.singularityCache=$SINGULARITY_CACHEDIR,image.env.gwasGlobusGroup=$GWAS_GLOBUS_GROUP,replicaCount=3,image.env.globusHostname=$GLOBUS_HOSTNAME,image.env.mappedCollectionId=$MAPPED_COLLECTION_ID,image.env.storageGatewayId=$STORAGE_GATEWAY_ID ./k8chart/ --wait
environment:
name: production
when: manual
only:
- tags