Reinstate 2024-11-18 pipeline and fix deployment (#2802)

* reinstate pipeline and fix deployment

* make suffixes align, add comment on parameter expansion in bash

Author: kenoir

builds/deploy_catalogue_pipeline.sh

@@ -92,7 +92,7 @@ then
 
   echo "Deploying λ pipeline services to catalogue-$PIPELINE_DATE"
   "$ROOT/builds/deploy_lambda_services.sh" \
-    batcher \
-    relation_embedder
+    batcher:r_embed_batcher \
+    relation_embedder:r_embed_embedder
 fi
 

builds/deploy_lambda_services.sh

@@ -7,10 +7,17 @@ set -o pipefail
 PIPELINE_NAMESPACE="catalogue-$PIPELINE_DATE"
 REPOSITORY_URI="760097843905.dkr.ecr.eu-west-1.amazonaws.com"
 
-for SERVICE_NAME in "$@"
+for FUNCTION_SUFFIX_IMAGE_NAME_PAIR in "$@"
 do
-  IMAGE_URI="${REPOSITORY_URI}"/uk.ac.wellcome/"${SERVICE_NAME}":"env.${PIPELINE_DATE}"
-  FUNCTION_NAME="${PIPELINE_NAMESPACE}"-"${SERVICE_NAME}"
+  # See https://www.gnu.org/software/bash/manual/html_node/Shell-Parameter-Expansion.html
+  # for an explanation of the syntax used here.
+  # %:* removes the shortest matching pattern of "colon followed by any characters"
+  # #*: removes the shortest matching pattern of "any characters followed by a colon"
+  IMAGE_NAME="${FUNCTION_SUFFIX_IMAGE_NAME_PAIR%:*}"
+  FUNCTION_SUFFIX="${FUNCTION_SUFFIX_IMAGE_NAME_PAIR#*:}"
+
+  IMAGE_URI="${REPOSITORY_URI}"/uk.ac.wellcome/"${IMAGE_NAME}":"env.${PIPELINE_DATE}"
+  FUNCTION_NAME="${PIPELINE_NAMESPACE}"-"${FUNCTION_SUFFIX}"
 
   echo "Deploying ${IMAGE_URI} to ${FUNCTION_NAME}, @ $(date) ..."
 

pipeline/terraform/2024-11-18/main.tf

@@ -1,38 +1,32 @@
-# DEACTIVATED - 2024-12-16
-# This pipeline has been deactivated over the winter break,
-# to save costs and will be reactivated in the new year.
-# We need to keep this folder, so that new deploys don't go
-# to the production environment.
+module "pipeline" {
+  source = "../modules/stack"
 
-# module "pipeline" {
-#   source = "../modules/stack"
-#
-#   reindexing_state = {
-#     listen_to_reindexer      = false
-#     scale_up_tasks           = false
-#     scale_up_elastic_cluster = false
-#     scale_up_id_minter_db    = false
-#     scale_up_matcher_db      = false
-#   }
-#
-#   index_config = {
-#     works = {
-#       identified = "works_identified.2023-05-26"
-#       merged     = "works_merged.2023-05-26"
-#       indexed    = "works_indexed.2024-11-14"
-#     }
-#     images = {
-#       indexed        = "images_indexed.2024-11-14"
-#       works_analysis = "works_indexed.2024-11-06"
-#     }
-#   }
-#
-#   allow_delete_indices = true
-#
-#   pipeline_date = local.pipeline_date
-#   release_label = local.pipeline_date
-#
-#   providers = {
-#     aws.catalogue = aws.catalogue
-#   }
-# }
+  reindexing_state = {
+    listen_to_reindexer      = true
+    scale_up_tasks           = true
+    scale_up_elastic_cluster = true
+    scale_up_id_minter_db    = true
+    scale_up_matcher_db      = true
+  }
+
+  index_config = {
+    works = {
+      identified = "works_identified.2023-05-26"
+      merged     = "works_merged.2023-05-26"
+      indexed    = "works_indexed.2024-11-14"
+    }
+    images = {
+      indexed        = "images_indexed.2024-11-14"
+      works_analysis = "works_indexed.2024-11-06"
+    }
+  }
+
+  allow_delete_indices = false
+
+  pipeline_date = local.pipeline_date
+  release_label = local.pipeline_date
+
+  providers = {
+    aws.catalogue = aws.catalogue
+  }
+}