remove ecs services

pipeline/terraform/2024-11-18/main.tf

@@ -2,7 +2,7 @@ module "pipeline" {
   source = "../modules/stack"
 
   reindexing_state = {
-    listen_to_reindexer      = true
+    listen_to_reindexer      = false
     scale_up_tasks           = false
     scale_up_elastic_cluster = false
     scale_up_id_minter_db    = false

pipeline/terraform/modules/pipeline_indices/main.tf

@@ -16,14 +16,6 @@ module "denormalised_index" {
   allow_delete  = var.allow_delete
 }
 
-module "lambda_denormalised_index" {
-  source        = "../es_index"
-  name          = var.es_lambda_works_denormalised_index
-  mappings_name = "empty"
-  config_path   = var.es_config_path
-  allow_delete  = var.allow_delete
-}
-
 module "images_initial_index" {
   source        = "../es_index"
   name          = var.es_images_initial_index

pipeline/terraform/modules/pipeline_indices/outputs.tf

@@ -2,7 +2,6 @@ output "indices" {
   value = {
     source              = module.source_index
     denormalised        = module.denormalised_index
-    lambda_denormalised = module.lambda_denormalised_index
     images_initial      = module.images_initial_index
     images_augmented    = module.images_augmented_index
     images_indexed      = module.images_indexed_index
@@ -16,7 +15,6 @@ output "index_names" {
   value = {
     source              = module.source_index.name
     denormalised        = module.denormalised_index.name
-    lambda_denormalised = module.lambda_denormalised_index.name
     images_initial      = module.images_initial_index.name
     images_augmented    = module.images_augmented_index.name
     images_indexed      = module.images_indexed_index.name

pipeline/terraform/modules/pipeline_indices/variables.tf

@@ -7,12 +7,10 @@ variable "es_works_merged_index" {
 variable "es_works_identified_index" {
   type = string
 }
+
 variable "es_works_denormalised_index" {
   type = string
 }
-variable "es_lambda_works_denormalised_index" {
-  type = string
-}
 variable "es_works_index" {
   type = string
 }

pipeline/terraform/modules/stack/elastic_pipeline.tf

@@ -113,12 +113,11 @@ module "pipeline_indices" {
 
   allow_delete = var.allow_delete_indices
 
-  es_works_source_index              = local.es_works_source_index
-  es_works_merged_index              = local.es_works_merged_index
-  es_works_identified_index          = local.es_works_identified_index
-  es_works_denormalised_index        = local.es_works_denormalised_index
-  es_lambda_works_denormalised_index = local.es_lambda_works_denormalised_index
-  es_works_index                     = local.es_works_index
+  es_works_source_index       = local.es_works_source_index
+  es_works_merged_index       = local.es_works_merged_index
+  es_works_identified_index   = local.es_works_identified_index
+  es_works_denormalised_index = local.es_works_denormalised_index
+  es_works_index              = local.es_works_index
 
   es_images_initial_index   = local.es_images_initial_index
   es_images_augmented_index = local.es_images_augmented_index
@@ -155,15 +154,15 @@ locals {
     }
     router = {
       read  = [local.indices.works_merged]
-      write = [local.indices.denormalised, local.indices.lambda_denormalised]
+      write = [local.indices.denormalised]
     }
     path_concatenator = {
       read  = [local.indices.works_merged]
       write = [local.indices.works_merged]
     }
     relation_embedder = {
       read  = [local.indices.works_merged]
-      write = [local.indices.denormalised, local.indices.lambda_denormalised]
+      write = [local.indices.denormalised]
     }
     work_ingestor = {
       read  = [local.indices.denormalised]

pipeline/terraform/modules/stack/locals.tf

@@ -1,12 +1,11 @@
 locals {
   namespace = "catalogue-${var.pipeline_date}"
 
-  es_works_source_index              = "works-source-${var.pipeline_date}"
-  es_works_merged_index              = "works-merged-${var.pipeline_date}"
-  es_works_identified_index          = "works-identified-${var.pipeline_date}"
-  es_works_denormalised_index        = "works-denormalised-${var.pipeline_date}"
-  es_lambda_works_denormalised_index = "works-denormalised-lambda-${var.pipeline_date}"
-  es_works_index                     = "works-indexed-${var.pipeline_date}"
+  es_works_source_index       = "works-source-${var.pipeline_date}"
+  es_works_merged_index       = "works-merged-${var.pipeline_date}"
+  es_works_identified_index   = "works-identified-${var.pipeline_date}"
+  es_works_denormalised_index = "works-denormalised-${var.pipeline_date}"
+  es_works_index              = "works-indexed-${var.pipeline_date}"
 
   es_images_initial_index   = "images-initial-${var.pipeline_date}"
   es_images_augmented_index = "images-augmented-${var.pipeline_date}"
@@ -17,8 +16,6 @@ locals {
 
   # The max number of connections allowed by the instance.
   # specified at /infrastructure/critical/rds_id_minter.tf
-  base_rds_instances             = 1
-  id_minter_rds_max_connections  = local.base_rds_instances * 45
   id_minter_task_max_connections = min(9, local.max_capacity)
 
   # We don't want to overload our databases if we're not reindexing

pipeline/terraform/modules/stack/relation_embedder/outputs.tf

@@ -0,0 +1,7 @@
+output "router_work_output_topic_arn" {
+  value = module.router_work_output_topic.arn
+}
+
+output "relation_embedder_output_topic_arn" {
+  value = module.embedder_lambda_output_topic.arn
+}

pipeline/terraform/modules/stack/service_ingestor_works.tf

@@ -32,8 +32,8 @@ module "ingestor_works" {
   container_image = local.ingestor_works_image
 
   topic_arns = [
-    module.router_work_output_topic.arn,
-    module.relation_embedder_output_topic.arn,
+    module.relation_embedder_sub.router_work_output_topic_arn,
+    module.relation_embedder_sub.relation_embedder_output_topic_arn
   ]
 
   # The ingestor_works_flush_interval_seconds will wait for up to ingestor_works_flush_interval_seconds

pipeline/terraform/modules/stack/subsystem_relation_embedder.tf

@@ -6,7 +6,7 @@ module "relation_embedder_sub" {
   reindexing_state = var.reindexing_state
 
   es_works_merged_index               = local.es_works_merged_index
-  es_works_denormalised_index         = local.es_lambda_works_denormalised_index
+  es_works_denormalised_index         = local.es_works_denormalised_index
   pipeline_storage_es_service_secrets = local.pipeline_storage_es_service_secrets
 
   # path_concatenator