From 78038854c94bbf7ca1c170ed57be96c1b218f294 Mon Sep 17 00:00:00 2001
From: Vasco Santos <santos.vasco10@gmail.com>
Date: Tue, 19 Sep 2023 08:19:31 +0200
Subject: [PATCH] fix: reduce costs on kinesis streams (#231)

We are currently being billed quite a bit in AWS for all the Kinesis
streams we keep open for dev/CI.

This PR addresses this by:
- not deploying 2 kinesis streams per env to keep historical content. We
really just need the production stream still deployed until we read it.
- we can also decrease the persistence of the streams to keep costs down

As follow ups, we should also consider (I will create issues):
- tool that proactively tears down running infrastructure for stalled
PRs (or setup tests in a way that kinesis is put on and off for the
tests)
- get historical content kinesis stream flushed out
---
 stacks/config.js                | 15 +++++++++++++++
 stacks/ucan-invocation-stack.js | 21 ++++++++++-----------
 2 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/stacks/config.js b/stacks/config.js
index 3a7c9755..65005b19 100644
--- a/stacks/config.js
+++ b/stacks/config.js
@@ -87,6 +87,21 @@ export function getKinesisEventSourceConfig (stack) {
   }
 }
 
+/**
+ * @param {import('@serverless-stack/resources').Stack} stack
+ */
+export function getKinesisStreamConfig (stack) {
+  if (stack.stage !== 'production') {
+    return {
+      retentionPeriod: Duration.hours(24)
+    }
+  }
+
+  return {
+    retentionPeriod: Duration.days(365)
+  }
+}
+
 export function getApiPackageJson () {
   // @ts-expect-error ts thinks this is unused becuase of the ignore
   const require = createRequire(import.meta.url)
diff --git a/stacks/ucan-invocation-stack.js b/stacks/ucan-invocation-stack.js
index cd414b9e..01eb28ca 100644
--- a/stacks/ucan-invocation-stack.js
+++ b/stacks/ucan-invocation-stack.js
@@ -5,13 +5,13 @@ import {
   Queue,
   use
 } from '@serverless-stack/resources'
-import { Duration } from 'aws-cdk-lib'
 
 import { CarparkStack } from './carpark-stack.js'
 import { UploadDbStack } from './upload-db-stack.js'
 import {
   getBucketConfig,
   getKinesisEventSourceConfig,
+  getKinesisStreamConfig,
   setupSentry
 } from './config.js'
 
@@ -190,20 +190,19 @@ export function UcanInvocationStack({ stack, app }) {
   })
 
   // TODO: keep for historical content that we might want to process
-  new KinesisStream(stack, 'ucan-stream', {
-    cdk: {
-      stream: {
-        retentionPeriod: Duration.days(365)
-      }
-    },
-  })
+  // only needed for production
+  if (stack.stage === 'production') {
+    new KinesisStream(stack, 'ucan-stream', {
+      cdk: {
+        stream: getKinesisStreamConfig(stack)
+      },
+    })
+  }
 
   // create a kinesis stream
   const ucanStream = new KinesisStream(stack, 'ucan-stream-v2', {
     cdk: {
-      stream: {
-        retentionPeriod: Duration.days(365)
-      }
+      stream: getKinesisStreamConfig(stack)
     },
     consumers: {
       metricsStoreAddTotalConsumer: {