Tfcloud migrate (#638)

* fix(tf): Move terraform state to S3.

* Remove pageryduty.

* Pagerduty fixes.

* More pagerduty fixes.

* Fix lint.

* Fix lint and improve the lint commands.

* Remove fix-dry-run from lint-check.

Author: cass-moz

.aws/src/backfillLambda.ts

@@ -80,20 +80,6 @@ export class BackfillLambda extends Resource {
             ],
           },
         ],
-        alarms: {
-          errors: {
-            // The backfill lambda is throttled to concurrency of 10.
-            evaluationPeriods: 1,
-            comparisonOperator: 'GreaterThanOrEqualToThreshold',
-            period: 1800, // 30 minutes
-            // approx. 5% failure rate (taken from test runs on EN_INTL,
-            // which is the shortest backfill run)
-            threshold: 150,
-            actions: config.isDev
-              ? []
-              : [pagerDuty!.snsNonCriticalAlarmTopic.arn],
-          },
-        },
       },
       tags: config.tags,
     });

.aws/src/datasyncLambda.ts

@@ -260,10 +260,8 @@ export class DatasyncLambda extends Resource {
       period: period,
       threshold: threshold,
       statistic: 'Sum',
-      alarmActions: config.isDev
-        ? []
-        : [this.pagerDuty.snsCriticalAlarmTopic.arn],
-      okActions: config.isDev ? [] : [this.pagerDuty.snsCriticalAlarmTopic.arn],
+      alarmActions: [],
+      okActions: [],
     });
   }
 

.aws/src/main.ts

@@ -1,15 +1,9 @@
 import { Construct } from 'constructs';
-import {
-  App,
-  DataTerraformRemoteState,
-  RemoteBackend,
-  TerraformStack,
-} from 'cdktf';
+import { App, S3Backend, TerraformStack } from 'cdktf';
 import { AwsProvider } from '@cdktf/provider-aws/lib/provider';
 import { S3Bucket } from '@cdktf/provider-aws/lib/s3-bucket';
 import { config } from './config';
-import { PocketPagerDuty, PocketVPC } from '@pocket-tools/terraform-modules';
-import { PagerdutyProvider } from '@cdktf/provider-pagerduty/lib/provider';
+import { PocketVPC } from '@pocket-tools/terraform-modules';
 import { LocalProvider } from '@cdktf/provider-local/lib/provider';
 import { NullProvider } from '@cdktf/provider-null/lib/provider';
 import { ArchiveProvider } from '@cdktf/provider-archive/lib/provider';
@@ -23,35 +17,33 @@ class CurationToolsDataSync extends TerraformStack {
     super(scope, name);
 
     new AwsProvider(this, 'aws', { region: 'us-east-1' });
-    new PagerdutyProvider(this, 'pagerduty_provider', { token: undefined });
     new LocalProvider(this, 'local_provider');
     new NullProvider(this, 'null_provider');
     new ArchiveProvider(this, 'archive_provider');
 
-    new RemoteBackend(this, {
-      hostname: 'app.terraform.io',
-      organization: 'Pocket',
-      workspaces: [{ prefix: `${config.name}-` }],
+    new S3Backend(this, {
+      bucket: `mozilla-content-team-${config.environment.toLowerCase()}-terraform-state`,
+      dynamodbTable: `mozilla-content-team-${config.environment.toLowerCase()}-terraform-state`,
+      key: config.name,
+      region: 'us-east-1',
     });
 
     // ** shared infrastructure between backfill and datasync
     const vpc = new PocketVPC(this, 'pocket-shared-vpc');
-    const pagerDuty = this.createPagerDuty();
     //dynamo db to map curatedRecId - scheduledItem's externalId and store approvedItem's externalId
     const idMapperDynamoDb = new DynamoDB(this, 'curation-migration-id-mapper');
 
     // ** infrastructure for backfill process **
     //bucket for storing all the required csv files
     this.createMigrationBucket();
 
-    new BackfillAuthorsLambda(this, 'backfill-author-lambda', vpc, pagerDuty);
+    new BackfillAuthorsLambda(this, 'backfill-author-lambda', vpc);
 
     new BackfillLambda(
       this,
       'backfill-lambda',
       vpc,
       idMapperDynamoDb.curationMigrationTable,
-      pagerDuty,
     );
 
     // ** infrastructure for datasync process **
@@ -60,44 +52,9 @@ class CurationToolsDataSync extends TerraformStack {
       'datasync-lambda',
       vpc,
       idMapperDynamoDb.curationMigrationTable,
-      pagerDuty,
     );
   }
 
-  /**
-   * Create PagerDuty service for alerts
-   * @private
-   */
-  private createPagerDuty() {
-    // don't create any pagerduty resources if in dev
-    if (config.isDev) {
-      return undefined;
-    }
-
-    const incidentManagement = new DataTerraformRemoteState(
-      this,
-      'incident_management',
-      {
-        organization: 'Pocket',
-        workspaces: {
-          name: 'incident-management',
-        },
-      },
-    );
-
-    return new PocketPagerDuty(this, 'pagerduty', {
-      prefix: config.prefix,
-      service: {
-        criticalEscalationPolicyId: incidentManagement
-          .get('policy_default_critical_id')
-          .toString(),
-        nonCriticalEscalationPolicyId: incidentManagement
-          .get('policy_default_non_critical_id')
-          .toString(),
-      },
-    });
-  }
-
   /**
    * Create the migration S3 bucket
    * This bucket is used to store all the required csv files

bin/load-queued-items.ts

@@ -53,7 +53,7 @@ function getRandomInt(min, max) {
 async function insertIntoCuratedFeedItem(
   queuedItem,
   timeLive,
-  trx: Knex.Transaction
+  trx: Knex.Transaction,
 ) {
   const now = Math.floor(new Date().getTime() / 1000);
 
@@ -107,7 +107,7 @@ async function moveQueuedItems(feedId, baseTime) {
       const curatedRecId = await insertIntoCuratedFeedItem(
         queuedItem,
         i * 3600 + baseTime,
-        trx
+        trx,
       );
       await updateQueuedItem(queuedItem['queued_id'], trx);
       await insertTileSource(curatedRecId, trx);
@@ -116,7 +116,9 @@ async function moveQueuedItems(feedId, baseTime) {
     } catch (e) {
       await trx.rollback();
       console.log(
-        `Number of items emptied from the queue before error occurred: ${i - 1}`
+        `Number of items emptied from the queue before error occurred: ${
+          i - 1
+        }`,
       );
       console.log(`Failed at queued item ID: ${queuedItem['queued_id']}`);
       console.log(`Failed for the time_live: ${i * 3600 + baseTime}`);
@@ -137,7 +139,7 @@ async function load() {
   const count = await moveQueuedItems(
     feedId,
     // Starting from the next hour
-    startTime
+    startTime,
   );
 
   return { count, startTime };

bin/rollback-queued-items.ts

@@ -28,7 +28,7 @@ async function rollbackQueuedItems(queuedIds: number[], trx: Knex.Transaction) {
 
 async function rollbackTileSource(
   curatedRecIds: number[],
-  trx: Knex.Transaction
+  trx: Knex.Transaction,
 ) {
   await dbClient('tile_source')
     .delete()
@@ -38,7 +38,7 @@ async function rollbackTileSource(
 
 async function rollbackCuratedItems(
   curatedRecIds: number[],
-  trx: Knex.Transaction
+  trx: Knex.Transaction,
 ) {
   return trx('curated_feed_items')
     .delete()
@@ -70,7 +70,7 @@ async function rollback() {
 rollback()
   .then((res) => {
     console.log(
-      `Successfully rollback feed ID: ${res.feedId} after time_live: ${res.timeLive}`
+      `Successfully rollback feed ID: ${res.feedId} after time_live: ${res.timeLive}`,
     );
     console.log(`Number of items affected back: ${res.curatedRecIds.length}`);
     console.log('Curated Rec IDs: ', JSON.stringify(res.curatedRecIds));

buildspec.yml

@@ -23,7 +23,6 @@ env:
     USERNAME: 'codebuild'
   secrets-manager:
     #Pull in the default terraform cloud token
-    TERRAFORM_TOKEN: 'CodeBuild/Default:terraform_token'
     PAGERDUTY_TOKEN: 'CodeBuild/Default:mozilla_pagerduty_token'
     GITHUB_ACCESS_TOKEN: 'CodeBuild/Default:github_access_token'
 
@@ -35,11 +34,6 @@ phases:
       - |
         . /home/circleci/.codebuild_shims_wrapper.sh
         echo $CODEBUILD_WEBHOOK_HEAD_REF
-        echo Setting Up Terraform Token
-        rc="credentials \"app.terraform.io\" { "
-        rc="${rc} token=\"$TERRAFORM_TOKEN\" "
-        rc="${rc}}"
-        echo "$rc" > ~/.terraformrc
         echo Setting Github Access Token
         echo "//npm.pkg.github.com/:_authToken=${GITHUB_ACCESS_TOKEN}" > ~/.npmrc
         echo Setting environment variables

package.json

@@ -12,8 +12,8 @@
     "test-integration": "jest \"\\.integration\\.ts\" --runInBand",
     "test-all": "npm run test-spec && npm run test-integration",
     "postinstall": "(cd bin && npm install); (cd curation-authors-backfill && npm install); (cd curation-migration-backfill && npm install); (cd curation-migration-datasync && npm install)",
-    "lint-check": "eslint --fix-dry-run \"curation-*/**/*.ts\"",
-    "lint-fix": "eslint --fix \"curation*/**/*.ts\""
+    "lint-check": "eslint \"**/*.ts\"",
+    "lint-fix": "eslint --fix \"**/*.ts\""
   },
   "repository": {
     "type": "git",