feat: Cloud Logging Alert Module (#77)

* add log alert module

* fix README lint

* update link for enabling export data to GCP

* fix error handling

* Rename superadmin use case

* update anomalous terraform sa usage use case

* Update BQ log alerting module documentation

* update BigQuery Log Alerting example

* fix for code review issues

* updated instructions in README files for clarification

* update enabled APIs list and needed IAM roles

.kitchen.yml

@@ -147,3 +147,16 @@ suites:
           backend: gcp
           controls:
             - gcp
+  - name: "bq-log-alerting"
+    driver:
+      root_module_directory: test/fixtures/bq-log-alerting/
+    verifier:
+      systems:
+        - name: local
+          backend: local
+          controls:
+            - gcloud
+        - name: inspec-gcp
+          backend: gcp
+          controls:
+            - gcp

examples/bq-log-alerting/README.md

@@ -0,0 +1,68 @@
+# Example: BigQuery Log Alerting
+
+This example deploys the BigQuery Log Alerting submodule in an existing project.
+
+## Prerequisites
+
+To run this example, you'll need:
+
+- An existing "logging" project
+- A [Log export](https://github.com/terraform-google-modules/terraform-google-log-export) with a [BigQuery destination](https://github.com/terraform-google-modules/terraform-google-log-export/tree/master/modules/bigquery) in the logging project. The export filter should include at least:
+  - "logName: /logs/cloudaudit.googleapis.com%2Factivity"
+  - "logName: /logs/cloudaudit.googleapis.com%2Fdata_access"
+  - "logName: /logs/compute.googleapis.com%2Fvpc_flows"
+- A Terraform Service Account with the [IAM Roles](../../../modules/bq-log-alerting/README.md) listed in the submodule documentation.
+- To enable in the logging project the [APIs](../../../modules/bq-log-alerting/README.md) listed in the submodule documentation.
+- To enable in the logging project [Google App Engine](https://cloud.google.com/appengine).
+To enable it manually use:
+
+```shell
+gcloud app create \
+--region=<REGION> \
+--project=<LOGGING_PROJECT>
+```
+
+**Note 1:** The selected Google App Engine region cannot be changed after creation and only project Owners (`role/owner`) can enable Google App Engine.
+
+**Note 2:** On deployment a Security Command Center Source called "BQ Log Alerts" will be created. If this source already exist due to the submodule been deployed at least once before, you need to obtain the existing Source name to be informed in the terraform variable **source_name**.
+Run:
+
+```shell
+gcloud scc sources describe <ORG_ID> \
+--source-display-name="BQ Log Alerts" \
+--format="value(name)" \
+--impersonate-service-account=<TERRAFORM_SERVICE_ACCOUNT_EMAIL>
+```
+
+The source name format is `organizations/<ORG_ID>/sources/<SOURCE_ID>`.
+
+The [terraform-example-foundation](https://github.com/terraform-google-modules/terraform-example-foundation) can be used as a reference for the creation of the logging project, the service account and the log export.
+
+## Instructions
+
+1. Run `terraform init`
+1. Run `terraform plan` provide the requested variables values and review the output.
+1. Run `terraform apply`
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| logging\_project | The project to deploy the submodule | `string` | n/a | yes |
+| org\_id | The organization ID for the associated services | `string` | n/a | yes |
+| region | Region for BigQuery resources. | `string` | n/a | yes |
+| source\_name | The Security Command Center Source name for the "BQ Log Alerts" Source if the source had been created before. The format is `organizations/<ORG_ID>/sources/<SOURCE_ID>` | `string` | `""` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| bq\_views\_dataset\_id | The ID of the BigQuery Views dataset |
+| cloud\_function\_service\_account\_email | The email of the service account created to be used by the cloud function |
+| cloud\_scheduler\_job | The Cloud Scheduler job instance |
+| cloud\_scheduler\_job\_name | The name of the Cloud Scheduler job created |
+| pubsub\_topic\_name | PubSub topic name |
+| source\_name | The Security Command Center Source name for the "BQ Log Alerts" Source |
+
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

examples/bq-log-alerting/main.tf

@@ -0,0 +1,31 @@
+/**
+ * Copyright 2020 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*****************************
+  Provider configuration
+ ****************************/
+provider "google" {
+  version = "~> 3.30"
+}
+
+module "bq-log-alerting" {
+  source          = "../..//modules/bq-log-alerting"
+  logging_project = var.logging_project
+  region          = var.region
+  org_id          = var.org_id
+  source_name     = var.source_name
+  dry_run         = false
+}

examples/bq-log-alerting/outputs.tf

@@ -0,0 +1,45 @@
+/**
+ * Copyright 2020 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+output "cloud_scheduler_job_name" {
+  value       = module.bq-log-alerting.cloud_scheduler_job_name
+  description = "The name of the Cloud Scheduler job created"
+}
+
+output "pubsub_topic_name" {
+  value       = module.bq-log-alerting.pubsub_topic_name
+  description = "PubSub topic name"
+}
+
+output "cloud_scheduler_job" {
+  value       = module.bq-log-alerting.cloud_scheduler_job
+  description = "The Cloud Scheduler job instance"
+}
+
+output "source_name" {
+  value       = module.bq-log-alerting.source_name
+  description = "The Security Command Center Source name for the \"BQ Log Alerts\" Source"
+}
+
+output "cloud_function_service_account_email" {
+  value       = module.bq-log-alerting.cloud_function_service_account_email
+  description = "The email of the service account created to be used by the cloud function"
+}
+
+output "bq_views_dataset_id" {
+  value       = module.bq-log-alerting.bq_views_dataset_id
+  description = "The ID of the BigQuery Views dataset"
+}

examples/bq-log-alerting/variables.tf

@@ -0,0 +1,36 @@
+/**
+ * Copyright 2020 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+variable "org_id" {
+  description = "The organization ID for the associated services"
+  type        = string
+}
+
+variable "region" {
+  description = "Region for BigQuery resources."
+  type        = string
+}
+
+variable "source_name" {
+  description = "The Security Command Center Source name for the \"BQ Log Alerts\" Source if the source had been created before. The format is `organizations/<ORG_ID>/sources/<SOURCE_ID>`"
+  type        = string
+  default     = ""
+}
+
+variable "logging_project" {
+  description = "The project to deploy the submodule"
+  type        = string
+}

examples/bq-log-alerting/versions.tf

@@ -0,0 +1,19 @@
+/**
+ * Copyright 2020 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+terraform {
+  required_version = ">=0.12.6, <0.14"
+}

modules/bq-log-alerting/.gitignore

@@ -0,0 +1 @@
+logging/cloud_function.zip