Skip to content

Commit

Permalink
improve data collection database management (#904)
Browse files Browse the repository at this point in the history
  • Loading branch information
iakov-aws authored Sep 17, 2024
1 parent 3205259 commit dc99f98
Show file tree
Hide file tree
Showing 10 changed files with 64 additions and 1,471 deletions.
4 changes: 2 additions & 2 deletions cfn-templates/cid-cfn.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
AWSTemplateFormatVersion: '2010-09-09'
Description: Deployment of Cloud Intelligence Dashboards v0.3.9
Description: Deployment of Cloud Intelligence Dashboards v0.3.10
Metadata:
AWS::CloudFormation::Interface:
ParameterGroups:
Expand Down Expand Up @@ -1611,7 +1611,7 @@ Resources:
Description: An AWS managed layer with a cid-cmd package installed
Content:
S3Bucket: !Sub '${LambdaLayerBucketPrefix}-${AWS::Region}'
S3Key: 'cid-resource-lambda-layer/cid-0.3.9.zip' #replace version here if needed
S3Key: 'cid-resource-lambda-layer/cid-0.3.10.zip' #replace version here if needed
CompatibleRuntimes:
- python3.10
- python3.11
Expand Down
2 changes: 1 addition & 1 deletion cid/_version.py
Original file line number Diff line number Diff line change
@@ -1 +1 @@
__version__ = '0.3.9'
__version__ = '0.3.10'
33 changes: 23 additions & 10 deletions cid/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -348,16 +348,29 @@ def get_template_parameters(self, parameters: dict, param_prefix: str='', others
choices=self.cur.tag_and_cost_category_fields + ["'none'"],
)
elif isinstance(value, dict) and value.get('type') == 'athena':
if 'query' not in value:
raise CidCritical(f'Failed fetching parameter {prefix}{key}: paramter with type ahena must have query value.')
query = value['query']
try:
res = self.athena.query(query)[0]
except (self.athena.client.exceptions.ClientError, CidError, CidCritical) as exc:
raise CidCritical(f'Failed fetching parameter {prefix}{key}: {exc}') from exc
if not res:
raise CidCritical(f'Failed fetching parameter {prefix}{key}, {value}. Athena returns empty result')
params[key] = res[0]
if get_parameters().get(prefix + key): # priority to user input
params[key] = get_parameters().get(prefix + key)
else:
if 'query' not in value:
raise CidCritical(f'Failed fetching parameter {prefix}{key}: parameter with type Athena must have query value.')
query = value['query']
try:
res_list = self.athena.query(query)
except (self.athena.client.exceptions.ClientError, CidError, CidCritical) as exc:
raise CidCritical(f'Failed fetching parameter {prefix}{key}: {exc}') from exc
if not res_list:
raise CidCritical(f'Failed fetching parameter {prefix}{key}, {value}. Athena returns empty results')
elif len(res_list) == 1:
params[key] = '-'.join(res_list[0])
else:
options = ['-'.join(res) for res in res_list]
default = value.get('default')
params[key] = get_parameter(
param_name=prefix + key,
message=f"Required parameter: {key} ({value.get('description')})",
choices=options,
default=default if default in options else None,
)
elif isinstance(value, dict):
params[key] = value.get('value')
while params[key] is None:
Expand Down
12 changes: 12 additions & 0 deletions dashboards/aws-feeds/aws-feeds.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2939,6 +2939,7 @@ datasets:
- aws_feeds
schedules:
- default

aws_feeds_blog_post:
data:
DataSetId: 173bf994-4763-4044-a96d-1cb6cce591f4
Expand Down Expand Up @@ -2998,9 +2999,12 @@ datasets:
- default
parameters:
data_collection_database_name:
type: athena
query: SELECT DISTINCT table_schema FROM information_schema.columns WHERE table_name = 'aws_feeds_youtube'
default: optimization_data
description: "Enter the name of the data collection database"
global: True

aws_feeds_security_bulletin:
data:
DataSetId: cd5b5e17-fdde-4563-a944-1c5fc3b27684
Expand Down Expand Up @@ -3054,9 +3058,12 @@ datasets:
- default
parameters:
data_collection_database_name:
type: athena
query: SELECT DISTINCT table_schema FROM information_schema.columns WHERE table_name = 'aws_feeds_youtube'
default: optimization_data
description: "Enter the name of the data collection database"
global: True

aws_feeds_whats_new:
data:
DataSetId: 00e4f86c-96b8-49b7-b68f-8b540e4fa6b7
Expand Down Expand Up @@ -3116,9 +3123,12 @@ datasets:
- default
parameters:
data_collection_database_name:
type: athena
query: SELECT DISTINCT table_schema FROM information_schema.columns WHERE table_name = 'aws_feeds_youtube'
default: optimization_data
description: "Enter the name of the data collection database"
global: True

aws_feeds_youtube:
data:
DataSetId: 5258b9ee-48d0-48f3-9ec4-baa4acd1d006
Expand Down Expand Up @@ -3236,6 +3246,8 @@ views:
"${data_collection_database_name}"."aws_feeds_youtube"
parameters:
data_collection_database_name:
type: athena
query: SELECT DISTINCT table_schema FROM information_schema.columns WHERE table_name = 'aws_feeds_youtube'
default: optimization_data
description: "Enter the name of the data collection database"
global: True
2 changes: 1 addition & 1 deletion dashboards/aws-marketplace/aws-marketplace-spg.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,6 @@ views:
, "sum"((CASE WHEN ("line_item_line_item_type" = 'SavingsPlanCoveredUsage') THEN "line_item_usage_amount" WHEN ("line_item_line_item_type" = 'DiscountedUsage') THEN "line_item_usage_amount" WHEN ("line_item_line_item_type" = 'Usage') THEN "line_item_usage_amount" ELSE 0 END)) "usage_quantity"
, "sum"("line_item_unblended_cost") "unblended_cost"
FROM
${cur_table_name}
"${cur_table_name}"
WHERE ((("bill_billing_period_start_date" >= ("date_trunc"('month', current_timestamp) - INTERVAL '36' MONTH)) AND (CAST("concat"("year", '-', "month", '-01') AS date) >= ("date_trunc"('month', current_date) - INTERVAL '36' MONTH))) AND ("bill_billing_entity" = 'AWS Marketplace'))
GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21
Loading

0 comments on commit dc99f98

Please sign in to comment.