From 799be0de1edb49a8f3f65b2167a7410be820441e Mon Sep 17 00:00:00 2001 From: Praven John Date: Fri, 8 Dec 2023 16:08:35 +0530 Subject: [PATCH 1/3] fixing the issue with RDS Storage pricing --- collector/aws/resources/rds.go | 12 +++++- configuration/collector.yaml | 78 ++++++++++++---------------------- docker-compose.yaml | 8 ++-- go.mod | 2 +- go.sum | 10 +---- 5 files changed, 44 insertions(+), 66 deletions(-) diff --git a/collector/aws/resources/rds.go b/collector/aws/resources/rds.go index ba2d5d67..2b945b55 100644 --- a/collector/aws/resources/rds.go +++ b/collector/aws/resources/rds.go @@ -48,6 +48,7 @@ type DetectedAWSRDS struct { // RDSVolumeType will hold the available volume types for RDS types var rdsStorageType = map[string]string{ "gp2": "General Purpose", + "gp3": "General Purpose", "standard": "Magnetic", "io1": "Provisioned IOPS", "aurora": "General Purpose-Aurora", @@ -234,7 +235,7 @@ func (r *RDSManager) getHourlyStoragePrice(instance *rds.DBInstance, pricingRegi storagePricingFilters = r.getPricingAuroraStorageFilterInput(rdsStorageType, pricingRegionPrefix) default: deploymentOption := r.getPricingDeploymentOption(instance) - storagePricingFilters = r.getPricingRDSStorageFilterInput(rdsStorageType, deploymentOption) + storagePricingFilters = r.getPricingRDSStorageFilterInput(instance, rdsStorageType, deploymentOption) } log.WithField("storage_filters", storagePricingFilters).Debug("pricing storage filters") @@ -308,7 +309,9 @@ func (r *RDSManager) getPricingDeploymentOption(instance *rds.DBInstance) string } // getPricingRDSStorageFilterInput will set the right filters for RDS Storage pricing -func (r *RDSManager) getPricingRDSStorageFilterInput(rdsStorageType string, deploymentOption string) pricing.GetProductsInput { +func (r *RDSManager) getPricingRDSStorageFilterInput(instance *rds.DBInstance, rdsStorageType string, deploymentOption string) pricing.GetProductsInput { + + databaseEngine := r.getPricingDatabaseEngine(instance) return pricing.GetProductsInput{ ServiceCode: &r.servicePricingCode, @@ -333,6 +336,11 @@ func (r *RDSManager) getPricingRDSStorageFilterInput(rdsStorageType string, depl Field: awsClient.String("deploymentOption"), Value: awsClient.String(deploymentOption), }, + { + Type: awsClient.String("TERM_MATCH"), + Field: awsClient.String("databaseEngine"), + Value: &databaseEngine, + }, }, } } diff --git a/configuration/collector.yaml b/configuration/collector.yaml index 95557126..a9e3efc2 100644 --- a/configuration/collector.yaml +++ b/configuration/collector.yaml @@ -15,6 +15,15 @@ providers: # role: regions: - us-east-1 + - ap-south-1 + - af-south-1 + - ap-northeast-1 + - ap-southeast-1 + - eu-north-1 + - eu-central-1 + - eu-west-1 + - us-east-2 + - us-west-1 - us-west-2 metrics: rds: @@ -28,17 +37,16 @@ providers: constraint: operator: "==" value: 0 - documentDB: - - description: Connection count + - description: CPU Utilization enable: true metrics: - - name: DatabaseConnections - statistic: Sum - period: 24h + - name: CPUUtilization + statistic: Maximum + period: 24h start_time: 168h # 24h * 7d constraint: - operator: "==" - value: 0 + operator: "<" + value: 40 elasticache: - description: Connection count enable: true @@ -49,7 +57,17 @@ providers: start_time: 168h # 24h * 7d constraint: operator: "==" - value: 0 + value: 0 + - description: CPU Utilization + enable: true + metrics: + - name: CPUUtilization + statistic: Maximum + period: 24h + start_time: 168h # 24h * 7d + constraint: + operator: "<" + value: 40 elb: - description: Request count enable: true @@ -82,7 +100,7 @@ providers: start_time: 168h # 24h * 7d constraint: operator: "<" - value: 6 + value: 40 dynamodb: - description: Provisioned read capacity units enable: true @@ -121,48 +139,6 @@ providers: constraint: operator: "==" value: 0 - neptune: - - description: Requests gremlin/sparql - enable: true - metrics: - - name: GremlinRequestsPerSec - statistic: Sum - - name: SparqlRequestsPerSec - statistic: Sum - period: 24h - start_time: 168h # 24h * 7d - constraint: - formula: GremlinRequestsPerSec + SparqlRequestsPerSec - operator: "==" - value: 0 - kinesis: - - description: Total put records - enable: true - metrics: - - name: "PutRecords.Bytes" - statistic: Sum - - name: "PutRecord.Bytes" - statistic: Sum - period: 24h - start_time: 168h # 24h * 7d - constraint: - # The go module Knetic/govaluate has a built in escaping - # https://github.com/Knetic/govaluate#escaping-characters - # [PutRecord.Bytes] will escape the parameter name - formula: "[PutRecord.Bytes] + [PutRecords.Bytes]" - operator: "==" - value: 0 - redshift: - - description: Connection count - enable: true - metrics: - - name: DatabaseConnections - statistic: Sum - period: 24h - start_time: 168h # 24h * 7d - constraint: - operator: "==" - value: 0 elasticsearch: - description: "IndexRate + SearchRate" enable: true diff --git a/docker-compose.yaml b/docker-compose.yaml index d5301851..56d839da 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -6,10 +6,12 @@ services: ports: - "9200:9200" - "5601:5601" + environment: + SSL_MODE: false networks: backend: aliases: - - elasticsearch + - elasticsearch:kibana7 ui: build: context: . @@ -40,8 +42,6 @@ services: - ./configuration/api.yaml:/etc/finala/config.yaml ports: - "8081:8081" - networks: - - backend depends_on: - elasticsearch networks: @@ -57,7 +57,7 @@ services: - -c - /etc/finala/config.yaml environment: - - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} - OVERRIDE_API_ENDPOINT=http://api:8081 volumes: diff --git a/go.mod b/go.mod index 476ed36e..40f78cc8 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.13 require ( github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible - github.com/aws/aws-sdk-go v1.31.3 + github.com/aws/aws-sdk-go v1.33.0 github.com/dustin/go-humanize v1.0.0 github.com/gorilla/handlers v1.4.2 github.com/gorilla/mux v1.7.4 diff --git a/go.sum b/go.sum index 49880209..51e24076 100644 --- a/go.sum +++ b/go.sum @@ -7,8 +7,8 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/aws/aws-sdk-go v1.30.7/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= -github.com/aws/aws-sdk-go v1.31.3 h1:vJDjoM+VlM/ZEmGyaIhUXaYAtB9lra7Qhr58SSHHjPE= -github.com/aws/aws-sdk-go v1.31.3/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/aws/aws-sdk-go v1.33.0 h1:Bq5Y6VTLbfnJp1IV8EL/qUU5qO1DYHda/zis/sqevkY= +github.com/aws/aws-sdk-go v1.33.0/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -71,7 +71,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -125,7 +124,6 @@ github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= @@ -141,7 +139,6 @@ github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGr github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= @@ -172,9 +169,7 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd h1:r7DufRZuZbWB7j439YfAzP8RPDa9unLkpwQKUYbIMPI= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f h1:68K/z8GLUxV76xGSqwTWw2gyk/jwn79LUL43rES2g8o= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -198,7 +193,6 @@ google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiq google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From d419ffd459c1756e866b3023f93917167f87af52 Mon Sep 17 00:00:00 2001 From: Praven John Date: Fri, 8 Dec 2023 16:34:32 +0530 Subject: [PATCH 2/3] Adding the default collector.yaml back in --- configuration/collector.yaml | 42 ++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/configuration/collector.yaml b/configuration/collector.yaml index a9e3efc2..a22b050d 100644 --- a/configuration/collector.yaml +++ b/configuration/collector.yaml @@ -139,6 +139,48 @@ providers: constraint: operator: "==" value: 0 + neptune: + - description: Requests gremlin/sparql + enable: true + metrics: + - name: GremlinRequestsPerSec + statistic: Sum + - name: SparqlRequestsPerSec + statistic: Sum + period: 24h + start_time: 168h # 24h * 7d + constraint: + formula: GremlinRequestsPerSec + SparqlRequestsPerSec + operator: "==" + value: 0 + kinesis: + - description: Total put records + enable: true + metrics: + - name: "PutRecords.Bytes" + statistic: Sum + - name: "PutRecord.Bytes" + statistic: Sum + period: 24h + start_time: 168h # 24h * 7d + constraint: + # The go module Knetic/govaluate has a built in escaping + # https://github.com/Knetic/govaluate#escaping-characters + # [PutRecord.Bytes] will escape the parameter name + formula: "[PutRecord.Bytes] + [PutRecords.Bytes]" + operator: "==" + value: 0 + redshift: + - description: Connection count + enable: true + metrics: + - name: DatabaseConnections + statistic: Sum + period: 24h + start_time: 168h # 24h * 7d + constraint: + operator: "==" + value: 0 elasticsearch: - description: "IndexRate + SearchRate" enable: true From 6d9f8ae08a6fa3e2534861593001e4272293fe3a Mon Sep 17 00:00:00 2001 From: Praven John Date: Fri, 8 Dec 2023 17:39:04 +0530 Subject: [PATCH 3/3] fixing the issue with RDS storage pricing --- collector/aws/resources/rds.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/collector/aws/resources/rds.go b/collector/aws/resources/rds.go index ba2d5d67..b0ceb346 100644 --- a/collector/aws/resources/rds.go +++ b/collector/aws/resources/rds.go @@ -234,7 +234,7 @@ func (r *RDSManager) getHourlyStoragePrice(instance *rds.DBInstance, pricingRegi storagePricingFilters = r.getPricingAuroraStorageFilterInput(rdsStorageType, pricingRegionPrefix) default: deploymentOption := r.getPricingDeploymentOption(instance) - storagePricingFilters = r.getPricingRDSStorageFilterInput(rdsStorageType, deploymentOption) + storagePricingFilters = r.getPricingRDSStorageFilterInput(instance, rdsStorageType, deploymentOption) } log.WithField("storage_filters", storagePricingFilters).Debug("pricing storage filters") @@ -308,8 +308,9 @@ func (r *RDSManager) getPricingDeploymentOption(instance *rds.DBInstance) string } // getPricingRDSStorageFilterInput will set the right filters for RDS Storage pricing -func (r *RDSManager) getPricingRDSStorageFilterInput(rdsStorageType string, deploymentOption string) pricing.GetProductsInput { +func (r *RDSManager) getPricingRDSStorageFilterInput(instance *rds.DBInstance, rdsStorageType string, deploymentOption string) pricing.GetProductsInput { + databaseEngine := r.getPricingDatabaseEngine(instance) return pricing.GetProductsInput{ ServiceCode: &r.servicePricingCode, Filters: []*pricing.Filter{ @@ -333,6 +334,11 @@ func (r *RDSManager) getPricingRDSStorageFilterInput(rdsStorageType string, depl Field: awsClient.String("deploymentOption"), Value: awsClient.String(deploymentOption), }, + { + Type: awsClient.String("TERM_MATCH"), + Field: awsClient.String("databaseEngine"), + Value: &databaseEngine, + }, }, } }