fixing the issue with RDS storage pricing

collector/aws/resources/rds.go

@@ -48,6 +48,7 @@ type DetectedAWSRDS struct {
 // RDSVolumeType will hold the available volume types for RDS types
 var rdsStorageType = map[string]string{
 	"gp2":      "General Purpose",
+	"gp3":      "General Purpose",
 	"standard": "Magnetic",
 	"io1":      "Provisioned IOPS",
 	"aurora":   "General Purpose-Aurora",
@@ -234,7 +235,7 @@ func (r *RDSManager) getHourlyStoragePrice(instance *rds.DBInstance, pricingRegi
 			storagePricingFilters = r.getPricingAuroraStorageFilterInput(rdsStorageType, pricingRegionPrefix)
 		default:
 			deploymentOption := r.getPricingDeploymentOption(instance)
-			storagePricingFilters = r.getPricingRDSStorageFilterInput(rdsStorageType, deploymentOption)
+			storagePricingFilters = r.getPricingRDSStorageFilterInput(instance, rdsStorageType, deploymentOption)
 		}
 
 		log.WithField("storage_filters", storagePricingFilters).Debug("pricing storage filters")
@@ -308,8 +309,10 @@ func (r *RDSManager) getPricingDeploymentOption(instance *rds.DBInstance) string
 }
 
 // getPricingRDSStorageFilterInput will set the right filters for RDS Storage pricing
-func (r *RDSManager) getPricingRDSStorageFilterInput(rdsStorageType string, deploymentOption string) pricing.GetProductsInput {
+func (r *RDSManager) getPricingRDSStorageFilterInput(instance *rds.DBInstance, rdsStorageType string, deploymentOption string) pricing.GetProductsInput {
+	databaseEngine := r.getPricingDatabaseEngine(instance)
 
+	databaseEngine := r.getPricingDatabaseEngine(instance)
 	return pricing.GetProductsInput{
 		ServiceCode: &r.servicePricingCode,
 		Filters: []*pricing.Filter{
@@ -333,6 +336,11 @@ func (r *RDSManager) getPricingRDSStorageFilterInput(rdsStorageType string, depl
 				Field: awsClient.String("deploymentOption"),
 				Value: awsClient.String(deploymentOption),
 			},
+			{
+				Type:  awsClient.String("TERM_MATCH"),
+				Field: awsClient.String("databaseEngine"),
+				Value: &databaseEngine,
+			},
 		},
 	}
 }

configuration/collector.yaml

@@ -15,6 +15,15 @@ providers:
         # role: 
         regions:
           - us-east-1
+          - ap-south-1
+          - af-south-1
+          - ap-northeast-1
+          - ap-southeast-1
+          - eu-north-1
+          - eu-central-1
+          - eu-west-1
+          - us-east-2
+          - us-west-1
           - us-west-2
     metrics:
       rds:
@@ -28,17 +37,16 @@ providers:
           constraint:
             operator: "=="
             value: 0
-      documentDB:
-        - description: Connection count
+        - description: CPU Utilization
           enable: true
           metrics:
-            - name: DatabaseConnections
-              statistic: Sum
-          period: 24h 
+            - name: CPUUtilization
+              statistic: Maximum
+          period: 24h
           start_time: 168h # 24h * 7d
           constraint:
-            operator: "=="
-            value: 0
+            operator: "<"
+            value: 40
       elasticache:
         - description: Connection count
           enable: true
@@ -49,7 +57,17 @@ providers:
           start_time: 168h # 24h * 7d
           constraint:
             operator: "=="
-            value: 0      
+            value: 0
+        - description: CPU Utilization
+          enable: true
+          metrics:
+            - name: CPUUtilization
+              statistic: Maximum
+          period: 24h
+          start_time: 168h # 24h * 7d
+          constraint:
+            operator: "<"
+            value: 40     
       elb:
         - description: Request count
           enable: true
@@ -82,7 +100,7 @@ providers:
           start_time: 168h # 24h * 7d
           constraint:
             operator: "<"
-            value: 6
+            value: 40
       dynamodb:
         - description: Provisioned read capacity units
           enable: true

docker-compose.yaml

@@ -6,10 +6,12 @@ services:
       ports:
         - "9200:9200"
         - "5601:5601"
+      environment:
+        SSL_MODE: false
       networks:
         backend:
           aliases:
-            - elasticsearch
+            - elasticsearch:kibana7
   ui:
       build:
         context: .
@@ -40,8 +42,6 @@ services:
           - ./configuration/api.yaml:/etc/finala/config.yaml
       ports:
         - "8081:8081"
-      networks:
-        - backend
       depends_on: 
         - elasticsearch
       networks:
@@ -57,7 +57,7 @@ services:
         - -c 
         - /etc/finala/config.yaml
       environment:
-        - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} 
+        - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
         - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
         - OVERRIDE_API_ENDPOINT=http://api:8081
       volumes:

go.mod

@@ -4,7 +4,7 @@ go 1.13
 
 require (
 	github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible
-	github.com/aws/aws-sdk-go v1.31.3
+	github.com/aws/aws-sdk-go v1.33.0
 	github.com/dustin/go-humanize v1.0.0
 	github.com/gorilla/handlers v1.4.2
 	github.com/gorilla/mux v1.7.4

go.sum

@@ -7,8 +7,8 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/aws/aws-sdk-go v1.30.7/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
-github.com/aws/aws-sdk-go v1.31.3 h1:vJDjoM+VlM/ZEmGyaIhUXaYAtB9lra7Qhr58SSHHjPE=
-github.com/aws/aws-sdk-go v1.31.3/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
+github.com/aws/aws-sdk-go v1.33.0 h1:Bq5Y6VTLbfnJp1IV8EL/qUU5qO1DYHda/zis/sqevkY=
+github.com/aws/aws-sdk-go v1.33.0/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
@@ -71,7 +71,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv
 github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
-github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
@@ -125,7 +124,6 @@ github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU
 github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8=
 github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
-github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
@@ -141,7 +139,6 @@ github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGr
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
-go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
@@ -172,9 +169,7 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd h1:r7DufRZuZbWB7j439YfAzP8RPDa9unLkpwQKUYbIMPI=
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191220142924-d4481acd189f h1:68K/z8GLUxV76xGSqwTWw2gyk/jwn79LUL43rES2g8o=
 golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -198,7 +193,6 @@ google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiq
 google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=