diff --git a/app-loadtest/app-loadtest.go b/app-loadtest/app-loadtest.go index 3ff23d3..d3fae22 100644 --- a/app-loadtest/app-loadtest.go +++ b/app-loadtest/app-loadtest.go @@ -18,11 +18,9 @@ import ( ) type Task struct { - ID int - Num int + ID int } - var ( totalRequestsProcessed = prometheus.NewCounter(prometheus.CounterOpts{ Name: "app_loadtest_requests_total", @@ -96,7 +94,7 @@ func getSSMParam(parameterName string) (string, error){ } var mutex sync.Mutex -func main() { +func main(){ taskQueue := make(chan Task, 1000) var wg sync.WaitGroup appLoadtestReplicasCount, err := getSSMParam("/khanh-thesis/app_loadtest_replicas") @@ -137,44 +135,54 @@ func main() { requestURL := fmt.Sprintf("http://app-simulate.app-simulate.svc.cluster.local:5000/bytes?%s", queryParams.Encode()) // requestURL := fmt.Sprintf("http://localhost:5000/bytes?%s", queryParams.Encode()) // requestURL := fmt.Sprintf("http://google.com") - duration := 20*time.Second - interval := duration/time.Duration(numRequests) fmt.Println("DATA INPUT: ") fmt.Println("Bytes response each request: ", bytesResponseEachRequest) fmt.Println("Number of request per minutes: ", numRequests) - fmt.Println("Goroutine interval: ", interval) fmt.Println("START SENDING REQUEST: ") + + currentTime := time.Now() + nextMinute := currentTime.Truncate(time.Minute).Add(time.Minute) + fmt.Println("currentTime: ", currentTime) + fmt.Println("nextMinute :", nextMinute) + duration := nextMinute.Sub(currentTime) + fmt.Println("duration :", duration) + nanoseconds := duration.Nanoseconds() + time.Sleep(time.Duration(nanoseconds) * time.Nanosecond) + // Create pool for goroutines - poolSize := 100 + poolSize := 1000 for i := 0; i < poolSize; i++ { go worker(taskQueue, i, requestURL, &wg) } for { - startTime := time.Now() + startTimeRequest := time.Now() // Generate tasks and send them to taskQueue numRequestsEachReplica := numRequests/numReplicas + fmt.Println("START:===============================") fmt.Println("numRequests each replica: ", numRequestsEachReplica) + time.Sleep(10000 * time.Millisecond) for i := 0; i < numRequestsEachReplica; i ++ { - task := Task{ID: i, Num: i + 1} + task := Task{ID: i} taskQueue <- task wg.Add(1) } - - // Close the taskQueue to signal that no more tasks will be added + currentTime := time.Now() + nextMinute := currentTime.Truncate(time.Minute).Add(time.Minute) + fmt.Println("currentTime: ", currentTime) + fmt.Println("nextMinute :", nextMinute ) + duration := nextMinute.Sub(currentTime) + fmt.Println("duration :", duration) + nanoseconds := duration.Nanoseconds() + time.Sleep(time.Duration(nanoseconds) * time.Nanosecond) + fmt.Println("nanoseconds :", nanoseconds) + fmt.Println("END:===============================") // close(taskQueue) wg.Wait() - endTime := time.Now() - loadtestDuration := endTime.Sub(startTime) + endTimeRequest := time.Now() + loadtestDuration := endTimeRequest.Sub(startTimeRequest) loadtestSeconds := loadtestDuration.Seconds() - timeSleep := 0.0 - if loadtestSeconds < 60.0 { - timeSleep = 60.0 - loadtestSeconds - } appLoadtestResponseDurationAll.Set(loadtestSeconds) fmt.Println("loadtestSeconds: ", loadtestSeconds) - fmt.Println("timeSleep: ",timeSleep) - time.Sleep(time.Duration(int(timeSleep)) * time.Second) - fmt.Println("=============================") } } @@ -201,6 +209,7 @@ func processTask(task Task, pool int, requestURL string) { responseSize.Observe(float64(numBytes)) successRequestsCount.Inc() } + // time.Sleep(100 * time.Millisecond) endTimeEachRequest := time.Now() loadtestDurationEachRequest := endTimeEachRequest.Sub(startTimeEachRequest) loadtestSecondsEachRequest := loadtestDurationEachRequest.Seconds() diff --git a/app-loadtest/app-test.go b/app-loadtest/app-test.go new file mode 100644 index 0000000..64689aa --- /dev/null +++ b/app-loadtest/app-test.go @@ -0,0 +1,223 @@ +package main + +import ( + "fmt" + "log" + "strconv" + "io/ioutil" + "net/http" + "net/url" + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ssm" +) + +type Task struct { + ID int +} + +var ( + totalRequestsProcessed = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "app_loadtest_requests_total", + Help: "Total number of app load test requests made by the app", + }) + + successRequestsCount = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "app_loadtest_requests_successful_total", + Help: "Total number of app load test requests successful made by the app", + }) + + errorRequestsCount = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "app_loadtest_requests_failed_total", + Help: "Total number of app load test requests failed made by the app", + }) + + responseSize = prometheus.NewHistogram(prometheus.HistogramOpts{ + Name: "app_loadtest_response_size_bytes", + Help: "Size of HTTP app load test responses", + Buckets: []float64{100, 200, 300, 400, 500, 1000, 2000}, + }) + + responseDurationEachRequest = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "app_loadtest_response_duration_each_request", + Help: "Duration of HTTP app load test response each request", + }) + + appLoadtestResponseDurationAll = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "app_loadtest_response_duration_all", + Help: "Duration of HTTP app load test response all requests received", + }) +) + +var ( + sess *session.Session + ssmClient *ssm.SSM +) + +func init() { + region := "ap-southeast-1" + // Initialize the AWS session and SSM client + sess, err := session.NewSession(&aws.Config{ + Region: aws.String(region), + }) + if err != nil { + log.Fatal(err) + } + ssmClient = ssm.New(sess) + // Add go runtime metrics and process collectors. + prometheus.MustRegister( + totalRequestsProcessed, + successRequestsCount, + errorRequestsCount, + responseSize, + responseDurationEachRequest, + appLoadtestResponseDurationAll, + ) +} + +func getSSMParam(parameterName string) (string, error){ + // Create an input object for the GetParameter API + input := &ssm.GetParameterInput{ + Name: aws.String(parameterName), + WithDecryption: aws.Bool(true), + } + result, err := ssmClient.GetParameter(input) + if err != nil { + return "", fmt.Errorf("failed to retrieve parameter: %v", err) + } + return *result.Parameter.Value, nil +} + +var mutex sync.Mutex +func main(){ + taskQueue := make(chan Task, 1000) + var wg sync.WaitGroup + appLoadtestReplicasCount, err := getSSMParam("/khanh-thesis/app_loadtest_replicas") + if err != nil { + fmt.Println("Error retrieving replicas number param:", err) + return + } + appLoadtestRequestParamName, err := getSSMParam("/khanh-thesis/app_loadtest_request") + if err != nil { + fmt.Println("Error retrieving request number param:", err) + return + } + appLoadtestBytesParamName, err := getSSMParam("/khanh-thesis/app_loadtest_bytes") + if err != nil { + fmt.Println("Error retrieving bytes:", err) + return + } + numReplicas, err := strconv.Atoi(appLoadtestReplicasCount) + if err != nil { + log.Fatal(err) + } + numBytes, err := strconv.Atoi(appLoadtestBytesParamName) + if err != nil { + log.Fatal(err) + } + // Start the HTTP server to expose metrics + go func() { + http.Handle("/metrics", promhttp.Handler()) + log.Fatal(http.ListenAndServe(":8000", nil)) + }() + numRequests, err := strconv.Atoi(appLoadtestRequestParamName) + if err != nil { + log.Fatal(err) + } + bytesResponseEachRequest := strconv.Itoa(numBytes/numRequests) + queryParams := url.Values{} + queryParams.Set("num_bytes", bytesResponseEachRequest) + // requestURL := fmt.Sprintf("http://app-simulate.app-simulate.svc.cluster.local:5000/bytes?%s", queryParams.Encode()) + // requestURL := fmt.Sprintf("http://localhost:5000/bytes?%s", queryParams.Encode()) + requestURL := fmt.Sprintf("http://google.com") + fmt.Println("DATA INPUT: ") + fmt.Println("Bytes response each request: ", bytesResponseEachRequest) + fmt.Println("Number of request per minutes: ", numRequests) + fmt.Println("START SENDING REQUEST: ") + + currentTime := time.Now() + nextMinute := currentTime.Truncate(time.Minute).Add(time.Minute) + fmt.Println("currentTime: ", currentTime) + fmt.Println("nextMinute :", nextMinute) + duration := nextMinute.Sub(currentTime) + fmt.Println("duration :", duration) + nanoseconds := duration.Nanoseconds() + time.Sleep(time.Duration(nanoseconds) * time.Nanosecond) + + // Create pool for goroutines + poolSize := 100 + for i := 0; i < poolSize; i++ { + go worker(taskQueue, i, requestURL, &wg) + } + for { + startTimeRequest := time.Now() + // Generate tasks and send them to taskQueue + numRequestsEachReplica := numRequests/numReplicas + fmt.Println("numRequests each replica: ", numRequestsEachReplica) + fmt.Println("START:===============================") + for i := 0; i < numRequestsEachReplica; i ++ { + task := Task{ID: i} + taskQueue <- task + wg.Add(1) + } + currentTime := time.Now() + nextMinute := currentTime.Truncate(time.Minute).Add(time.Minute) + fmt.Println("currentTime: ", currentTime) + fmt.Println("nextMinute :", nextMinute ) + duration := nextMinute.Sub(currentTime) + fmt.Println("duration :", duration) + nanoseconds := duration.Nanoseconds() + time.Sleep(time.Duration(nanoseconds) * time.Nanosecond) + fmt.Println("nanoseconds :", nanoseconds) + fmt.Println("END:===============================") + // close(taskQueue) + wg.Wait() + endTimeRequest := time.Now() + loadtestDuration := endTimeRequest.Sub(startTimeRequest) + loadtestSeconds := loadtestDuration.Seconds() + appLoadtestResponseDurationAll.Set(loadtestSeconds) + fmt.Println("loadtestSeconds: ", loadtestSeconds) + } +} + +func worker(taskQueue chan Task, pool int, requestURL string, wg *sync.WaitGroup) { + for task := range taskQueue { + processTask(task, pool, requestURL) + wg.Done() + } +} + +func processTask(task Task, pool int, requestURL string) { + startTimeEachRequest := time.Now() + rs, err := http.Get(requestURL) + if err != nil { + errorRequestsCount.Inc() + } else { + defer rs.Body.Close() + // Read the response body to get the bytes + responseBody, err := ioutil.ReadAll(rs.Body) + if err != nil { + errorRequestsCount.Inc() + } + numBytes := len(responseBody) + responseSize.Observe(float64(numBytes)) + successRequestsCount.Inc() + } + endTimeEachRequest := time.Now() + loadtestDurationEachRequest := endTimeEachRequest.Sub(startTimeEachRequest) + loadtestSecondsEachRequest := loadtestDurationEachRequest.Seconds() + // if loadtestSecondsEachRequest < 0.002 { + // fmt.Println("loadtestSecondsEachRequest ",loadtestSecondsEachRequest) + // } + responseDurationEachRequest.Set(loadtestSecondsEachRequest) + + // Increment the requests counter + mutex.Lock() + totalRequestsProcessed.Inc() + mutex.Unlock() +} diff --git a/app-loadtest/deploy/02-deployment.yaml b/app-loadtest/deploy/02-deployment.yaml index c5e6f23..67200f3 100644 --- a/app-loadtest/deploy/02-deployment.yaml +++ b/app-loadtest/deploy/02-deployment.yaml @@ -33,9 +33,16 @@ spec: values: - app-loadtest topologyKey: "kubernetes.io/hostname" + tolerations: + - key: "node-group" + operator: "Equal" + value: "request" + effect: "NoExecute" + nodeSelector: + eks.amazonaws.com/nodegroup: app-request containers: - name: app-loadtest - image: 832438989008.dkr.ecr.ap-southeast-1.amazonaws.com/app-loadtest@sha256:7ae3c0d1436c21fabbea6af8a2615ba1cd80fcbfc724ae2cf72bc8bc41fcf6a5 + image: 832438989008.dkr.ecr.ap-southeast-1.amazonaws.com/app-loadtest@sha256:05f059352185538396bd6f4056dca78070ca17c646d5069d04e9cb9e45098918 ports: - name: prometheus containerPort: 8000 diff --git a/app-simulate/deploy/02-deployment.yaml b/app-simulate/deploy/02-deployment.yaml index 849bcbf..fc3cc1d 100644 --- a/app-simulate/deploy/02-deployment.yaml +++ b/app-simulate/deploy/02-deployment.yaml @@ -10,7 +10,7 @@ metadata: prometheus.io/port: "15020" prometheus.io/scrape: "true" spec: - replicas: 4 + replicas: 1 selector: matchLabels: app: app-simulate @@ -25,14 +25,31 @@ spec: spec: affinity: podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - app-simulate - topologyKey: "kubernetes.io/hostname" + # requiredDuringSchedulingIgnoredDuringExecution: + # - labelSelector: + # matchExpressions: + # - key: app + # operator: In + # values: + # - app-simulate + # topologyKey: "kubernetes.io/hostname" + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - app-simulate + topologyKey: "kubernetes.io/hostname" + tolerations: + - key: "node-group" + operator: "Equal" + value: "app" + effect: "NoExecute" + nodeSelector: + eks.amazonaws.com/nodegroup: app-app containers: - name: app-simulate image: 832438989008.dkr.ecr.ap-southeast-1.amazonaws.com/app-simulate@sha256:593a9d2443ac0b2bd8111202baab0c66b6acef0116d167ae1b6777173a5e27bd diff --git a/predict-controller/deploy/02-deployment.yaml b/predict-controller/deploy/02-deployment.yaml index b61b133..d82ff1c 100644 --- a/predict-controller/deploy/02-deployment.yaml +++ b/predict-controller/deploy/02-deployment.yaml @@ -25,7 +25,7 @@ spec: spec: containers: - name: predict-controller - image: 832438989008.dkr.ecr.ap-southeast-1.amazonaws.com/predict-controller@sha256:cbd23f87b7e99fdcf216b171f67fa2f0e25ce3361bef8efb1b3a7dab99fbc853 + image: 832438989008.dkr.ecr.ap-southeast-1.amazonaws.com/predict-controller@sha256:6d39815fab0271c624f809ff1654921b3120d264c544b834bccbb9c4d106ea99 ports: - containerPort: 8000 livenessProbe: diff --git a/predict-controller/metric-collect.py b/predict-controller/metric-collect.py new file mode 100644 index 0000000..bc84573 --- /dev/null +++ b/predict-controller/metric-collect.py @@ -0,0 +1,59 @@ +import pandas as pd +import numpy as np +import time +import boto3 +from datetime import datetime +from datetime import timedelta +from prometheus_api_client import PrometheusConnect +from prometheus_api_client.utils import parse_datetime +from prometheus_client import start_http_server, Gauge + +region = 'ap-southeast-1' + +# Create a new Gauge metric named "my_gauge" with a help message +predicted_prometheus = Gauge('predicted_prometheus', 'This is predicted_prometheus metric') +start_http_server(8000) + +# Connect to Prometheus server +prom = PrometheusConnect(url="http://prometheus.khanh-thesis.online", disable_ssl=True) + + +def get_metrics(start_time, end_time, step): + metric_data_request_count = prom.custom_query_range( + query='sum(rate(istio_requests_total{source_app="app-loadtest", destination_app="app-simulate", response_code="200", connection_security_policy="unknown", job="envoy-stats"}[1m])) * 50', + start_time=start_time, + end_time=end_time, + step=step, + ) + metric_data_sum_bytes = prom.custom_query_range( + query='sum(rate(istio_response_bytes_sum{source_app="app-loadtest", destination_app="app-simulate", response_code="200", connection_security_policy="unknown", job="envoy-stats"}[1m]) * 50)', + start_time=start_time, + end_time=end_time, + step=step, + ) + print(metric_data_request_count) + print(metric_data_sum_bytes) + if len(metric_data_request_count) == 0: + return pd.DataFrame() + if len(metric_data_request_count[0].get("values")) != 10: + return pd.DataFrame() + + df = pd.DataFrame(metric_data_request_count[0].get("values"), columns=['timestamp', 'request_count']) + sum_bytes = [ i[1] for i in metric_data_sum_bytes[0].get("values") ] + df["sum_bytes"] = sum_bytes + df['request_count'] = df['request_count'].astype(float) + df['sum_bytes'] = df['sum_bytes'].astype(float) + df['request_count'] = df['request_count'].astype(int) + df['sum_bytes'] = df['sum_bytes'].astype(int) + return df + + +format_str = "%Y-%m-%d %H:%M:%S" +# start_time = parse_datetime("9minutes") +start_time = strftime("%Y-%m-%d %H:%M:%S", timestamp(time()) - 9m) +end_time = strftime("%Y-%m-%d %H:%M:%S", timestamp(time()) - mod(timestamp(time()), 60)) +# end_time = parse_datetime("now") +print(end_time) +step = "1m" +rs = get_metrics(start_time, end_time, step) +print(rs) diff --git a/predict-controller/predict-controller.py b/predict-controller/predict-controller.py index d216850..f63d30f 100644 --- a/predict-controller/predict-controller.py +++ b/predict-controller/predict-controller.py @@ -56,6 +56,7 @@ def get_metrics(start_time, end_time, step): + print("start_time: ", type(start_time)) metric_data_request_count = prom.custom_query_range( query='sum(rate(istio_requests_total{source_app="request-simulate", destination_app="app-simulate", response_code="200", connection_security_policy="unknown", job="envoy-stats"}[1m])) * 50', start_time=start_time, @@ -68,7 +69,6 @@ def get_metrics(start_time, end_time, step): end_time=end_time, step=step, ) - if len(metric_data_request_count) == 0: return pd.DataFrame() if len(metric_data_request_count[0].get("values")) != 10: @@ -99,22 +99,34 @@ def predict_values(time_series, scaler_eventcount, scaler): predicted_value = scaler_eventcount.inverse_transform(predicted_value) return predicted_value - +# calib time +calib_current_time = datetime.now() +calib_next_minute = calib_current_time.replace(second=0, microsecond=0) + timedelta(minutes=1) +calib_delta_time = (calib_next_minute - calib_current_time).total_seconds() + 1 +print("delta_time :", int(calib_delta_time)) +time.sleep(calib_delta_time) while True: - start_time = parse_datetime("9minutes") - end_time = parse_datetime("now") + # predict + now = datetime.now() + start_time = (now - timedelta(minutes=9)).strftime("%Y-%m-%d %H:%M:00") + start_time = datetime.strptime(start_time, "%Y-%m-%d %H:%M:%S") + end_time = now.strftime("%Y-%m-%d %H:%M:00") + end_time = datetime.strptime(end_time, "%Y-%m-%d %H:%M:%S") step = "1m" rs = get_metrics(start_time, end_time, step) if not rs.empty: - rs['timestamp'] = pd.to_datetime(rs['timestamp'], unit='s').apply(lambda x: x + timedelta(minutes=1) - timedelta(seconds=x.second)) - rs['timestamp'] = rs['timestamp'] - time_offset + rs['timestamp'] = pd.to_datetime(rs['timestamp'], unit='s') + # minus 1 minutes because prometheus read data the data is in the past + rs['timestamp'] = rs['timestamp'] - time_offset - timedelta(minutes=1) merged_df = pd.merge(wc_dataset, rs, left_on='event_time', right_on='timestamp', how='right') merged_df['num_match_event'] = merged_df['num_match_event'].fillna(0) # fill missing values with 0 merged_df = merged_df.reindex(columns=['timestamp', 'request_count', 'sum_bytes', 'num_match_event']) merged_df = merged_df.drop('timestamp', axis=1) - predicted_value = predict_values(merged_df, scaler_eventcount, scaler) print(int(predicted_value[0][0])) predicted_prometheus.set(int(predicted_value[0][0])) - - time.sleep(60) + predict_end_time = datetime.now() + predict_next_minute = predict_end_time.replace(second=0, microsecond=0) + timedelta(minutes=1) + predict_delta_time= (predict_next_minute - predict_end_time).total_seconds() + 1 + print("delta_time :", int(predict_delta_time)) + time.sleep(predict_delta_time) diff --git a/request-simulate/Dockerfile b/request-simulate/Dockerfile index 77e69be..e8ea3e4 100644 --- a/request-simulate/Dockerfile +++ b/request-simulate/Dockerfile @@ -1,17 +1,27 @@ -FROM python:3.9-slim-buster +# Use the official Golang base image +FROM golang:1.20-alpine +# Set the working directory inside the container WORKDIR /app -COPY requirements.txt . - -RUN pip install --no-cache-dir -r requirements.txt - COPY healthy healthy +# Copy the Go module files +COPY go.mod go.sum ./ + COPY wc_dataset_processed.csv wc_dataset_processed.csv -COPY simulate-request.py app.py +# Download and cache Go dependencies +RUN go mod download + +# Copy the rest of the application source code +COPY request-simulate.go request-simulate.go + +# Build the Go application +RUN go build -o main +# Expose the port that the application listens on EXPOSE 8000 -CMD ["python", "app.py"] \ No newline at end of file +# Set the entry point for the container +CMD ["./main"] diff --git a/request-simulate/deploy/02-deployment.yaml b/request-simulate/deploy/02-deployment.yaml index a3d34f2..a0fd410 100644 --- a/request-simulate/deploy/02-deployment.yaml +++ b/request-simulate/deploy/02-deployment.yaml @@ -25,7 +25,7 @@ spec: spec: containers: - name: request-simulate - image: 832438989008.dkr.ecr.ap-southeast-1.amazonaws.com/request-simulate@sha256:92dfa3226fb3dc8d1fab36c925b4d4547fd411560264407435a5805d2cdb2104 + image: 832438989008.dkr.ecr.ap-southeast-1.amazonaws.com/request-simulate@sha256:4962452cca06c7023978afbee520ce58632c7114724d11de7f3e7d0f58e548ce ports: - name: prometheus containerPort: 8000 diff --git a/request-simulate/go.mod b/request-simulate/go.mod new file mode 100644 index 0000000..a30c38d --- /dev/null +++ b/request-simulate/go.mod @@ -0,0 +1,24 @@ +module request-simulate.go + +go 1.20 + +require ( + github.com/aws/aws-sdk-go v1.44.265 + github.com/go-gota/gota v0.12.0 + github.com/prometheus/client_golang v1.15.1 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/prometheus/client_model v0.3.0 // indirect + github.com/prometheus/common v0.42.0 // indirect + github.com/prometheus/procfs v0.9.0 // indirect + golang.org/x/net v0.7.0 // indirect + golang.org/x/sys v0.6.0 // indirect + gonum.org/v1/gonum v0.9.1 // indirect + google.golang.org/protobuf v1.30.0 // indirect +) diff --git a/request-simulate/go.sum b/request-simulate/go.sum new file mode 100644 index 0000000..694ac82 --- /dev/null +++ b/request-simulate/go.sum @@ -0,0 +1,134 @@ +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= +github.com/aws/aws-sdk-go v1.44.265 h1:rlBuD8OYjM5Vfcf7jDa264oVHqlPqY7y7o+JmrjNFUc= +github.com/aws/aws-sdk-go v1.44.265/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/go-fonts/dejavu v0.1.0/go.mod h1:4Wt4I4OU2Nq9asgDCteaAaWZOV24E+0/Pwo0gppep4g= +github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3T0ecnM9pNujks= +github.com/go-fonts/liberation v0.1.1/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= +github.com/go-fonts/stix v0.1.0/go.mod h1:w/c1f0ldAUlJmLBvlbkvVXLAD+tAMqobIIQpmnUIzUY= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gota/gota v0.12.0 h1:T5BDg1hTf5fZ/CO+T/N0E+DDqUhvoKBl+UVckgcAAQg= +github.com/go-gota/gota v0.12.0/go.mod h1:UT+NsWpZC/FhaOyWb9Hui0jXg0Iq8e/YugZHTbyW/34= +github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2CSIqUrmQPqA0gdRIlnLEY0gK5JGjh37zN5U= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY= +github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= +github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= +github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= +github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= +github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= +github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= +github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3 h1:n9HxLrNxWWtEb1cA950nuEEj3QnKbtsCJ6KjcgisNUs= +golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3/go.mod h1:NOZ3BPKG0ec/BKJQgnvsSFpcKLM5xXVWnvZS97DWHgE= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20190910094157-69e4b8554b2a/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200430140353-33d19683fad8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200618115811-c13761719519/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20201208152932-35266b937fa6/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20210216034530-4410531fe030/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210304124612-50617c2ba197/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= +gonum.org/v1/gonum v0.9.1 h1:HCWmqqNoELL0RAQeKBXWtkp04mGk8koafcB4He6+uhc= +gonum.org/v1/gonum v0.9.1/go.mod h1:TZumC3NeyVQskjXqmyWt4S3bINhy7B4eYwW69EbyX+0= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0 h1:OE9mWmgKkjJyEmDAAtGMPjXu+YNeGvK9VTSHY6+Qihc= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= +gonum.org/v1/plot v0.9.0/go.mod h1:3Pcqqmp6RHvJI72kgb8fThyUnav364FOsdDo2aGW5lY= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/request-simulate/request-simulate.go b/request-simulate/request-simulate.go new file mode 100644 index 0000000..160b0da --- /dev/null +++ b/request-simulate/request-simulate.go @@ -0,0 +1,219 @@ +package main + +import ( + "fmt" + "log" + "os" + "sync" + "time" + "io/ioutil" + "net/http" + "net/url" + "strconv" + + "github.com/go-gota/gota/dataframe" + // "github.com/go-gota/gota/series" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ssm" +) + +type Task struct { + ID int + RequestURL string +} + +var ( + totalRequestsProcessed = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "simulate_requests_total", + Help: "Total number of simulate requests made by the app", + }) + + successRequestsCount = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "simulate_requests_successful_total", + Help: "Total number of app load test requests successful made by the app", + }) + + errorRequestsCount = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "simulate_requests_failed_total", + Help: "Total number of app load test requests failed made by the app", + }) + + responseSize = prometheus.NewHistogram(prometheus.HistogramOpts{ + Name: "simulate_response_size_bytes", + Help: "Size of HTTP simulate responses", + Buckets: []float64{100, 200, 300, 400, 500, 1000, 2000}, + }) + + responseDurationEachRequest = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "simulate_response_duration_each_request", + Help: "Duration of HTTP app load test response each request", + }) + + appLoadtestResponseDurationAll = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "app_loadtest_response_duration_all", + Help: "Duration of HTTP app load test response all requests received", + }) +) + +var ( + sess *session.Session + ssmClient *ssm.SSM +) + +func init() { + region := "ap-southeast-1" + sess, err := session.NewSession(&aws.Config{ + Region: aws.String(region), + }) + if err != nil { + log.Fatal(err) + } + ssmClient = ssm.New(sess) + prometheus.MustRegister( + totalRequestsProcessed, + successRequestsCount, + errorRequestsCount, + responseSize, + responseDurationEachRequest, + appLoadtestResponseDurationAll, + ) +} + +func getSSMParam(parameterName string) (string, error){ + input := &ssm.GetParameterInput{ + Name: aws.String(parameterName), + WithDecryption: aws.Bool(true), + } + result, err := ssmClient.GetParameter(input) + if err != nil { + return "", fmt.Errorf("failed to retrieve parameter: %v", err) + } + return *result.Parameter.Value, nil +} + +func putSSMParam(parameterName string, currentTriggerTime string) (error){ + input := &ssm.PutParameterInput{ + Name: aws.String(parameterName), + Value: aws.String(currentTriggerTime), + Type: aws.String("String"), + Overwrite: aws.Bool(true), + } + _, err := ssmClient.PutParameter(input) + if err != nil { + return fmt.Errorf("failed to put parameter: %v", err) + } + return nil +} + +func main(){ + pastTriggerTime, _ := getSSMParam("/khanh-thesis/past_trigger_time") + pastTriggerTimeParse, err := time.Parse("2006-01-02 15:04:00", pastTriggerTime) + if err != nil { + log.Fatal(err) + } + fmt.Println("pastTriggerTime: ", pastTriggerTimeParse) + // temp data + numReplicas := 1 + taskQueue := make(chan Task, 1000) + var wg sync.WaitGroup + file, _ := os.Open("wc_dataset_processed.csv") + defer file.Close() + df := dataframe.ReadCSV(file) + queryParams := url.Values{} + eventTime := df.Col("event_time") + eventCount := df.Col("event_count") + sumBytes := df.Col("sum_bytes") + + // Start the HTTP server to expose metrics + go func() { + http.Handle("/metrics", promhttp.Handler()) + log.Fatal(http.ListenAndServe(":8000", nil)) + }() + // Calib time + currentTime := time.Now() + nextMinute := currentTime.Truncate(time.Minute).Add(time.Minute) + fmt.Println("currentTime: ", currentTime) + fmt.Println("nextMinute :", nextMinute) + duration := nextMinute.Sub(currentTime) + fmt.Println("duration :", duration) + nanoseconds := duration.Nanoseconds() + time.Sleep(time.Duration(nanoseconds) * time.Nanosecond) + // SSM Trigger time + triggerTime := time.Now() + currentTriggerTime := triggerTime.Format("2006-01-02 15:04:00") + fmt.Println("currentTriggerTime (ssm): ", currentTriggerTime) + putSSMParam("/khanh-thesis/current_trigger_time", currentTriggerTime) + poolSize := 1000 + for i := 0; i < poolSize; i++ { + go worker(taskQueue, i, &wg) + } + for i := 0; i < df.Nrow(); i++ { + eventTime, _ := time.Parse("2006-01-02 15:04:05", eventTime.Elem(i).String()) + if !eventTime.Before(pastTriggerTimeParse) { + startTimeRequest := time.Now() + fmt.Println("START:===============================") + fmt.Println("DATA INPUT: ") + numRequests, _ := strconv.Atoi(eventCount.Elem(i).String()) + numBytes, _ := strconv.Atoi(sumBytes.Elem(i).String()) + numRequestsEachReplica := numRequests/numReplicas + bytesResponseEachRequest := strconv.Itoa(numBytes/numRequests) + fmt.Println("Number of request per minutes: ", numRequests) + fmt.Println("Bytes response all requests: ", numBytes) + fmt.Println("Bytes response each request: ", bytesResponseEachRequest) + fmt.Println("START SENDING REQUEST: ") + queryParams.Set("num_bytes", bytesResponseEachRequest) + requestURL := fmt.Sprintf("http://app-simulate.app-simulate.svc.cluster.local:5000/bytes?%s", queryParams.Encode()) + time.Sleep(10000 * time.Millisecond) + for i := 0; i < numRequestsEachReplica; i ++ { + task := Task{ID: i, RequestURL: requestURL} + taskQueue <- task + wg.Add(1) + } + currentTime := time.Now() + nextMinute := currentTime.Truncate(time.Minute).Add(time.Minute) + duration := nextMinute.Sub(currentTime) + nanoseconds := duration.Nanoseconds() + fmt.Println("Need to sleep :", duration) + time.Sleep(time.Duration(nanoseconds) * time.Nanosecond) + fmt.Println("END:===============================") + wg.Wait() + endTimeRequest := time.Now() + loadtestDuration := endTimeRequest.Sub(startTimeRequest) + loadtestSeconds := loadtestDuration.Seconds() + appLoadtestResponseDurationAll.Set(loadtestSeconds) + fmt.Println("loadtestSeconds: ", loadtestSeconds) + } + } +} + +func worker(taskQueue chan Task, pool int, wg *sync.WaitGroup) { + for task := range taskQueue { + processTask(task, pool) + wg.Done() + } +} + +func processTask(task Task, pool int) { + startTimeEachRequest := time.Now() + rs, err := http.Get(task.RequestURL) + if err != nil { + errorRequestsCount.Inc() + } else { + defer rs.Body.Close() + // Read the response body to get the bytes + responseBody, _ := ioutil.ReadAll(rs.Body) + numBytes := len(responseBody) + // fmt.Println("numBytes ", numBytes) + responseSize.Observe(float64(numBytes)) + successRequestsCount.Inc() + } + endTimeEachRequest := time.Now() + loadtestDurationEachRequest := endTimeEachRequest.Sub(startTimeEachRequest) + loadtestSecondsEachRequest := loadtestDurationEachRequest.Seconds() + // fmt.Println("load test in seconds each request: ", loadtestSecondsEachRequest) + responseDurationEachRequest.Set(loadtestSecondsEachRequest) + totalRequestsProcessed.Inc() +} diff --git a/system/infra/modules/eks-nodegroup/main.tf b/system/infra/modules/eks-nodegroup/main.tf index 7b68744..b01f646 100644 --- a/system/infra/modules/eks-nodegroup/main.tf +++ b/system/infra/modules/eks-nodegroup/main.tf @@ -16,6 +16,15 @@ resource "aws_eks_node_group" "this" { instance_types = each.value.instance_types capacity_type = each.value.capacity_type version = var.kubernetes_version + dynamic "taint" { + for_each = each.value.kubernetes_taints + + content { + key = taint.value["key"] + value = taint.value["value"] + effect = taint.value["effect"] + } + } dynamic "launch_template" { for_each = [each.key] content { diff --git a/system/infra/modules/eks-nodegroup/variables.tf b/system/infra/modules/eks-nodegroup/variables.tf index c8060d0..19f9b46 100644 --- a/system/infra/modules/eks-nodegroup/variables.tf +++ b/system/infra/modules/eks-nodegroup/variables.tf @@ -23,12 +23,13 @@ variable "node_groups" { type = any default = { on_demand = { - disk_size = 20 - instance_types = ["t2.medium", "t3.medium", "t3a.medium"] - capacity_type = "SPOT" - desired_capacity = 1 - max_size = 3 - min_size = 1 + disk_size = 20 + instance_types = ["t2.medium", "t3.medium", "t3a.medium"] + capacity_type = "SPOT" + desired_capacity = 1 + max_size = 3 + min_size = 1 + kubernetes_taints = [] } } } diff --git a/system/infra/modules/skeleton-infra/variables.tf b/system/infra/modules/skeleton-infra/variables.tf index 49270e7..f431793 100644 --- a/system/infra/modules/skeleton-infra/variables.tf +++ b/system/infra/modules/skeleton-infra/variables.tf @@ -42,22 +42,43 @@ variable "eks_cluster_name" { variable "node_groups" { type = any default = { - spot = { + app = { disk_size = 20 instance_types = ["m5.2xlarge", "m5a.2xlarge", "m6a.2xlarge"] capacity_type = "SPOT" desired_capacity = 1 max_size = 3 min_size = 1 + kubernetes_taints = [{ + key = "node-group" + value = "app" + effect = "NO_EXECUTE" + }] + } + request = { + disk_size = 20 + instance_types = ["m5.2xlarge", "m5a.2xlarge", "m6a.2xlarge"] + capacity_type = "SPOT" + desired_capacity = 1 + max_size = 3 + min_size = 1 + kubernetes_taints = [ + { + key = "node-group" + value = "request" + effect = "NO_EXECUTE" + } + ] + } + system = { + disk_size = 20 + instance_types = ["t3.medium", "t3a.medium"] + capacity_type = "SPOT" + desired_capacity = 1 + max_size = 3 + min_size = 1 + kubernetes_taints = [] } - # system = { - # disk_size = 20 - # instance_types = ["m5.2xlarge", "m5a.2xlarge"] - # capacity_type = "SPOT" - # desired_capacity = 1 - # max_size = 3 - # min_size = 1 - # } } } diff --git a/system/ops/istio-monitor/prometheus.yaml b/system/ops/istio-monitor/prometheus.yaml index 751423f..0195dcd 100644 --- a/system/ops/istio-monitor/prometheus.yaml +++ b/system/ops/istio-monitor/prometheus.yaml @@ -51,7 +51,7 @@ data: action: keep regex: '.*-envoy-prom' - job_name: request-simulate - metrics_path: /stats/prometheus + metrics_path: /metrics kubernetes_sd_configs: - role: endpoints relabel_configs: