diff --git a/container/ccnp-example/Dockerfile b/container/ccnp-example/Dockerfile index ef34668..2e72e42 100644 --- a/container/ccnp-example/Dockerfile +++ b/container/ccnp-example/Dockerfile @@ -36,6 +36,7 @@ RUN chown $USER:$GROUP /run/ccnp COPY test ./ COPY sdk/python3/example/py_sdk_example.py ./ +COPY test/perf/py_perf.py ./ COPY --from=python-builder cc-trusted-api/common/python/dist/cctrusted_base*.whl ./ COPY --from=python-builder ccnp-sdk/dist/ccnp*.whl ./ diff --git a/deployment/kubernetes/manifests/ccnp-example-perf-deployment.yaml b/deployment/kubernetes/manifests/ccnp-example-perf-deployment.yaml new file mode 100644 index 0000000..5b9ea04 --- /dev/null +++ b/deployment/kubernetes/manifests/ccnp-example-perf-deployment.yaml @@ -0,0 +1,26 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ccnp-perf-example-PLACEHOLDER + namespace: perf +spec: + selector: + matchLabels: + app: ccnp-perf-example-PLACEHOLDER + template: + metadata: + labels: + app: ccnp-perf-example-PLACEHOLDER + annotations: + "ccnp.cc-api/require": "true" + spec: + containers: + - name: ccnp-perf-example-PLACEHOLDER + image: "docker.io/library/ccnp-example:latest" + imagePullPolicy: Always + resources: + limits: + memory: "128Mi" + cpu: "100m" + nodeSelector: + "feature.node.kubernetes.io/cpu-security.tdx.protected": "true" diff --git a/test/perf/README.md b/test/perf/README.md index 84d9a34..bfd139c 100644 --- a/test/perf/README.md +++ b/test/perf/README.md @@ -1,102 +1,49 @@ -# Performance Test +# CCNP Performance Test -We have these KPIs for performance test. - -| KPI​ | HIB/LIB​ | Unit​ | Comment​ | -| ------------------------------------------- | -------- | ----- | ----------------------------------------------------- | -| CCNP service get measurement throughput​ | HIB​ | ops​ | Service Throughput​ | -| CCNP service get measurement response time​ | LIB​ | ms​ | Service Response time​ | -| CCNP service get eventlog throughput​ | HIB​ | ops​ | Service Throughput​ | -| CCNP service get eventlog response time​ | LIB​ | ms​ | Service Response time​ | -| CCNP service get quote throughput​ | HIB​ | ops​ | Service Throughput​ | -| CCNP service get quote response time​ | LIB​ | ms​ | Service Response time​ | -| CCNP initialization time​ | LIB​ | s​ | CCNP device plugin, DaemonSet and service readiness.​ | - -*Note: we use the CCNP SDK to access the CCNP service because it's convenient to prepare the request data (e.g. container ID, etc.)​ +CCNP performance tests focus on the latency of calling CCNP SDK key APIs: `get_cc_eventlog`, `get_cc_report` and `get_cc_measurement`. +It will simulate requests from multiple pods in parallel and calculate average time of all the requests. Below are the steps for you to build and run the performance test. ## Prerequisites -To run the test, you need a K8S cluster with CCNP enabled (CCNP Device Plugin and CCNP Service deployed and ready). - -## Build - -```bash -# Make sure you are on the repo's top dir -cd +Please make sure you have CCNP deployed in a K8S cluster, and ccnp-example image has been built. +Please refer to [here](../../deployment/README.md) for image building and CCNP deployment. -# Run doker build -docker build --build-arg http_proxy=$http_proxy --build-arg https_proxy=$https_proxy --build-arg no_proxy=$no_proxy -t ccnp-perf:latest -f container/ccnp-perf/Dockerfile . +## Run Tests -# View build result -docker image ls | grep ccnp-perf +### Deploy pods for performance testing -# Save the docker image for later use -docker save ccnp-perf:latest > ccnp-perf_latest.tar +```bash +# Deploy ccnp-example pods +$ sudo ./deploy-perf.sh -r -g -n ``` -## Deploy +### Run Tests + +The script will run tests in parallel. The log will be saved in files with prefix `perf_output` under current directory. ```bash -# Load the docker image for K8S using containerd. -# You need to run this on the node where you want to deploy the ccnp-perf test -ctr -n=k8s.io image import ccnp-perf_latest.tar +# Test for get event log +$ sudo ./perf-para.sh -n -e -# Make sure you are on the repo's top dir -cd +# Test for get measurement +$ sudo ./perf-para.sh -n -m -# Deploy ccnp-perf test -kubectl apply -f deployment/kubernetes/manifests/ccnp-perf-deployment.yaml +# Test for get quote +$ sudo ./perf-para.sh -n -r ``` -## Test +Run below script to calculate average time of a request. ```bash -# Get the pod name of ccnp-perf -kubectl get pod | grep ccnp-perf - -# Run all perf test on the specified pod name got from above command -kubectl exec -ti -- python3 -m pytest --log-cli-level=INFO --verbose ccnp_perf.py +$ sudo ./average.sh -f perf_output_quote ``` -Sample test output looks like this: +### Clear + +Run below command to delete the pods for performance testing. ```bash -root@ccnp-perf-0:~/ccnp/confidential-cloud-native-primitives# kubectl exec -ti ccnp-perf-7f8798bf85-8s6zg -- python3 -m pytest --log-cli-level=INFO --verbose - ccnp_perf.py -==================================================================== test session starts ==================================================================== -platform linux -- Python 3.12.2, pytest-8.1.1, pluggy-1.4.0 -- /usr/local/bin/python3 -cachedir: .pytest_cache -rootdir: /run/ccnp -collected 7 items - -ccnp_perf.py::test_svc_get_cc_measurement_throughput ------------------------------------------------------------------------ live log call ----------------------------------------------------------------------- -INFO ccnp_perf:ccnp_perf.py:191 Perf test average throughput is: 70.75 ops (operations per second) -PASSED [ 14%] -ccnp_perf.py::test_svc_get_cc_measurement_response ------------------------------------------------------------------------ live log call ----------------------------------------------------------------------- -INFO ccnp_perf:ccnp_perf.py:213 Perf test average response time is: 25.89662575 ms (milliseconds) -PASSED [ 28%] -ccnp_perf.py::test_svc_get_cc_eventlog_throughput ------------------------------------------------------------------------ live log call ----------------------------------------------------------------------- -INFO ccnp_perf:ccnp_perf.py:191 Perf test average throughput is: 57.8 ops (operations per second) -PASSED [ 42%] -ccnp_perf.py::test_svc_get_cc_eventlog_response ------------------------------------------------------------------------ live log call ----------------------------------------------------------------------- -INFO ccnp_perf:ccnp_perf.py:213 Perf test average response time is: 76.130223 ms (milliseconds) -PASSED [ 57%] -ccnp_perf.py::test_svc_get_cc_report_throughput ------------------------------------------------------------------------ live log call ----------------------------------------------------------------------- -INFO ccnp_perf:ccnp_perf.py:191 Perf test average throughput is: 54.9 ops (operations per second) -PASSED [ 71%] -ccnp_perf.py::test_svc_get_cc_report_response ------------------------------------------------------------------------ live log call ----------------------------------------------------------------------- -INFO ccnp_perf:ccnp_perf.py:213 Perf test average response time is: 29.38618825 ms (milliseconds) -PASSED [ 85%] -ccnp_perf.py::test_ccnp_init PASSED [100%] - -=============================================================== 7 passed in 66.95s (0:01:06) ================================================================ -root@ccnp-perf-0:~/ccnp/confidential-cloud-native-primitives# -``` +$ sudo ./deploy-per.sh -n -d +``` \ No newline at end of file diff --git a/test/perf/average.sh b/test/perf/average.sh new file mode 100644 index 0000000..8b8a01c --- /dev/null +++ b/test/perf/average.sh @@ -0,0 +1,73 @@ +#!/bin/bash +# Input filename keyword and keyword for grep. Find the files in current dir, search the numbers needed, calculate sum and average + +grep_keyword="total command " + +usage() { + cat << EOM +usage: $(basename "$0") [OPTION]... + -f Input file name keyword + -k Keyword to be searched in the input files +EOM + exit 1 +} + +process_args() { + while getopts ":f:k:h" option; do + case "${option}" in + f) filename_keyword=${OPTARG};; + k) grep_keyword=${OPTARG};; + h) usage;; + *) echo "Invalid option: -${OPTARG}" >&2 + usage + ;; + esac + done +} + + +# Find all files containing the filename keyword +files=$(find . -type f -name "*${filename_keyword}*"| cut -d/ -f2) + +total_sum=0 +total_count=0 +total_aver=0 + +# Loop through the found files +for file in $files; do + sum=0 + count=0 + + tag=$(echo "$file" | cut -d- -f2) + tmpfile="tmp_output_"$tag + touch "$tmpfile" + + # Grep the keyword and extract the number. The number is the 2nd last word in the line. + grep -i "${grep_keyword}" "$file" | awk '{print $(NF-1)}' > "$tmpfile" + + while IFS= read -r line; do + # Add the number to sum + sum=$(echo "$sum + $line" | bc) + # Increment the count + count=$((count + 1)) + done < "$tmpfile" + + # Calculate the average + if [ $count -ne 0 ]; then + average=$(echo "scale=6; $sum / $count" | bc) + echo "Output file: $file" + echo "Number file: $tmpfile" + echo "Count: $count" + echo "Sum: $sum" + echo "Average: $average" + else + echo "The file is empty" + fi + total_sum=$(echo "$sum + $total_sum" | bc) + total_count=$(echo "$count + $total_count" | bc) +done + +echo "Total Sum: $total_sum" +echo "Total count: $total_count" +total_aver=$(echo "scale=6; $total_sum / $total_count" | bc) +echo "Total average: $total_aver" diff --git a/test/perf/ccnp-pytest.md b/test/perf/ccnp-pytest.md new file mode 100644 index 0000000..84d9a34 --- /dev/null +++ b/test/perf/ccnp-pytest.md @@ -0,0 +1,102 @@ +# Performance Test + +We have these KPIs for performance test. + +| KPI​ | HIB/LIB​ | Unit​ | Comment​ | +| ------------------------------------------- | -------- | ----- | ----------------------------------------------------- | +| CCNP service get measurement throughput​ | HIB​ | ops​ | Service Throughput​ | +| CCNP service get measurement response time​ | LIB​ | ms​ | Service Response time​ | +| CCNP service get eventlog throughput​ | HIB​ | ops​ | Service Throughput​ | +| CCNP service get eventlog response time​ | LIB​ | ms​ | Service Response time​ | +| CCNP service get quote throughput​ | HIB​ | ops​ | Service Throughput​ | +| CCNP service get quote response time​ | LIB​ | ms​ | Service Response time​ | +| CCNP initialization time​ | LIB​ | s​ | CCNP device plugin, DaemonSet and service readiness.​ | + +*Note: we use the CCNP SDK to access the CCNP service because it's convenient to prepare the request data (e.g. container ID, etc.)​ + +Below are the steps for you to build and run the performance test. + +## Prerequisites + +To run the test, you need a K8S cluster with CCNP enabled (CCNP Device Plugin and CCNP Service deployed and ready). + +## Build + +```bash +# Make sure you are on the repo's top dir +cd + +# Run doker build +docker build --build-arg http_proxy=$http_proxy --build-arg https_proxy=$https_proxy --build-arg no_proxy=$no_proxy -t ccnp-perf:latest -f container/ccnp-perf/Dockerfile . + +# View build result +docker image ls | grep ccnp-perf + +# Save the docker image for later use +docker save ccnp-perf:latest > ccnp-perf_latest.tar +``` + +## Deploy + +```bash +# Load the docker image for K8S using containerd. +# You need to run this on the node where you want to deploy the ccnp-perf test +ctr -n=k8s.io image import ccnp-perf_latest.tar + +# Make sure you are on the repo's top dir +cd + +# Deploy ccnp-perf test +kubectl apply -f deployment/kubernetes/manifests/ccnp-perf-deployment.yaml +``` + +## Test + +```bash +# Get the pod name of ccnp-perf +kubectl get pod | grep ccnp-perf + +# Run all perf test on the specified pod name got from above command +kubectl exec -ti -- python3 -m pytest --log-cli-level=INFO --verbose ccnp_perf.py +``` + +Sample test output looks like this: + +```bash +root@ccnp-perf-0:~/ccnp/confidential-cloud-native-primitives# kubectl exec -ti ccnp-perf-7f8798bf85-8s6zg -- python3 -m pytest --log-cli-level=INFO --verbose + ccnp_perf.py +==================================================================== test session starts ==================================================================== +platform linux -- Python 3.12.2, pytest-8.1.1, pluggy-1.4.0 -- /usr/local/bin/python3 +cachedir: .pytest_cache +rootdir: /run/ccnp +collected 7 items + +ccnp_perf.py::test_svc_get_cc_measurement_throughput +----------------------------------------------------------------------- live log call ----------------------------------------------------------------------- +INFO ccnp_perf:ccnp_perf.py:191 Perf test average throughput is: 70.75 ops (operations per second) +PASSED [ 14%] +ccnp_perf.py::test_svc_get_cc_measurement_response +----------------------------------------------------------------------- live log call ----------------------------------------------------------------------- +INFO ccnp_perf:ccnp_perf.py:213 Perf test average response time is: 25.89662575 ms (milliseconds) +PASSED [ 28%] +ccnp_perf.py::test_svc_get_cc_eventlog_throughput +----------------------------------------------------------------------- live log call ----------------------------------------------------------------------- +INFO ccnp_perf:ccnp_perf.py:191 Perf test average throughput is: 57.8 ops (operations per second) +PASSED [ 42%] +ccnp_perf.py::test_svc_get_cc_eventlog_response +----------------------------------------------------------------------- live log call ----------------------------------------------------------------------- +INFO ccnp_perf:ccnp_perf.py:213 Perf test average response time is: 76.130223 ms (milliseconds) +PASSED [ 57%] +ccnp_perf.py::test_svc_get_cc_report_throughput +----------------------------------------------------------------------- live log call ----------------------------------------------------------------------- +INFO ccnp_perf:ccnp_perf.py:191 Perf test average throughput is: 54.9 ops (operations per second) +PASSED [ 71%] +ccnp_perf.py::test_svc_get_cc_report_response +----------------------------------------------------------------------- live log call ----------------------------------------------------------------------- +INFO ccnp_perf:ccnp_perf.py:213 Perf test average response time is: 29.38618825 ms (milliseconds) +PASSED [ 85%] +ccnp_perf.py::test_ccnp_init PASSED [100%] + +=============================================================== 7 passed in 66.95s (0:01:06) ================================================================ +root@ccnp-perf-0:~/ccnp/confidential-cloud-native-primitives# +``` diff --git a/test/perf/deploy-perf.sh b/test/perf/deploy-perf.sh new file mode 100644 index 0000000..daa1c90 --- /dev/null +++ b/test/perf/deploy-perf.sh @@ -0,0 +1,120 @@ +#!/bin/bash +# Script to deploy multiple CCNP example pod for perf + +set -e + +DEFAULT_DOCKER_REPO=docker.io/library +DEFAULT_TAG=latest +WORK_DIR=$(cd "$(dirname "$0")" || exit; pwd) +TEMP_MANIFEST_FILE=/tmp/ccnp-example-perf-deployment.yaml +delete_deployment=false +number=1 +registry="" +tag="" + + +usage() { + cat << EOM +usage: $(basename "$0") [OPTION]... + -n Number of pods to run parallel perf testing + -r Image registry + -g Image tag + -d Delete perf example pods +EOM + exit 1 +} + +process_args() { + while getopts ":r:g:i:n:dh" option; do + case "${option}" in + r) registry=${OPTARG};; + g) tag=${OPTARG};; + d) delete_deployment=true;; + n) number=${OPTARG};; + h) usage;; + *) echo "Invalid option: -${OPTARG}" >&2 + usage + ;; + esac + done +} + +deploy_perf() { + echo "Deploy pods for performance testing " + pushd "${WORK_DIR}/../.." || exit + # replace registry and image tag according to user input + cp deployment/kubernetes/manifests/ccnp-example-perf-deployment.yaml $TEMP_MANIFEST_FILE + if [[ -n "$registry" ]]; then + sed -i "s#${DEFAULT_DOCKER_REPO}#${registry}#g" $TEMP_MANIFEST_FILE + fi + if [[ -n "$tag" ]];then + sed -i "s#${DEFAULT_TAG}#${tag}#g" $TEMP_MANIFEST_FILE + fi + + # Loop to deploy deployments + for ((i=1; i<=number; i++)); do + # Replace the placeholder in the template with the deployment name + sed "s/PLACEHOLDER/$i/g" $TEMP_MANIFEST_FILE > /tmp/ccnp-perf-example-$i.yaml + + # Apply the deployment to Kubernetes + kubectl apply -f /tmp/ccnp-perf-example-$i.yaml + done + + echo "==> Checking ccnp-example deployment are ready" + for i in {1..10} + do + pod_num=$(kubectl get po -n ccnp | grep perf-example | grep -i -c running) + if [ "$pod_num" -eq "$number" ] + then + echo "Perf example pods are ready" + break + else + sleep 2 + echo "Not ready yet ..." + fi + done + popd || exit +} + +# Delete all the perf deployment +delete_perf() { + for (( i=1; i<=number; i++ )) + do kubectl delete deployment ccnp-perf-example-$i -n ccnp; + echo "Deleting deployment ccnp-perf-example-$i ..." + done + + echo "==> Checking ccnp-example deployment are deleted" + for i in {1..10} + do + pod_num=$(kubectl get deployment -n ccnp | grep -c perf-example) + if [ "$pod_num" -eq 0 ] + then + echo "Perf example pods are deleted" + break + else + sleep 2 + echo "Not all deleted yet ..." + fi + done +} + +process_args "$@" + +echo "" +echo "-------------------------" +echo "Number of pods for perf testing: ${number}" +echo "Image registry: ${registry}" +echo "Image tag: ${tag}" +echo "delete pods: ${delete_deployment}" +echo "-------------------------" +echo "" + +if [[ $delete_deployment == true ]]; then + delete_perf +fi + +if [[ -n "$registry" && -n "$tag" ]]; then + deploy_perf +else + echo "Image registry and tag are not set." +fi diff --git a/test/perf/perf-para.sh b/test/perf/perf-para.sh new file mode 100644 index 0000000..5bbd9fd --- /dev/null +++ b/test/perf/perf-para.sh @@ -0,0 +1,150 @@ +#!/bin/bash +# Script to execute CCNP example pod + +set -e + +number=1 +is_eventlog=false +is_measurement=false +is_quote=false +command_eventlog="parallel perf_eventlog ::: " +command_measurement="parallel perf_measurement ::: " +command_quote="parallel perf_quote ::: " + +usage() { + cat << EOM +usage: $(basename "$0") [OPTION]... + -n Number of pods to run parallel perf testing + -e Run perf test for get eventlog + -m Run perf test for get measurement + -r Run perf test for get quote +EOM + exit 1 +} + +process_args() { + while getopts ":n:emrh" option; do + case "${option}" in + n) number=${OPTARG};; + e) is_eventlog=true;; + m) is_measurement=true;; + r) is_quote=true;; + h) usage;; + *) echo "Invalid option: -${OPTARG}" >&2 + usage + ;; + esac + done +} + +# generate a series of perf scripts +gen_commands() { + for ((i=1; i<=number; i++)); do + pod_name=$(kubectl get po -n ccnp | grep -i ccnp-perf-example-$i- | grep Running | awk '{ print $1 }') + + echo "kubectl exec -it $pod_name -n ccnp -- python3 py_perf.py -e" > /tmp/ccnp-perf-eventlog-$i.sh + chmod +x /tmp/ccnp-perf-eventlog-$i.sh + + echo "kubectl exec -it $pod_name -n ccnp -- python3 py_perf.py -m" > /tmp/ccnp-perf-measurement-$i.sh + chmod +x /tmp/ccnp-perf-measurement-$i.sh + + echo "kubectl exec -it $pod_name -n ccnp -- python3 py_perf.py -r" > /tmp/ccnp-perf-quote-$i.sh + chmod +x /tmp/ccnp-perf-quote-$i.sh + + command_eventlog+="/tmp/ccnp-perf-eventlog-$i.sh " + command_measurement+="/tmp/ccnp-perf-measurement-$i.sh " + command_quote+="/tmp/ccnp-perf-quote-$i.sh " + done +} + +# Function to execute a script and measure time +perf_eventlog() { + script_name=$1 + echo "Running $script_name..." + # Use time to measure execution time + output=perf_output_eventlog-$(date +"%Y%m%d_%H%M%S") + /usr/bin/time -f "$script_name took %E" bash "$script_name" >> "$output" 2>&1 +} + +perf_measurement() { + script_name=$1 + echo "Running $script_name..." + # Use time to measure execution time + output=perf_output_measurement-$(date +"%Y%m%d_%H%M%S") + /usr/bin/time -f "$script_name took %E" bash "$script_name" >> "$output" 2>&1 +} + +perf_quote() { + script_name=$1 + echo "Running $script_name..." + # Use time to measure execution time + output=perf_output_quote-$(date +"%Y%m%d_%H%M%S") + /usr/bin/time -f "$script_name took %E" bash "$script_name" >> "$output" 2>&1 +} + +run_perf_eventlog() { + export -f perf_eventlog + # Start time + start=$(date +%s) + $command_eventlog + + # End time + end=$(date +%s) + + # Calculate total duration + duration=$((end - start)) + echo "Parallel - Total execution time for all perf_eventlog scripts: $duration seconds" +} + +run_perf_measurement() { + export -f perf_measurement + # Start time + start=$(date +%s) + $command_measurement + + # End time + end=$(date +%s) + + # Calculate total duration + duration=$((end - start)) + echo "Parallel - Total execution time for all perf_measurement scripts: $duration seconds" +} + +run_perf_quote() { + export -f perf_quote + # Start time + start=$(date +%s) + $command_quote + + # End time + end=$(date +%s) + + # Calculate total duration + duration=$((end - start)) + echo "Parallel - Total execution time for all perf_quote scripts: $duration seconds" +} + +process_args "$@" + +echo "" +echo "-------------------------" +echo "number of pods for perf testing: ${number}" +echo "run perf_eventlog: ${is_eventlog}" +echo "run perf_measurement: ${is_measurement}" +echo "run perf_quote: ${is_quote}" +echo "-------------------------" +echo "" + +gen_commands + +if [[ $is_eventlog == true ]]; then + run_perf_eventlog +fi + +if [[ $is_measurement == true ]]; then + run_perf_measurement +fi + +if [[ $is_quote == true ]]; then + run_perf_quote +fi diff --git a/test/perf/py_perf.py b/test/perf/py_perf.py new file mode 100644 index 0000000..5595904 --- /dev/null +++ b/test/perf/py_perf.py @@ -0,0 +1,55 @@ +""" +CCNP SDK Perf Test +""" + +import logging +import argparse +import time + +from ccnp import CcnpSdk + +LOG = logging.getLogger(__name__) + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", + handlers=[ + logging.StreamHandler() + ] +) + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="The utility to show how to use CCNP SDK") + parser.add_argument('-r', action='store_true', help='get cc report', dest='report') + parser.add_argument('-e', action='store_true', help='get cc eventlog', dest='eventlog') + parser.add_argument('-m', action='store_true', help='get cc measurement', dest='measurement') + + args = parser.parse_args() + + inst = CcnpSdk.inst() + + if args.report: + start_time = time.time() + report = inst.get_cc_report() + end_time = time.time() + elapsed_time = end_time - start_time + LOG.info(f"Quote - Total command executed in {elapsed_time:.6f} seconds") + report.dump() + elif args.eventlog: + start_time = time.time() + evt = inst.get_cc_eventlog() + end_time = time.time() + elapsed_time = end_time - start_time + LOG.info(f"Eventlog - Total command executed in {elapsed_time:.6f} seconds") + for e in evt: + e.dump() + elif args.measurement: + start_time = time.time() + for i in [0, 1, 3]: + m = inst.get_cc_measurement([i, 12]) + LOG.info("IMR index: %d, hash: %s", i, m.hash.hex()) + end_time = time.time() + elapsed_time = end_time - start_time + LOG.info(f"Measurememt - Total command executed in {elapsed_time:.6f} seconds") + else: + parser.print_help()