diff --git a/scripts/gen_sai_dbg_dump.sh b/scripts/gen_sai_dbg_dump.sh new file mode 100644 index 0000000000..96fcc7abdc --- /dev/null +++ b/scripts/gen_sai_dbg_dump.sh @@ -0,0 +1,140 @@ +#!/bin/sh + +. /usr/local/bin/gen_sai_dbg_dump_lib.sh + +############################################################################### +# Prints the usage information. +# Globals: +# None +# Arguments: +# None +# Returns: +# None +############################################################################### +usage() { + cat < + +Generate and retrieve the SAI debug dump file from the syncd Docker container. + +OPTIONS: + -f Specify the destination file path for the SAI debug dump. + -h Display this help and exit. + +EXAMPLES: + $0 -f /var/log/dbg_gen_dump.log +EOF +} + +############################################################################### +# Copies a given file from a specified Docker container to the given target location. +# Globals: +# TIMEOUT_MIN +# Arguments: +# docker: Docker container name +# filename: The filename to copy +# destination: Destination filename +# Returns: +# None +############################################################################### +copy_from_docker() { + TIMEOUT_MIN="1" + local docker=$1 + local filename=$2 + local dstpath=$3 + local timeout_cmd="timeout --foreground ${TIMEOUT_MIN}m" + + local touch_cmd="sudo docker exec ${docker} touch ${filename}" + local cp_cmd="sudo docker cp ${docker}:${filename} ${dstpath}" + + RC=0 + eval "${timeout_cmd} ${touch_cmd}" || RC=$? + if [ $RC -ne 0 ]; then + echo "Command: $touch_cmd timed out after ${TIMEOUT_MIN} minutes." + fi + eval "${timeout_cmd} ${cp_cmd}" || RC=$? + if [ $RC -ne 0 ]; then + echo "Command: $cp_cmd timed out after ${TIMEOUT_MIN} minutes." + fi +} + +############################################################################### +# Main script logic +# Description: +# This is the main entry point of the script, which handles the generation +# and retrieval of the SAI debug dump file. It parses command-line arguments, +# ensures necessary directories and the `syncd` container are available, and +# triggers the SAI debug dump process through Redis. The script waits for the +# dump file to be generated and then copies it from the Docker container to +# the specified location on the local system. +# +# Globals: +# None +# +# Arguments: +# -f : Specifies the output filename for the SAI debug dump file. +# -h : Displays usage information. +# +# Returns: +# 0 - On success +# 1 - On failure +############################################################################### +main() { + # Parse arguments + while getopts ":f:h" opt; do + case $opt in + f) + sai_dump_filename="$OPTARG" + ;; + h) + usage + exit 0 + ;; + /?) + echo "Invalid option: -$OPTARG" >&2 + usage + exit 1 + ;; + esac + done + + local syncd_sai_dump_filename="/var/log/dbg_gen_dump.log" + + # Ensure a filename was provided + if [ -z "$sai_dump_filename" ]; then + echo "Error: Missing filename." + usage + exit 1 + fi + + # Ensure the directory exists, create it if it doesn't + if [ ! -d "$(dirname "$sai_dump_filename")" ]; then + sudo mkdir -p "$(dirname "$sai_dump_filename")" + fi + + # Ensure the syncd container is running + if [ "$(docker container inspect -f '{{.State.Running}}' syncd)" != "true" ]; then + echo "Error: syncd container is not running." + exit 1 + fi + + generate_sai_dump "$syncd_sai_dump_filename" + if [ $? -ne 0 ]; then + echo "Failed to generate SAI debug dump." + exit 1 + fi + + # Copy the dump file from the Docker container + local + if ! copy_from_docker syncd $syncd_sai_dump_filename $sai_dump_filename; then + echo "Error: Failed to copy the SAI dump file from the container." + exit 1 + fi + + # Remove the dump file from the Docker container + docker exec syncd rm -rf $syncd_sai_dump_filename; + echo "$sai_dump_filename is ready!!!" + exit 0 +} + +main "$@" diff --git a/scripts/gen_sai_dbg_dump_lib.sh b/scripts/gen_sai_dbg_dump_lib.sh new file mode 100644 index 0000000000..f730ccb5b5 --- /dev/null +++ b/scripts/gen_sai_dbg_dump_lib.sh @@ -0,0 +1,92 @@ +#!/bin/sh + +############################################################################### +# generate_sai_dump +# Description: +# This function triggers the generation of a SAI debug dump file in the +# `syncd` Docker container through Redis and waits for the file to be ready. +# +# Arguments: +# $1 - Filename for the SAI debug dump file. +# +# Returns: +# 0 - On success +# 1 - On failure +############################################################################### +generate_sai_dump() { + local DB=4 + local KEY="DBG_GEN_DUMP_TABLE|DUMP" + local STATUS_KEY="DBG_GEN_DUMP_STATS_TABLE|DUMP" + local FIELD="file" + local STATUS_FIELD="status" + local STATUS="1" + + local TIMEOUT_FOR_GEN_DBG_DUMP_FILE_READYNESS=10 + local INTERVAL=1 + local TIME_PASSED=0 + local EXISTS + + local SYNCD_DUMP_FILE="$1" + if [ -z "$SYNCD_DUMP_FILE" ]; then + echo "Error: No filename provided for the SAI debug dump file." + return 1 + fi + + # Extract the directory from the SYNCD_DUMP_FILE path + local SYNCD_DUMP_DIR + SYNCD_DUMP_DIR=$(dirname "$SYNCD_DUMP_FILE") + + # Ensure the directory exists in the syncd container; if not, create it + if ! docker exec syncd test -d "$SYNCD_DUMP_DIR"; then + echo "Directory '$SYNCD_DUMP_DIR' does not exist in the syncd container. Creating it..." + if ! docker exec syncd mkdir -p "$SYNCD_DUMP_DIR"; then + echo "Error: Failed to create directory '$SYNCD_DUMP_DIR' inside the syncd container." + return 1 + fi + fi + + # Delete the tables from STATE_DB before triger the dump file + redis-cli -n $DB DEL $KEY > /dev/null 2>&1 + redis-cli -n $DB DEL $STATUS_KEY > /dev/null 2>&1 + + # Set the DBG_GEN_DUMP in the Redis DB to trigger the dump generation + if ! redis-cli -n $DB HSET $KEY $FIELD $SYNCD_DUMP_FILE > /dev/null 2>&1; then + echo "Error: Failed to set Redis key." + return 1 + fi + + # Timeout and interval for checking status of file readiness + + while [ $TIME_PASSED -lt $TIMEOUT_FOR_GEN_DBG_DUMP_FILE_READYNESS ]; do + EXISTS=$(redis-cli -n $DB EXISTS "$STATUS_KEY" 2>/dev/null | grep -o '^[0-9]*$') + if [ "$EXISTS" -eq 1 ]; then + STATUS=$(redis-cli -n $DB HGET "$STATUS_KEY" "$STATUS_FIELD" 2>/dev/null | grep -o '^[0-9]*$') + break + fi + + sleep $INTERVAL + TIME_PASSED=$((TIME_PASSED + INTERVAL)) + done + + # Delete the tables from STATE_DB after triger the dump file + redis-cli -n $DB DEL $KEY > /dev/null 2>&1 + redis-cli -n $DB DEL $STATUS_KEY > /dev/null 2>&1 + + if [ "$STATUS" -ne 0 ]; then + echo "Error: dump file operation failed, Status $STATUS" + return 1 + fi + + if [ $TIME_PASSED -ge $TIMEOUT_FOR_GEN_DBG_DUMP_FILE_READYNESS ]; then + echo "Timeout reached. Status was not ready in time." + return 1 + fi + + # Ensure the file exists in the Docker container + if ! docker exec syncd test -f $SYNCD_DUMP_FILE; then + echo "Error: SAI dump file does not exist in the syncd container." + return 1 + fi + + return 0 +} diff --git a/scripts/generate_dump b/scripts/generate_dump index 3d0ef3430d..48208d063c 100755 --- a/scripts/generate_dump +++ b/scripts/generate_dump @@ -1107,6 +1107,83 @@ save_file() { echo "[ save_file:$orig_path] : $(($end_t-$start_t)) msec" >> $TECHSUPPORT_TIME_INFO } +############################################################################### +# generate_sai_dbg_dump_file +# Description: +# This function triggers the generation of a SAI debug dump file in the +# `syncd` Docker container and saves it to the local system. The function +# communicates with the Redis database to initiate the dump process +# +# Globals: +# None +# +# Arguments: +# None +# +# Returns: +# 0 - On success +# 1 - On failure +############################################################################### +generate_sai_dbg_dump_file() { + trap 'handle_error $? $LINENO' ERR + local syncd_sai_dump_filename="/var/log/dbg_gen_dump.log" + + if $NOOP; then + echo "generate_sai_dump" + return 1 + fi + + # Validate the argument + local sai_dump_filename="$1" + if [ -z "$sai_dump_filename" ]; then + echo "Error: No filename provided for the SAI debug dump file." + return 1 + fi + + # Extract the directory from the filename + local sai_dump_dir + sai_dump_dir=$(dirname "$sai_dump_filename") + + # Check if the directory exists; if not, create it + if [ ! -d "$sai_dump_dir" ]; then + echo "Directory '$sai_dump_dir' does not exist. Creating it..." + mkdir -p "$sai_dump_dir" + if [ $? -ne 0 ]; then + echo "Error: Failed to create directory '$sai_dump_dir'." + return 1 + fi + fi + + # Ensure the syncd container is running + if [[ "$( docker container inspect -f '{{.State.Running}}' syncd )" != "true" ]]; then + echo "Error: syncd container is not running." + return 1 + fi + + # Generate the SAI dump file using the refactored function + source /usr/local/bin/gen_sai_dbg_dump_lib.sh + if ! generate_sai_dump "$syncd_sai_dump_filename"; then + echo "Error: Failed to generate SAI debug dump." + return 1 + fi + + # Copy the dump file from the Docker container + if ! copy_from_docker syncd $syncd_sai_dump_filename $sai_dump_filename; then + echo "Error: Failed to copy the SAI dump file from the container." + return 1 + fi + + # Save the file using the save_file function + if ! save_file $sai_dump_filename sai_sdk_dump false; then + echo "Error: Failed to save the SAI dump file." + return 1 + fi + + # remove the dump files after saving to show techsupport package + rm -rf $sai_dump_filename + docker exec syncd rm -rf $syncd_sai_dump_filename; + +} ############################################################################### # find_files routine # Globals: @@ -1341,6 +1418,10 @@ save_marvellcmd() { ############################################################################### collect_marvell() { trap 'handle_error $? $LINENO' ERR + + local sai_dump_folder="/tmp/saisdkdump" + local sai_dump_filename="${sai_dump_folder}/sai_sdk_dump_$(date +"%m_%d_%Y_%I_%M_%p")" + generate_sai_dbg_dump_file "$sai_dump_filename" save_marvellcmd "show version" "CPSS_version" save_marvellcmd "debug-mode;xps-api call xpsDataIntegrityDumpSerInfo all" "SER_table" diff --git a/setup.py b/setup.py index 6a66f012f9..3fcb7d9519 100644 --- a/setup.py +++ b/setup.py @@ -188,7 +188,9 @@ 'scripts/verify_image_sign.sh', 'scripts/verify_image_sign_common.sh', 'scripts/check_db_integrity.py', - 'scripts/sysreadyshow' + 'scripts/sysreadyshow', + 'scripts/gen_sai_dbg_dump.sh', + 'scripts/gen_sai_dbg_dump_lib.sh', ], entry_points={ 'console_scripts': [