classify-irods-errors

#!/usr/bin/env bash

show_help() {
	cat <<EOF

$EXEC_NAME version $VERSION

Usage:
 $EXEC_NAME [options] CLASS-BASE

Reads the output of the \`phymv\` or \`repl\` command and groups the data
objects that failed to be handled into different files based on the error that
occurred.

Parameters:
 CLASS-BASE  the common basename of the files that hold the groups of data
             objects

Options:
 -h, --help     show help and exit
 -v, --version  show verion and exit

Summary:

The program reads the output of the \`phymv\` or \`repl\` command from standard
in. It groups the data objects that failed to be handled into different files
based on the error that occurred. Here's a breakdown of what each output file
holds.

CLASS-BASE.src_path_errors holds the paths to data objects that have paths that
cause \`iphymv\` or \`irepl\` problems.

CLASS-BASE.catalog_errors holds the paths to data objects that failed to be
handled because of DBMS related issue.

CLASS-BASE.chksum_mismatches holds the paths to data objects with checksums that
differ from the checksums of their respective physical files.

CLASS-BASE.fs_errors holds the paths to data objects where there are issues with
the file systems holding the objects' physical files.

CLASS-BASE.missing_replicas holds the paths to data objects that are missing
their physical files.

CLASS-BASE.network holds the paths to data objects where network connection
issues prevented the data object from being handled.

CLASS-BASE.short_file holds the paths to data objects having sizes larger than
their respective physical files' sizes.

CLASS-BASE.write holds the paths to data objects where the replica failed to be
written to destination resource server.

CLASS-BASE.unclassified_tool_errors holds the log messages for tool related
errors that couldn't be classified.

CLASS-BASE.unclassified_errors holds the full log messages for errors that
couldn't be classified at all.

© 2022, The Arizona Board of Regents on behalf of The University of Arizona. For
license information, see https://cyverse.org/license.
EOF
}

readonly VERSION=6

set -o errexit -o nounset -o pipefail

EXEC_NAME="$(basename "$(realpath --canonicalize-existing "$0")")"
readonly EXEC_NAME

declare -a TempFiles

main() {
	local opts
	if ! opts="$(getopt --name "$EXEC_NAME" --options hv --longoptions help,version -- "$@")"; then
		show_help >&2
		return 1
	fi

	eval set -- "$opts"

	local help version

	while true; do
		case "$1" in
		-h | --help)
			help=help
			shift
			;;
		-v | --version)
			version=version
			shift
			;;
		--)
			shift
			break
			;;
		*)
			show_help >&2
			return 1
			;;
		esac
	done

	if [[ -n "${help-}" ]]; then
		show_help
		return 0
	fi

	if [[ -n "${version-}" ]]; then
		printf '%s\n' "$VERSION"
		return 0
	fi

	if [[ "$#" -lt 1 ]]; then
		show_help >&2
		return 1
	fi

	local logBase="$1"

	trap remove_temp_files EXIT
	process_input "$logBase"
}

remove_temp_files() {
	if ((${#TempFiles[@]} > 0)); then
		rm --force "${TempFiles[@]}"
	fi
}

process_input() {
	local logBase="$1"

	local errorsFile
	errorsFile="$(mktemp)"
	TempFiles+=("$errorsFile")

	filter_errors | sort >"$errorsFile"

	local errCnt
	errCnt="$(wc --lines <"$errorsFile")"

	display_error_count errors "$errCnt" "$errCnt"
	classify_errors "$errorsFile" "$errCnt" "$logBase"
}

filter_errors() {
	sed --quiet --file - <(cat) <<'EOF'
/.*ERROR: [^\[].*$/ {
	s/.*ERROR: \([^\[].*$\)/\1/
	/connectToRhost: readVersion to .* failed, msg=/d
	/^cliReconnManager: .*, status = -305111/d
	/^cliReconnManager: readReconMsg to host [^ ]\+ failed, status = -4000/d
	/^replUtil: invalid repl objType 0 for /d
	/^_rcConnect: connectToRhost timed out, retrying/d
	/^_rcConnect: connectToRhost error, server on .* is probably down status = -115000 SYS_SOCK_READ_TIMEDOUT/d
	p
}
EOF
}

classify_errors() {
	local errorsFile="$1"
	local errCnt="$2"
	local logBase="$3"

	local sPECnt
	sPECnt="$(split_out_class \
		"$errorsFile" \
		'^\(phymv\|repl\)Util: srcPath \(.*\) does not exist$' \
		'\2' \
		"$logBase".src_path_errors)"

	display_error_count 'source path errors' "$sPECnt" "$errCnt"

	local toolErrorsFile
	toolErrorsFile="$(mktemp)"
	TempFiles+=("$toolErrorsFile")

	split_out_class \
		"$errorsFile" '^\(phymv\|repl\)Util: \(phymv\|repl\) error for ' '' "$toolErrorsFile" \
		>/dev/null

	sort --output "$toolErrorsFile" "$toolErrorsFile"
	classify_tool_errors "$toolErrorsFile" "$errCnt" "$logBase"

	local unclassifiedErrCnt
	unclassifiedErrCnt="$(wc --lines <"$errorsFile")"

	display_error_count 'unclassified errors' "$unclassifiedErrCnt" "$errCnt"

	if ((unclassifiedErrCnt > 0)); then
		mv "$errorsFile" "$logBase".unclassified_errors
	fi
}

classify_tool_errors() {
	local errorsFile="$1"
	local totErrCnt="$2"
	local logBase="$3"

	local sCLECnt
	sCLECnt="$(split_out_tool_class "$errorsFile" -27000 SYS_COPY_LEN_ERR "$logBase".short_file)"

	local ncCnt
	ncCnt="$(split_out_conn_errs "$errorsFile" "$logBase".network)"

	local uCMCnt
	uCMCnt="$(split_out_tool_class \
		"$errorsFile" -314000 USER_CHKSUM_MISMATCH "$logBase".chksum_mismatches )"

	local uFOEMissingCnt
	uFOEMissingCnt="$(split_out_tool_class \
		"$errorsFile" \
		-510002 \
		'UNIX_FILE_OPEN_ERR, No such file or directory' \
		"$logBase".missing_replicas )"

	local uFOEFSErrCnt
	uFOEFSErrCnt="$(split_out_fs_errs "$errorsFile" "$logBase".fs_errors)"

	local writeErrCnt
	writeErrCnt="$(split_out_write_errs "$errorsFile" "$logBase".write)"

	local catErrCnt
	catErrCnt="$(split_out_tool_class "$errorsFile" -806000 CAT_SQL_ERR "$logBase".catalog_errors)"

	local unclassifiedToolErrCnt
	unclassifiedToolErrCnt="$(wc --lines <"$errorsFile")"

	if (( unclassifiedToolErrCnt > 0 )); then
		mv "$errorsFile" "$logBase".unclassified_tool_errors
	fi

	display_error_count 'catalog errors' "$catErrCnt" "$totErrCnt"
	display_error_count 'checksum mismatches' "$uCMCnt" "$totErrCnt"
	display_error_count 'file open errors' "$uFOEMissingCnt" "$totErrCnt"
	display_error_count 'filesystem errors' "$uFOEFSErrCnt" "$totErrCnt"
	display_error_count 'network connection issues' "$ncCnt" "$totErrCnt"
	display_error_count 'short files' "$sCLECnt" "$totErrCnt"
	display_error_count 'write failures' "$writeErrCnt" "$totErrCnt"
	display_error_count 'unclassified tool errors' "$unclassifiedToolErrCnt" "$totErrCnt"
}

display_error_count() {
	local label="$1"
	local count="$2"
	local total="$3"

	printf '%*d %s\n' "${#total}" "$count" "$label"
}

split_out_conn_errs() {
	local errorsFile="$1"
	local classFile="$2"

	local cnt=0

	: $((cnt += $(split_out_tool_class "$errorsFile" -4000 SYS_HEADER_READ_LEN_ERR "$classFile")))

	local copyCnt
	copyCnt="$(split_out_tool_class \
		"$errorsFile" -27110 'SYS_COPY_LEN_ERR, Connection timed out' "$classFile")"

	: $((cnt += copyCnt))
	: $((cnt += $(split_out_tool_class "$errorsFile" -38000 SYS_AGENT_INIT_ERR "$classFile")))
	: $((cnt += $(split_out_tool_class "$errorsFile" -115000 SYS_SOCK_READ_TIMEDOUT "$classFile")))

	local connTOCnt
	connTOCnt="$(split_out_tool_class \
		"$errorsFile" -116110 'SYS_SOCK_READ_ERR, Connection timed out' "$classFile")"
	: $((cnt += connTOCnt))

	local connRstCnt
	connRstCnt="$(split_out_tool_class \
		"$errorsFile" -116104 'SYS_SOCK_READ_ERR, Connection reset by peer' "$classFile")"
	: $((cnt += connRstCnt))

	: $((cnt += $(split_out_tool_class "$errorsFile" -305101 USER_SOCK_CONNECT_ERR "$classFile")))

	local connRefuseCnt
	connRefuseCnt="$(split_out_tool_class \
		"$errorsFile" -305111 'USER_SOCK_CONNECT_ERR, Connection refused' "$classFile")"
	: $((cnt += connRefuseCnt))

	local connTOCnt
	connTO="$(split_out_tool_class "$errorsFile" -347000 USER_SOCK_CONNECT_TIMEDOUT "$classFile")"
	: $((cnt += connTO))

	echo "$cnt"
}

split_out_fs_errs() {
	local errorsFile="$1"
	local classFile="$2"

	local uFOEIOErrCnt
	uFOEIOErrCnt="$(split_out_tool_class \
		"$errorsFile" \
		-510005 \
		'UNIX_FILE_OPEN_ERR, Input\/output error' \
		"$classFile" )"

	local uFOESNCErrCnt
	uFOESNCErrCnt="$(split_out_tool_class \
		"$errorsFile" \
		-510117 \
		'UNIX_FILE_OPEN_ERR, Structure needs cleaning' \
		"$classFile" )"

	local uFWEDQEErrCnt
	uFWEDQEErrCnt="$(split_out_tool_class \
		"$errorsFile" \
		-513122 \
		'UNIX_FILE_WRITE_ERR, Disk quota exceeded' \
		"$classFile" )"

	local uFGFFErrCnt
	uFGFFErrCnt="$(split_out_tool_class \
		"$errorsFile" \
		-526020 \
		'UNIX_FILE_GET_FS_FREESPACE_ERR, Not a directory' \
		"$classFile" )"

	echo $(( uFOEIOErrCnt + uFOESNCErrCnt + uFWEDQEErrCnt + uFGFFErrCnt ))
}

split_out_write_errs() {
	local errorsFile="$1"
	local classFile="$2"

	local ufwErrCnt
	ufwErrCnt="$(
		split_out_tool_class \
			"$errorsFile" \
			-513002 \
			'UNIX_FILE_WRITE_ERR, No such file or directory' \
			"$classFile" )"

	local uftlCnt
	uftlCnt="$(split_out_tool_class "$errorsFile" -311000 USER_FILE_TOO_LARGE "$classFile")"

	echo $(( ufwErrCnt + uftlCnt ))
}

split_out_tool_class() {
	local errorsFile="$1"
	local status="$2"
	local error="$3"
	local classFile="$4"

	split_out_class "$errorsFile" ", status = $status status = $status $error\$" '' "$classFile"
}

split_out_class() {
	local errorsFile="$1"
	local classifier="$2"
	local substitution="$3"
	local classFile="$4"

	local errors
	errors="$(cat "$errorsFile")"

	local classifiedErrors
	classifiedErrors="$(sed --quiet "/$classifier/p" <<<"$errors")"

	if [[ -n "$classifiedErrors" ]]; then
		comm -2 -3 <(echo "$errors") <(echo "$classifiedErrors") >"$errorsFile"
		#shellcheck disable=SC2001
		sed "s/$classifier/$substitution/" <<<"$classifiedErrors" >>"$classFile"
		wc --lines <<<"$classifiedErrors"
	else
		printf '0'
	fi
}

main "$@"