diff --git a/check_data_and_create_pr.sh b/check_data_and_create_pr.sh index 27dbc55..962bd24 100755 --- a/check_data_and_create_pr.sh +++ b/check_data_and_create_pr.sh @@ -1,23 +1,57 @@ #!/bin/bash -set -e -bash scripts/validate_token.sh +# setted by GitHub +set +o pipefail + +export TERM=xterm-color +GRN="\033[1;92m" # success messages, green +RED="\033[1;91m" # error messages, red +BL="\033[1;94m" # info messages (usually skip messages), blue +YEL="\033[1;93m" # warnings, yellow +MGN='\033[1;95m' # start step info, magenta +CYA='\033[1;96m' # more bright info messages, cyan +ENDC="\033[0m" # end of color message + +color_message() { + message=$1 + color=$2 + echo -e "${color}${message}${ENDC}" +} +. scripts/validate_token.sh + +set -e # checkout fork repo (via temp dir as current dir is not emply and it does't allow to check out repo in it) git clone "https://${REPO_OWNER}:${ACTION_TOKEN}@github.com/${REPO_OWNER}/${REPO_NAME}.git" temp +shopt -s dotglob mv temp/* . -mv temp//.git* . rmdir temp +if [ ! -d "data" ]; then + echo $(color_message "ERROR: 'data' directory is empty (for git) or not exist. Please add '.gitkeep' file in 'data' folder if need" $RED) + exit 1 +fi + +set +e +set +o pipefail +# remove unused branches before creating new +git checkout master +git branch --list | cat +merged_branches=$(git branch -r --merged | grep "update_*" -c) +nomerged_branches=$(git branch -r --no-merged | grep "update_*" -c) +# delete all merged and unmerged remote `update_*` branches +if [[ $merged_branches > 0 ]] +then + git branch -r --merged | grep "update_*" | cut -d "/" -f 2 | xargs git push --delete origin +fi -git config diff.renameLimit 999999 -CHANGED_DATA_FILES=$(git diff --name-only -r HEAD^1 HEAD | wc -l) -if [[ $CHANGED_DATA_FILES -gt 3000 ]]; then - echo "More than 3000 files added/changed ($CHANGED_DATA_FILES files total). Please, push commit with changes in less than 3000 files" - exit 1 +if [[ $nomerged_branches > 0 ]] +then + git branch -r --no-merged | grep "update_*" | cut -d "/" -f 2 | xargs git push --delete origin fi +set -e # create a new branch for update git checkout master @@ -30,8 +64,10 @@ export GROUP=${REPO_NAME} python3 scripts/simple_data_check.py # close previous PR if it exists -bash scripts/close_pr_if_exists.sh +. scripts/close_pr_if_exists.sh # create new PR +set +e export GH_TOKEN=${ACTION_TOKEN} gh api -X POST /repos/tradingview-pine-seeds/${REPO_NAME}/pulls -f base="master" -f head="${REPO_OWNER}:${PR_BRANCH_NAME}" -f title="Upload data" > /dev/null 2>&1 +echo diff --git a/close_pr_if_exists.sh b/close_pr_if_exists.sh index 3912118..5a5a7cf 100755 --- a/close_pr_if_exists.sh +++ b/close_pr_if_exists.sh @@ -3,7 +3,7 @@ echo "${ACTION_TOKEN}" | gh auth login --with-token > /dev/null 2>&1 if [ -z $? ] then - echo "Authorization failed. Update ACTION_TOKEN in the repository secrets." + echo $(color_message "Authorization failed. Update ACTION_TOKEN in the repository secrets." $RED) exit 1 fi @@ -11,18 +11,18 @@ EXISTING_PRS=$(gh api -X GET /repos/tradingview-pine-seeds/"$REPO_NAME"/pulls) if [ "$EXISTING_PRS" != "[]" ]; then NUMBER_OF_PRS=$(echo "$EXISTING_PRS" | jq length) if [ "$NUMBER_OF_PRS" != 1 ]; then - echo "There is more than one PR open. To resolve the issue, contact pine.seeds@tradingview.com with the Pine Seeds Issue subject." + echo $(color_message "There is more than one PR open. To resolve the issue, contact pine.seeds@tradingview.com with the Pine Seeds Issue subject." $RED) exit 1 fi BASE_LABEL=$(echo "$EXISTING_PRS" | jq -r ".[0].base.label") if [ "$BASE_LABEL" != "tradingview-pine-seeds:master" ]; then - echo "base = $BASE_LABEL is incorrect" + echo $(color_message "base = $BASE_LABEL is incorrect" $RED) exit 1 fi HEAD_LABEL=$(echo "$EXISTING_PRS" | jq -r ".[0].head.label") OWNER="${HEAD_LABEL%:*}" if [ "${OWNER}" != "$REPO_OWNER" ]; then - echo "head = $HEAD_LABEL is incorrect" + echo $(color_message "head = $HEAD_LABEL is incorrect" $RED) exit 1 fi NUM=$(echo "$EXISTING_PRS" | jq -r ".[0].number") diff --git a/simple_data_check.py b/simple_data_check.py index 1d3bfa4..fb48b3b 100755 --- a/simple_data_check.py +++ b/simple_data_check.py @@ -8,7 +8,6 @@ import glob import json import os -from os import getenv from os.path import exists, isfile from sys import exit as sys_exit, argv from datetime import datetime @@ -22,6 +21,11 @@ DESCRIPTION_RE: Pattern[str] = re_compile(r'^.+$') PRICESCALE_RE: Pattern[str] = re_compile(r'^1(0){0,22}$') REPORTS_PATH: str = argv[1] if len(argv) > 1 else None +MAX_ERRORS_IN_MSG: int = int(os.getenv("MAX_ERRORS_IN_MSG", 50)) # max show errors in console, file or PR message +THRESHOLD_ERR: int = int(os.getenv('THRESHOLD_ERR', 10)) +RED="\033[1;91m" # error messages, red +YEL="\033[1;93m" # warnings, yellow +ENDC="\033[0m" # end of color message def check_type(values: Any, val_type: type) -> bool: @@ -137,7 +141,7 @@ def check_data_line(data_line: str, file_path: str, i: int) -> Tuple[List[str], if len(vals) != 6: # YYYYMMDDT, o, h, l, c, v messages.append(F'{file_path}:{i} contains incorrect number of elements (expected: 6, actual: {len(vals)})') return messages, date - + check_ok = True # validate float try: @@ -147,7 +151,7 @@ def check_data_line(data_line: str, file_path: str, i: int) -> Tuple[List[str], open_price, high_price, low_price, close_price, volume = float(vals[1]), float(vals[2]), float(vals[3]), float(vals[4]), float(vals[5]) except ValueError: check_ok = False - messages.append(F'{file_path}:{i} float values validation error. The float value can\'t be NAN/+INF/-INF') + messages.append(F'{file_path}:{i} float values validation error. The float value can\'t be NAN/+INF/-INF') # validate date try: if len(vals[0]) != 9: # value '202291T' is considered as correct date 2022/09/01 by datetime.strptime but specification require zero-padded values @@ -156,7 +160,7 @@ def check_data_line(data_line: str, file_path: str, i: int) -> Tuple[List[str], except (ValueError, TypeError): check_ok = False messages.append(F'{file_path}:{i} date validation error, date format have to be YYYYMMDDT, for example: 20230101T') - + if check_ok: date = vals[0] if not (open_price <= high_price >= close_price >= low_price <= open_price and high_price >= low_price): @@ -198,7 +202,10 @@ def check_data_files(sym_file_path: str, symbols: List[str], problems: Dict[str, if len(parts) != 2: problems["errors"].append(F'Invalid file name. Check that {file} has a valid name and extension.') continue - if parts[1] not in ("csv", "CSV"): + if parts[1] in {"gitkeep"}: + sym_set.discard(parts[0]) + continue + if parts[1] not in {"csv", "CSV"}: problems["errors"].append(F'Invalid file extension. The {file} file format must be CSV.') continue if parts[0] not in sym_set: @@ -213,7 +220,7 @@ def check_data_files(sym_file_path: str, symbols: List[str], problems: Dict[str, def fail(msg: str) -> None: """ report about fail and exit with non-zero exit code""" if REPORTS_PATH is None: - print(msg) + print(RED+msg+ENDC) sys_exit(1) with open(os.path.join(REPORTS_PATH, "report.txt"), "a") as file: file.write(msg) @@ -222,7 +229,7 @@ def fail(msg: str) -> None: def main() -> None: """ main routine """ - group = getenv("GROUP") + group = os.getenv("GROUP") if group == "": fail("ERROR: the GROUP environment variable is not set") sym_file_path = F"symbol_info/{group}.json" @@ -234,20 +241,32 @@ def main() -> None: problems["errors"] = sym_errors if len(symbols) > 0: check_data_files(sym_file_path, symbols, problems) - + # report warnings - if len(problems["missed_files"]) > 0: - warning = F'WARNING: the following symbols have no corresponding CSV files in the data folder: {", ".join(problems["missed_files"])}\n' + len_problems_missed_files = len(problems["missed_files"]) + if len_problems_missed_files > 0: + if len_problems_missed_files > MAX_ERRORS_IN_MSG + THRESHOLD_ERR: + warning = F'WARNING: the following symbols have no corresponding CSV files in the data folder: {", ".join(problems["missed_files"][:MAX_ERRORS_IN_MSG])} and {len_problems_missed_files - MAX_ERRORS_IN_MSG} other CSV files.\n' + else: + warning = F'WARNING: the following symbols have no corresponding CSV files in the data folder: {", ".join(problems["missed_files"])}\n' + if REPORTS_PATH is None: - print(warning) + print(YEL+warning+ENDC) else: with open(os.path.join(REPORTS_PATH, "warnings.txt"), "a") as file: file.write(warning) - + # report errors - if len(problems["errors"]) > 0: - problems_list = "\n ".join(problems["errors"]) - fail(F'ERROR: the following issues were found in the repository files:\n {problems_list}\n') + len_problems_errors = len(problems["errors"]) + if len_problems_errors > 0: + if len_problems_errors > MAX_ERRORS_IN_MSG + THRESHOLD_ERR: + problems_list = "\n ".join(problems["errors"][:MAX_ERRORS_IN_MSG]) + error_msg = F'ERROR: the following issues were found in the repository files:\n {problems_list} and {len_problems_errors - MAX_ERRORS_IN_MSG} other errors. \n' + else: + problems_list = "\n ".join(problems["errors"]) + error_msg = F'ERROR: the following issues were found in the repository files:\n {problems_list}\n' + + fail(error_msg) if __name__ == "__main__": diff --git a/validate_token.sh b/validate_token.sh index 7bdcfa7..6da75a6 100755 --- a/validate_token.sh +++ b/validate_token.sh @@ -1,32 +1,33 @@ +#!/bin/bash if [ -z "${ACTION_TOKEN}" ] then - echo "Failed to find ACTION_TOKEN. Make sure that ACTION_TOKEN is set in the repository secrets." + echo $(color_message "Failed to find ACTION_TOKEN. Make sure that ACTION_TOKEN is set in the repository secrets." $RED) exit 1 fi RESP=$(echo "${ACTION_TOKEN}" | gh auth login --with-token 2>&1) if [ $? -ne 0 ] then - echo "Authorization failed. Make sure that your ACTION_TOKEN is valid and not expired." + echo $(color_message "Authorization failed. Make sure that your ACTION_TOKEN is valid and not expired." $RED) exit 1 fi if [[ "$RESP" == *"error"* ]]; then - echo "Insufficient scope error. Provide ACTION_TOKEN with the repo, workflow, and admin:org scopes." + echo $(color_message "Insufficient scope error. Provide ACTION_TOKEN with the repo, workflow, and admin:org scopes." $RED) exit 1 fi SCOPES=$(gh auth status 2>&1 | grep "scopes") valid=true if [[ "$SCOPES" != *"workflow"* ]]; then - echo "Insufficient scope error. Provide ACTION_TOKEN with the workflow scope." + echo echo $(color_message "Insufficient scope error. Provide ACTION_TOKEN with the workflow scope." $RED) valid=false fi if [[ "$SCOPES" != *"admin:org"* ]]; then - echo "Insufficient scope error. Provide ACTION_TOKEN with the admin:org scope." + echo $(color_message "Insufficient scope error. Provide ACTION_TOKEN with the admin:org scope." $RED) valid=false fi @@ -34,4 +35,4 @@ if ! $valid ; then exit 1 fi -echo "ACTION_TOKEN validation successful" +echo $(color_message "ACTION_TOKEN validation successful" $GRN)