DF-2449: check empty data folder

check_data_and_create_pr.sh

@@ -1,23 +1,57 @@
 #!/bin/bash
-set -e
 
-bash scripts/validate_token.sh
+# setted by GitHub
+set +o pipefail
+
+export TERM=xterm-color
+GRN="\033[1;92m"  # success messages, green
+RED="\033[1;91m"  # error messages, red
+BL="\033[1;94m"   # info messages (usually skip messages), blue
+YEL="\033[1;93m"  # warnings, yellow
+MGN='\033[1;95m'  # start step info, magenta
+CYA='\033[1;96m'  # more bright info messages, cyan
+ENDC="\033[0m"    # end of color message
+
+color_message() {
+    message=$1
+    color=$2
+    echo -e "${color}${message}${ENDC}"
+}
 
+. scripts/validate_token.sh
+
+set -e
 # checkout fork repo (via temp dir as current dir is not emply and it does't allow to check out repo in it)
 git clone "https://${REPO_OWNER}:${ACTION_TOKEN}@github.com/${REPO_OWNER}/${REPO_NAME}.git" temp
+shopt -s dotglob
 mv temp/* .
-mv temp//.git* .
 rmdir temp
 
+if [ ! -d "data" ]; then
+    echo $(color_message "ERROR: 'data' directory is empty (for git) or not exist. Please add '.gitkeep' file in 'data' folder if need" $RED)
+    exit 1
+fi
+
+set +e
+set +o pipefail
+# remove unused branches before creating new
+git checkout master
+git branch --list | cat
+merged_branches=$(git branch -r --merged | grep "update_*" -c)
+nomerged_branches=$(git branch -r --no-merged | grep "update_*" -c)
 
+# delete all merged and unmerged remote `update_*` branches
+if [[ $merged_branches > 0 ]]
+then
+    git branch -r --merged | grep "update_*" | cut -d "/" -f 2 | xargs git push --delete origin
+fi
 
-git config diff.renameLimit 999999
-CHANGED_DATA_FILES=$(git diff --name-only -r HEAD^1 HEAD | wc -l)
-if [[ $CHANGED_DATA_FILES -gt 3000 ]]; then
-    echo "More than 3000 files added/changed ($CHANGED_DATA_FILES files total). Please, push commit with changes in less than 3000 files"
-    exit 1
+if [[ $nomerged_branches > 0 ]]
+then
+    git branch -r --no-merged | grep "update_*" | cut -d "/" -f 2 | xargs git push --delete origin
 fi
 
+set -e
 
 # create a new branch for update
 git checkout master
@@ -30,8 +64,10 @@ export GROUP=${REPO_NAME}
 python3 scripts/simple_data_check.py
 
 # close previous PR if it exists
-bash scripts/close_pr_if_exists.sh
+. scripts/close_pr_if_exists.sh
 
 # create new PR
+set +e
 export GH_TOKEN=${ACTION_TOKEN}
 gh api -X POST /repos/tradingview-pine-seeds/${REPO_NAME}/pulls -f base="master" -f head="${REPO_OWNER}:${PR_BRANCH_NAME}" -f title="Upload data" > /dev/null 2>&1
+echo

close_pr_if_exists.sh

@@ -3,26 +3,26 @@
 echo "${ACTION_TOKEN}" | gh auth login --with-token > /dev/null 2>&1
 if [ -z $? ]
 then
-    echo "Authorization failed. Update ACTION_TOKEN in the repository secrets."
+    echo $(color_message "Authorization failed. Update ACTION_TOKEN in the repository secrets." $RED)
     exit 1
 fi
 
 EXISTING_PRS=$(gh api -X GET /repos/tradingview-pine-seeds/"$REPO_NAME"/pulls)
 if [ "$EXISTING_PRS" != "[]" ]; then
     NUMBER_OF_PRS=$(echo "$EXISTING_PRS" | jq length)
     if [ "$NUMBER_OF_PRS" != 1 ]; then
-        echo "There is more than one PR open. To resolve the issue, contact [email protected] with the Pine Seeds Issue subject."
+        echo $(color_message "There is more than one PR open. To resolve the issue, contact [email protected] with the Pine Seeds Issue subject." $RED)
         exit 1
     fi
     BASE_LABEL=$(echo "$EXISTING_PRS" | jq -r ".[0].base.label")
     if [ "$BASE_LABEL" != "tradingview-pine-seeds:master" ]; then
-        echo "base = $BASE_LABEL is incorrect"
+        echo $(color_message "base = $BASE_LABEL is incorrect" $RED)
         exit 1
     fi
     HEAD_LABEL=$(echo "$EXISTING_PRS" | jq -r ".[0].head.label")
     OWNER="${HEAD_LABEL%:*}"
     if [ "${OWNER}" != "$REPO_OWNER" ]; then
-        echo "head = $HEAD_LABEL is incorrect"
+        echo $(color_message "head = $HEAD_LABEL is incorrect" $RED)
         exit 1
     fi
     NUM=$(echo "$EXISTING_PRS" | jq -r ".[0].number")

simple_data_check.py

@@ -8,7 +8,6 @@
 import glob
 import json
 import os
-from os import getenv
 from os.path import exists, isfile
 from sys import exit as sys_exit, argv
 from datetime import datetime
@@ -22,6 +21,11 @@
 DESCRIPTION_RE: Pattern[str] = re_compile(r'^.+$')
 PRICESCALE_RE: Pattern[str] = re_compile(r'^1(0){0,22}$')
 REPORTS_PATH: str = argv[1] if len(argv) > 1 else None
+MAX_ERRORS_IN_MSG: int = int(os.getenv("MAX_ERRORS_IN_MSG", 50))   # max show errors in console, file or PR message
+THRESHOLD_ERR: int = int(os.getenv('THRESHOLD_ERR', 10))
+RED="\033[1;91m"  # error messages, red
+YEL="\033[1;93m"  # warnings, yellow
+ENDC="\033[0m"    # end of color message
 
 
 def check_type(values: Any, val_type: type) -> bool:
@@ -137,7 +141,7 @@ def check_data_line(data_line: str, file_path: str, i: int) -> Tuple[List[str],
     if len(vals) != 6:  # YYYYMMDDT, o, h, l, c, v
         messages.append(F'{file_path}:{i} contains incorrect number of elements (expected: 6, actual: {len(vals)})')
         return messages, date
-    
+
     check_ok = True
     # validate float
     try:
@@ -147,7 +151,7 @@ def check_data_line(data_line: str, file_path: str, i: int) -> Tuple[List[str],
         open_price, high_price, low_price, close_price, volume = float(vals[1]), float(vals[2]), float(vals[3]), float(vals[4]), float(vals[5])
     except ValueError:
         check_ok = False
-        messages.append(F'{file_path}:{i} float values validation error. The float value can\'t be NAN/+INF/-INF') 
+        messages.append(F'{file_path}:{i} float values validation error. The float value can\'t be NAN/+INF/-INF')
     # validate date
     try:
         if len(vals[0]) != 9:  # value '202291T' is considered as correct date 2022/09/01 by datetime.strptime but specification require zero-padded values
@@ -156,7 +160,7 @@ def check_data_line(data_line: str, file_path: str, i: int) -> Tuple[List[str],
     except (ValueError, TypeError):
         check_ok = False
         messages.append(F'{file_path}:{i} date validation error, date format have to be YYYYMMDDT, for example: 20230101T')
-    
+
     if check_ok:
         date = vals[0]
         if not (open_price <= high_price >= close_price >= low_price <= open_price and high_price >= low_price):
@@ -198,7 +202,10 @@ def check_data_files(sym_file_path: str, symbols: List[str], problems: Dict[str,
         if len(parts) != 2:
             problems["errors"].append(F'Invalid file name. Check that {file} has a valid name and extension.')
             continue
-        if parts[1] not in ("csv", "CSV"):
+        if parts[1] in {"gitkeep"}:
+            sym_set.discard(parts[0])
+            continue
+        if parts[1] not in {"csv", "CSV"}:
             problems["errors"].append(F'Invalid file extension. The {file} file format must be CSV.')
             continue
         if parts[0] not in sym_set:
@@ -213,7 +220,7 @@ def check_data_files(sym_file_path: str, symbols: List[str], problems: Dict[str,
 def fail(msg: str) -> None:
     """ report about fail and exit with non-zero exit code"""
     if REPORTS_PATH is None:
-        print(msg)
+        print(RED+msg+ENDC)
         sys_exit(1)
     with open(os.path.join(REPORTS_PATH, "report.txt"), "a") as file:
         file.write(msg)
@@ -222,7 +229,7 @@ def fail(msg: str) -> None:
 
 def main() -> None:
     """ main routine """
-    group = getenv("GROUP")
+    group = os.getenv("GROUP")
     if group == "":
         fail("ERROR: the GROUP environment variable is not set")
     sym_file_path = F"symbol_info/{group}.json"
@@ -234,20 +241,32 @@ def main() -> None:
         problems["errors"] = sym_errors
         if len(symbols) > 0:
             check_data_files(sym_file_path, symbols, problems)
-    
+
     # report warnings
-    if len(problems["missed_files"]) > 0:
-        warning = F'WARNING: the following symbols have no corresponding CSV files in the data folder: {", ".join(problems["missed_files"])}\n'
+    len_problems_missed_files = len(problems["missed_files"])
+    if len_problems_missed_files > 0:
+        if len_problems_missed_files > MAX_ERRORS_IN_MSG + THRESHOLD_ERR:
+            warning = F'WARNING: the following symbols have no corresponding CSV files in the data folder: {", ".join(problems["missed_files"][:MAX_ERRORS_IN_MSG])} and {len_problems_missed_files - MAX_ERRORS_IN_MSG} other CSV files.\n'
+        else:
+            warning = F'WARNING: the following symbols have no corresponding CSV files in the data folder: {", ".join(problems["missed_files"])}\n'
+
         if REPORTS_PATH is None:
-            print(warning)
+            print(YEL+warning+ENDC)
         else:
             with open(os.path.join(REPORTS_PATH, "warnings.txt"), "a") as file:
                 file.write(warning)
-    
+
     # report errors
-    if len(problems["errors"]) > 0:
-        problems_list = "\n ".join(problems["errors"])
-        fail(F'ERROR: the following issues were found in the repository files:\n {problems_list}\n')
+    len_problems_errors = len(problems["errors"])
+    if len_problems_errors > 0:
+        if len_problems_errors > MAX_ERRORS_IN_MSG + THRESHOLD_ERR:
+            problems_list = "\n ".join(problems["errors"][:MAX_ERRORS_IN_MSG])
+            error_msg = F'ERROR: the following issues were found in the repository files:\n {problems_list} and {len_problems_errors - MAX_ERRORS_IN_MSG} other errors. \n'
+        else:
+            problems_list = "\n ".join(problems["errors"])
+            error_msg = F'ERROR: the following issues were found in the repository files:\n {problems_list}\n'
+
+        fail(error_msg)
 
 
 if __name__ == "__main__":

validate_token.sh

@@ -1,37 +1,38 @@
+#!/bin/bash
 
 if [ -z "${ACTION_TOKEN}" ]
 then
-  echo "Failed to find ACTION_TOKEN. Make sure that ACTION_TOKEN is set in the repository secrets."
+  echo $(color_message "Failed to find ACTION_TOKEN. Make sure that ACTION_TOKEN is set in the repository secrets." $RED)
   exit 1
 fi
 
 RESP=$(echo "${ACTION_TOKEN}" | gh auth login --with-token 2>&1)
 if [ $? -ne 0 ]
 then
-    echo "Authorization failed. Make sure that your ACTION_TOKEN is valid and not expired."
+    echo $(color_message "Authorization failed. Make sure that your ACTION_TOKEN is valid and not expired." $RED)
     exit 1
 fi
 
 if  [[ "$RESP" == *"error"* ]];
 then
-  echo "Insufficient scope error. Provide ACTION_TOKEN with the repo, workflow, and admin:org scopes."
+  echo $(color_message "Insufficient scope error. Provide ACTION_TOKEN with the repo, workflow, and admin:org scopes." $RED)
   exit 1
 fi
 
 SCOPES=$(gh auth status  2>&1 | grep "scopes")
 valid=true
 if [[ "$SCOPES" != *"workflow"* ]]; then
-  echo "Insufficient scope error. Provide ACTION_TOKEN with the workflow scope."
+  echo echo $(color_message "Insufficient scope error. Provide ACTION_TOKEN with the workflow scope." $RED)
   valid=false
 fi
 
 if [[ "$SCOPES" != *"admin:org"* ]]; then
-  echo "Insufficient scope error. Provide ACTION_TOKEN with the admin:org scope."
+  echo $(color_message "Insufficient scope error. Provide ACTION_TOKEN with the admin:org scope." $RED)
   valid=false
 fi
 
 if ! $valid ; then
   exit 1
 fi
 
-echo "ACTION_TOKEN validation successful"
+echo $(color_message "ACTION_TOKEN validation successful" $GRN)