Refactoring with init (#78)

Refactoring with init

Add init files
Refacor scripts for better handling

Reviewed-by: Anton Sidelnikov
Reviewed-by: Vladimir Vshivkov

.dockerignore

@@ -0,0 +1,8 @@
+*.png
+__pycache__/
+*.pyc
+dashboards/
+.github/
+*.ini
+*.yaml
+*.md

Dockerfile

@@ -2,10 +2,10 @@ FROM registry.access.redhat.com/ubi9/python-311:1-72.1724040033
 
 WORKDIR /app
 
-COPY --chown=1001:0 *.py requirements.txt ./
+COPY --chown=1001:0 . .
 
 USER 1001
 
 RUN pip install --no-cache-dir -r requirements.txt
 
-CMD sleep 60
+CMD find . -name "*.py" -exec python {} \; && sleep 60

README.md

@@ -3,13 +3,19 @@ Eyes-On-Docs
 ************
 Scripts gathering HC3.0 PRs, issues and docs info from Github and Gitea. Scripts are works both for OTC and Swiss clouds in a one run. Data store in three Postgres databases: **_CSV_** is for a service tables and open PRs, failed Zuul PRs, issues and commits updates, **_ORPH_** is a dedicated DB special for orphan PRs
 
-1) **otc_services_dict.py:** service script gathering metadata for service, its full names, categories and types. Should be run first, since all of the following scripts are relay on it in terms of repo names, service titles and squad names.
-2) **gitea_info.py:** this script is using for open & orphan PRs data collecting
-3) **github_info.py:** add info regarding child PRs in Github
-4) **failed_zuul.py:** collecting info about PRs which checks in Zuul has been failed
-5) **open_issues.py:** this script gather info regarding all open issues both from Gitea and Gitnub
-6) **last_commit_info.py:** this script gather and calculate date of the last update of a prospective doc for all the services and types of docs (user manual, API reference and so on)
-7) **scheduler.py:** this script checking postgres for orphans, unattended issues and outdated docs, and send notifications to Zulip, via OTC Bot. Runs as cronjob (see 'system-config' repo)
+1) **eod_1_otc_services_dict.py:** service script gathering metadata for service, its full names, categories and types. 
+   Should be run first, since all of the following scripts are relay on it in terms of repo names, service titles and squad names.
+2) **eod_2_gitea_info.py:** this script is using for open & orphan PRs data collecting
+3) **eod_3_github_info.py:** add info regarding child PRs in Github
+4) **eod_4_failed_zuul.py:** collecting info about PRs which checks in Zuul has been failed
+5) **eod_5_open_issues.py:** this script gather info regarding all open issues both from Gitea and Gitnub
+6) **eod_6_last_commit_info.py:** this script gather and calculate date of the last update of a prospective doc for all 
+   the services and types of docs (user manual, API reference and so on)
+7) **eod_7_request_changes.py:** this script gather info about PRs where changes has been requested, from both 
+   sides: OTC squads and upstream team, which is generating documentation
+8) **eod_8_ecosystem_issue.py:** script for gathering issues for Eco Squad only
+9) **eod_9_scheduler.py:** this script checking postgres for orphans, unattended issues and outdated docs, and send 
+   notifications to Zulip, via OTC Bot. Runs as cronjob (see 'system-config' repo)
 Postgres database names, table names, Gitea & Github organization names and access tokens are store in environment variables.
 
 Notification schedule

config/__init__.py

@@ -0,0 +1,10 @@
+import logging
+
+from .classes import Database, EnvVariables, Timer
+
+
+def setup_logging():
+    logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+
+
+__all__ = ['EnvVariables', 'Database', 'Timer']

config/classes.py

@@ -0,0 +1,78 @@
+"""
+This script contains data classes for code reusing
+"""
+import logging
+import os
+import time
+
+import psycopg2
+
+
+class EnvVariables:
+    required_env_vars = [
+        "DB_HOST", "DB_PORT", "DB_CSV", "DB_USER", "DB_ORPH", "DB_ZUUL", "DB_PASSWORD", "GITEA_TOKEN", "GITHUB_TOKEN",
+        "GITHUB_FALLBACK_TOKEN"
+    ]
+
+    def __init__(self):
+        self.db_host = os.getenv("DB_HOST")
+        self.db_port = os.getenv("DB_PORT")
+        self.db_csv = os.getenv("DB_CSV")  # main postgres db, open PRs tables for public and hybrid clouds are stored
+        self.db_user = os.getenv("DB_USER")
+        self.db_orph = os.getenv("DB_ORPH")
+        self.db_zuul = os.getenv("DB_ZUUL")
+        self.db_password = os.getenv("DB_PASSWORD")
+        self.gitea_token = os.getenv("GITEA_TOKEN")
+        self.github_token = os.getenv("GITHUB_TOKEN")
+        self.github_fallback_token = os.getenv("GITHUB_FALLBACK_TOKEN")
+        self.api_key = os.getenv("OTC_BOT_API")
+        self.check_env_variables()
+
+    def check_env_variables(self):
+        for var in self.required_env_vars:
+            if os.getenv(var) is None:
+                raise Exception("Missing environment variable: %s" % var)
+
+
+class Database:
+    def __init__(self, env):
+        self.db_host = env.db_host
+        self.db_port = env.db_port
+        self.db_user = env.db_user
+        self.db_password = env.db_password
+
+    def connect_to_db(self, db_name):
+        logging.info("Connecting to Postgres (%s)...", db_name)
+        try:
+            return psycopg2.connect(
+                host=self.db_host,
+                port=self.db_port,
+                dbname=db_name,
+                user=self.db_user,
+                password=self.db_password
+            )
+        except psycopg2.Error as e:
+            logging.error("Connecting to Postgres: an error occurred while trying to connect to the database: %s", e)
+            return None
+
+
+class Timer:
+    def __init__(self):
+        self.start_time = None
+        self.end_time = None
+
+    def start(self):
+        self.start_time = time.time()
+
+    def stop(self):
+        self.end_time = time.time()
+        self.report()
+
+    def report(self):
+        if self.start_time and self.end_time:
+            execution_time = self.end_time - self.start_time
+            minutes, seconds = divmod(execution_time, 60)
+            logging.info(f"Script executed in {int(minutes)} minutes {int(seconds)} seconds! Let's go drink some beer "
+                         ":)")
+        else:
+            logging.error("Timer was not properly started or stopped")

main.py

@@ -0,0 +1,45 @@
+"""
+This script is an entry point for all other modules included in Eyes-on-Docs
+"""
+
+import argparse
+
+from scripts import (eod_1_otc_services_dict, eod_2_gitea_info, eod_3_github_info, eod_4_failed_zuul, eod_5_open_issues,
+                     eod_6_last_commit_info, eod_7_request_changes, eod_8_ecosystem_issues, eod_9_scheduler)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Eyes-on-Docs scripts run")
+    parser.add_argument('--eod1', action='store_true', help='OTC services dict')
+    parser.add_argument('--eod2', action='store_true', help='Gitea info')
+    parser.add_argument('--eod3', action='store_true', help='Github info')
+    parser.add_argument('--eod4', action='store_true', help='Failed Zuul')
+    parser.add_argument('--eod5', action='store_true', help='Open issues')
+    parser.add_argument('--eod6', action='store_true', help='Last commit info')
+    parser.add_argument('--eod7', action='store_true', help='Request changes')
+    parser.add_argument('--eod8', action='store_true', help='Ecosystem issues')
+    parser.add_argument('--eod9', action='store_true', help='Scheduler')
+    args = parser.parse_args()
+
+    if args.eod1:
+        eod_1_otc_services_dict.run()
+    if args.eod2:
+        eod_2_gitea_info.run()
+    if args.eod3:
+        eod_3_github_info.run()
+    if args.eod4:
+        eod_4_failed_zuul.run()
+    if args.eod5:
+        eod_5_open_issues.run()
+    if args.eod6:
+        eod_6_last_commit_info.run()
+    if args.eod7:
+        eod_7_request_changes.run()
+    if args.eod8:
+        eod_8_ecosystem_issues.run()
+    if args.eod9:
+        eod_9_scheduler.run()
+
+
+if __name__ == "__main__":
+    main()

1_otc_services_dict.py → scripts/eod_1_otc_services_dict.py

@@ -5,19 +5,12 @@
 
 import base64
 import logging
-import time
 
 import psycopg2
 import requests
 import yaml
 
-from classes import Database, EnvVariables
-
-logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
-
-start_time = time.time()
-
-logging.info("-------------------------OTC SERVICES DICT SCRIPT IS RUNNING-------------------------")
+from config import Database, EnvVariables, Timer, setup_logging
 
 BASE_URL = "https://gitea.eco.tsi-dev.otc-service.com/api/v1"
 
@@ -281,7 +274,14 @@ def main(base_dir, rtctable, doctable, styring_path):
     conn_csv.close()
 
 
-if __name__ == "__main__":
+def run():
+    timer = Timer()
+    timer.start()
+
+    setup_logging()
+
+    logging.info("-------------------------OTC SERVICES DICT SCRIPT IS RUNNING-------------------------")
+
     BASE_DIR_SWISS = "/repos/infra/otc-metadata-swiss/contents/"
     BASE_DIR_REGULAR = "/repos/infra/otc-metadata/contents/"
     STYRING_URL_REGULAR = "/repos/infra/gitstyring/contents/data/github/orgs/opentelekomcloud-docs/data.yaml?token="
@@ -297,7 +297,8 @@ def main(base_dir, rtctable, doctable, styring_path):
     conn_csv.commit()
     conn_csv.close()
 
-    end_time = time.time()
-    execution_time = end_time - start_time
-    minutes, seconds = divmod(execution_time, 60)
-    logging.info("Script executed in %s minutes %s seconds! Let's go drink some beer :)", int(minutes), int(seconds))
+    timer.stop()
+
+
+if __name__ == "__main__":
+    run()

2_gitea_info.py → scripts/eod_2_gitea_info.py

@@ -7,19 +7,12 @@
 import logging
 import pathlib
 import re
-import time
 
 import psycopg2
 import requests
 from github import Github
 
-from classes import Database, EnvVariables
-
-logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
-
-start_time = time.time()
-
-logging.info("-------------------------OPEN PRs SCRIPT IS RUNNING-------------------------")
+from config import Database, EnvVariables, Timer, setup_logging
 
 GITEA_API_ENDPOINT = "https://gitea.eco.tsi-dev.otc-service.com/api/v1"
 session = requests.Session()
@@ -498,7 +491,6 @@ def update_squad_and_title(cursors, conns, rtctable, opentable):
 
 
 def main(org, gh_org, rtctable, opentable, string, token):
-
     csv_erase(["proposalbot_prs.csv", "doc_exports_prs.csv", "orphaned_prs.csv"])
 
     conn_csv = database.connect_to_db(env_vars.db_csv)
@@ -538,7 +530,13 @@ def main(org, gh_org, rtctable, opentable, string, token):
         conn.close()
 
 
-if __name__ == "__main__":
+def run():
+    timer = Timer()
+    timer.start()
+
+    setup_logging()
+    logging.info("-------------------------OPEN PRs SCRIPT IS RUNNING-------------------------")
+
     RTC_TABLE = "repo_title_category"
     OPEN_TABLE = "open_prs"
     ORG_STRING = "docs"
@@ -561,7 +559,8 @@ def main(org, gh_org, rtctable, opentable, string, token):
         logging.info("Github operations successfully done!")
 
     csv_erase(["proposalbot_prs.csv", "doc_exports_prs.csv", "orphaned_prs.csv"])
-    end_time = time.time()
-    execution_time = end_time - start_time
-    minutes, seconds = divmod(execution_time, 60)
-    logging.info("Script executed in %s minutes %s seconds! Let's go drink some beer :)", int(minutes), int(seconds))
+    timer.stop()
+
+
+if __name__ == "__main__":
+    run()