Skip to content

Commit

Permalink
cut the strings
Browse files Browse the repository at this point in the history
  • Loading branch information
@ptichoid
ptichoid committed Jun 3, 2024
1 parent 8bb0331 commit 76b583c
Show file tree
Hide file tree
Showing 8 changed files with 109 additions and 62 deletions.
3 changes: 2 additions & 1 deletion ecosystem_issues.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,8 @@ def create_open_issues_table(conn, cur, table_name):
conn.commit()
logging.info(f"Table {table_name} has been created successfully")
except psycopg2.Error as e:
logging.error(f"Tables creating: an error occurred while trying to create a table {table_name} in the database {db_name}: {e}")
logging.error(f"Tables creating: an error occurred while trying to create a table {table_name} in the database \
{db_name}: {e}")


def insert_issue_data(conn, cur, table_name, repo, issue):
Expand Down
18 changes: 12 additions & 6 deletions failed_zuul.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@
db_host = os.getenv("DB_HOST")
db_port = os.getenv("DB_PORT")
db_name = os.getenv(
"DB_ZUUL") # here we're using dedicated postgres db 'zuul' since Failed Zuul PRs panel should be placed on a same dashboard such as Open PRs
"DB_ZUUL") # here we're using dedicated postgres db 'zuul' since Failed Zuul PRs panel should be placed on a same \
# dashboard such as Open PRs
db_user = os.getenv("DB_USER")
db_password = os.getenv("DB_PASSWORD")

Expand Down Expand Up @@ -87,7 +88,8 @@ def is_repo_empty(org, repo, gitea_token):
return True
return False
except requests.exceptions.HTTPError as e:
if e.response.status_code == 409: # Conflict error which might mean empty repo, skip this repo to avoid script hangs
if e.response.status_code == 409: # Conflict error which might mean empty repo, skip this repo to avoid script\
# hangs
logging.info(f"Repo {repo} is empty, skipping")
return True
logging.error(f"Check repo: an error occurred while trying to get commits for repo {repo}: {e}")
Expand Down Expand Up @@ -173,7 +175,8 @@ def get_f_pr_commits(org, repo, f_pr_number, gitea_token):

except requests.exceptions.RequestException as e:
logging.error(
f"Get failed PR commits: an error occurred while trying to get pull requests of {repo} repo for {org} org: {e}")
f"Get failed PR commits: an error occurred while trying to get pull requests of {repo} repo for {org} org: \
{e}")


def get_failed_prs(org, repo, gitea_token, conn_zuul, cur_zuul, table_name):
Expand All @@ -182,9 +185,11 @@ def get_failed_prs(org, repo, gitea_token, conn_zuul, cur_zuul, table_name):
if repo != "doc-exports":
page = 1
while True:
# logging.info(f"Fetching PRs for {repo}, page {page}...") # Debug print, uncomment in case of script hangs
# logging.info(f"Fetching PRs for {repo}, page {page}...") # Debug print, uncomment in case of script\
# hangs
repo_resp = session.get(
f"{gitea_api_endpoint}/repos/{org}/{repo}/pulls?state=open&page={page}&limit=1000&token={gitea_token}")
f"{gitea_api_endpoint}/repos/{org}/{repo}/pulls?state=open&page={page}&limit=1000&token=\
{gitea_token}")
pull_requests = []
if repo_resp.status_code == 200:
try:
Expand Down Expand Up @@ -213,7 +218,8 @@ def get_failed_prs(org, repo, gitea_token, conn_zuul, cur_zuul, table_name):
if all(item is not None for item in [zuul_url, status, created_at, days_passed]):
cur_zuul.execute(f"""
INSERT INTO public.{table_name}
("Service Name", "Failed PR Title", "Failed PR URL", "Squad", "Failed PR State", "Zuul URL", "Zuul Check Status", "Days Passed", "Parent PR Number")
("Service Name", "Failed PR Title", "Failed PR URL", "Squad", "Failed PR State"\
, "Zuul URL", "Zuul Check Status", "Days Passed", "Parent PR Number")
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
""",
(
Expand Down
46 changes: 31 additions & 15 deletions gitea_info.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,11 @@

db_host = os.getenv("DB_HOST")
db_port = os.getenv("DB_PORT")
db_csv = os.getenv("DB_CSV") # this is main postgres db, where open PRs tables for both public and hybrid clouds are stored
db_orph = os.getenv("DB_ORPH") # this is dedicated db for orphans PRs (for both clouds) tables. This is so because grafana dashboards query limitations
db_csv = os.getenv("DB_CSV") # this is main postgres db, where open PRs tables for both public and hybrid clouds are
# stored
db_orph = os.getenv(
"DB_ORPH") # this is dedicated db for orphans PRs (for both clouds) tables. This is so because grafana dashboards
# query limitations
db_user = os.getenv("DB_USER")
db_password = os.getenv("DB_PASSWORD")

Expand Down Expand Up @@ -87,7 +90,8 @@ def create_prs_table(conn_csv, cur_csv, table_name):
conn_csv.commit()
logging.info(f"Table {table_name} has been created successfully")
except psycopg2.Error as e:
logging.error(f"Tables creating: an error occurred while trying to create a table {table_name} in the database: {e}")
logging.error(f"Tables creating: an error occurred while trying to create a table {table_name} in the database:"
f" {e}")


def get_repos(org, cur_csv, gitea_token, rtc_table):
Expand Down Expand Up @@ -134,7 +138,8 @@ def get_repos(org, cur_csv, gitea_token, rtc_table):

def check_pull_requests_exist(org, repo):
try:
initial_resp = session.get(f"{gitea_api_endpoint}/repos/{org}/{repo}/pulls?state=all&limit=1&token={gitea_token}")
initial_resp = session.get(f"{gitea_api_endpoint}/repos/{org}/{repo}/pulls?state=all&limit=1&token=\
{gitea_token}")
initial_resp.raise_for_status()
pulls = json.loads(initial_resp.content.decode())
if not pulls:
Expand All @@ -161,7 +166,8 @@ def get_parent_pr(org, repo):
if path.exists() is False:
csv_2 = open("proposalbot_prs.csv", "w")
csv_writer = csv.writer(csv_2)
csv_writer.writerow(["Parent PR number", "Service Name", "Auto PR URL", "Auto PR State", "If merged", "Environment"])
csv_writer.writerow(["Parent PR number", "Service Name", "Auto PR URL", "Auto PR State", "If merged",
"Environment"])
else:
csv_2 = open("proposalbot_prs.csv", "a")
csv_writer = csv.writer(csv_2)
Expand All @@ -172,7 +178,8 @@ def get_parent_pr(org, repo):
page = 1
while True:
try:
repo_resp = session.get(f"{gitea_api_endpoint}/repos/{org}/{repo}/pulls?state=all&page={page}&limit=1000&token={gitea_token}")
repo_resp = session.get(f"{gitea_api_endpoint}/repos/{org}/{repo}/pulls?state=all&page={page}\
&limit=1000&token={gitea_token}")
repo_resp.raise_for_status()
except requests.exceptions.RequestException as e:
logging.error(f"Error occurred while trying to get repo pull requests: {e}")
Expand Down Expand Up @@ -248,7 +255,8 @@ def get_pull_requests(org, repo):
page = 1
while True:
try:
pull_requests_resp = session.get(f"{gitea_api_endpoint}/repos/{org}/{repo}/pulls?state={state}&page={page}&limit=50&token={gitea_token}")
pull_requests_resp = session.get(f"{gitea_api_endpoint}/repos/{org}/{repo}/pulls?state={state}&page=\
{page}&limit=50&token={gitea_token}")
pull_requests_resp.raise_for_status()
except requests.exceptions.RequestException as e:
logging.error(f"Child PRs: an error occurred while trying to get pull requests of {repo} repo: {e}")
Expand Down Expand Up @@ -403,12 +411,14 @@ def compare_csv_files(conn_csv, cur_csv, conn_orph, cur_orph, opentable):
try:
cur_orph.execute(f"""
INSERT INTO public.{opentable}
("Parent PR Number", "Service Name", "Squad", "Auto PR URL", "Auto PR State", "If merged", "Environment", "Parent PR State", "Parent PR merged")
("Parent PR Number", "Service Name", "Squad", "Auto PR URL", "Auto PR State", "If merged",
"Environment", "Parent PR State", "Parent PR merged")
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
""", tuple(pr1))
conn_orph.commit()
except Exception as e:
logging.error(f"Open and orphans for ORPHANS and {opentable}: an error occurred while inserting into the orphaned_prs table: {e}")
logging.error(f"Open and orphans for ORPHANS and {opentable}: an error occurred while inserting"
f" into the orphaned_prs table: {e}")

elif pr1[0] == pr2[0] and pr1[4] == pr2[3] == "open":
if pr1 not in open_prs:
Expand All @@ -417,12 +427,14 @@ def compare_csv_files(conn_csv, cur_csv, conn_orph, cur_orph, opentable):
try:
cur_csv.execute(f"""
INSERT INTO public.{opentable}
("Parent PR Number", "Service Name", "Squad", "Auto PR URL", "Auto PR State", "If merged", "Environment", "Parent PR State", "Parent PR merged")
("Parent PR Number", "Service Name", "Squad", "Auto PR URL", "Auto PR State", "If merged",
"Environment", "Parent PR State", "Parent PR merged")
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
""", tuple(pr1))
conn_csv.commit()
except Exception as e:
logging.error(f"Open and orphans for OPEN and {opentable}: an error occurred while inserting into the open_prs table: {e}")
logging.error(f"Open and orphans for OPEN and {opentable}: an error occurred while inserting "
f"into the open_prs table: {e}")


def gitea_pr_info(org, parent_pr_name):
Expand Down Expand Up @@ -465,10 +477,12 @@ def get_github_open_prs(github_org, conn_csv, cur_csv, opentable, string):
parent_pr_num, parent_pr_state, parent_pr_merged = gitea_pr_info(parent_api_name, string)
cur_csv.execute(
f"""
INSERT INTO {opentable} ("Parent PR Number", "Service Name", "Squad", "Auto PR URL", "Auto PR State", "If merged", "Environment", "Parent PR State", "Parent PR merged")
INSERT INTO {opentable} ("Parent PR Number", "Service Name", "Squad", "Auto PR URL",
"Auto PR State", "If merged", "Environment", "Parent PR State", "Parent PR merged")
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s);
""",
(parent_pr_num, name_service, squad, github_pr_url, auto_pr_state, merged, env, parent_pr_state, parent_pr_merged)
(parent_pr_num, name_service, squad, github_pr_url, auto_pr_state, merged, env,
parent_pr_state, parent_pr_merged)
)
conn_csv.commit()
except Exception as e:
Expand Down Expand Up @@ -556,12 +570,14 @@ def main(org, gh_org, rtctable, opentable, string, token):

try:
main(org_string, gh_org_string, rtc_table, open_table, org_string, github_token)
main(f"{org_string}-swiss", f"{gh_org_string}-swiss", f"{rtc_table}_swiss", f"{open_table}_swiss", f"{org_string}-swiss", github_token)
main(f"{org_string}-swiss", f"{gh_org_string}-swiss", f"{rtc_table}_swiss", f"{open_table}_swiss",
f"{org_string}-swiss", github_token)
done = True
except Exception as e:
logging.info(f"Error has been occurred: {e}")
main(org_string, gh_org_string, rtc_table, open_table, org_string, github_fallback_token)
main(f"{org_string}-swiss", f"{gh_org_string}-swiss", f"{rtc_table}_swiss", f"{open_table}_swiss", f"{org_string}-swiss", github_fallback_token)
main(f"{org_string}-swiss", f"{gh_org_string}-swiss", f"{rtc_table}_swiss", f"{open_table}_swiss",
f"{org_string}-swiss", github_fallback_token)
done = True
if done:
logging.info("Github operations successfully done!")
Expand Down
6 changes: 4 additions & 2 deletions github_info.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,8 @@ def extract_pull_links(cur, table_name):
pull_links = [row[0] for row in cur.fetchall()]
return pull_links
except Exception as e:
logging.info(f"Extracting pull links: an error occurred while extracting pull links from {table_name}: {str(e)}")
logging.info(f"Extracting pull links: an error occurred while extracting pull links from {table_name}: {str(e)}"
)


def get_auto_prs(gh_string, repo_name, access_token, pull_links):
Expand Down Expand Up @@ -112,7 +113,8 @@ def update_orphaned_prs(org_str, cur, conn, rows, auto_prs, table_name):
(state, merged, pr_id)
)
except Exception as e:
logging.info(f"Orphanes: an error occurred while updating orphaned PRs in the {table_name} table: {str(e)}")
logging.info(f"Orphanes: an error occurred while updating orphaned PRs in the {table_name} table: "
f"{str(e)}")

else:
continue
Expand Down
20 changes: 10 additions & 10 deletions last_commit_info.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,8 @@ def create_commits_table(conn, cur, table_name):
conn.commit()
logging.info(f"Table {table_name} has been created successfully")
except psycopg2.Error as e:
logging.error(f"Tables creating: an error occurred while trying to create a table {table_name} in the database: {e}")
logging.error(f"Tables creating: an error occurred while trying to create a table {table_name} "
f"in the database: {e}")


def get_last_commit_url(github_repo, path):
Expand All @@ -75,14 +76,16 @@ def get_last_commit_url(github_repo, path):
logging.debug(f"COMMIT--------------------------------------- {commit}")
files_changed = commit.files
if any(file.filename.endswith('.rst') for file in files_changed):
logging.debug(f"COMMIT URL AND DATE---------------------------- {commit.html_url} {commit.commit.author.date}")
logging.debug(f"COMMIT URL AND DATE---------------------------- {commit.html_url} "
f"{commit.commit.author.date}")
return commit.html_url, commit.commit.author.date # Return the commit URL and its date
return None, None


def get_last_commit(org, conn, cur, doctype, string, table_name):
logging.info(f"Gathering last commit info for {string}...")
exclude_repos = ["docsportal", "doc-exports", "docs_on_docs", ".github", "presentations", "sandbox", "security", "template", "content-delivery-network", "data-admin-service", "resource-template-service"]
exclude_repos = ["docsportal", "doc-exports", "docs_on_docs", ".github", "presentations", "sandbox", "security",
"template", "content-delivery-network", "data-admin-service", "resource-template-service"]
for repo in org.get_repos():

if repo.name in exclude_repos:
Expand All @@ -95,23 +98,19 @@ def get_last_commit(org, conn, cur, doctype, string, table_name):
path = doctype
last_commit_url, last_commit_date = get_last_commit_url(repo, path)
if last_commit_url and last_commit_date:
# logging.info("*************************************************************************new block of commit")
last_commit_url, _ = get_last_commit_url(repo, path)
# logging.info(f"last commit url------------------------------------------ {last_commit_url}")
formatted_commit_date = last_commit_date.strftime('%Y-%m-%d')
# logging.info(f"LAST COMMIT DATE-------------------------------------- {formatted_commit_date}")
now = datetime.utcnow()
# logging.info(f"NOW---------------------------------------- {now}")
duration = now - last_commit_date
duration_days = duration.days
# logging.info(f"DURATION DAYS______________________________________________ {duration_days}")
if doctype == "umn/source":
doc_type = "UMN"
else:
doc_type = "API"
service_name = repo.name
cur.execute(
f'INSERT INTO {table_name} ("Service Name", "Doc Type", "Last commit at", "Days passed", "Commit URL") VALUES (%s, %s, %s, %s, %s);',
f'INSERT INTO {table_name} ("Service Name", "Doc Type", "Last commit at", "Days passed", '
f'"Commit URL") VALUES (%s, %s, %s, %s, %s);',
(service_name, doc_type, formatted_commit_date, duration_days, last_commit_url,))
conn.commit()

Expand Down Expand Up @@ -180,7 +179,8 @@ def main(gorg, table_name, rtc, gh_str, token):
except Exception as e:
logging.info(f"Error has been occurred: {e}")
main(gh_org_str, commit_table, rtc_table, gh_org_str, github_fallback_token)
main(f"{gh_org_str}-swiss", f"{commit_table}_swiss", f"{rtc_table}_swiss", f"{gh_org_str}-swiss", github_fallback_token)
main(f"{gh_org_str}-swiss", f"{commit_table}_swiss", f"{rtc_table}_swiss", f"{gh_org_str}-swiss",
github_fallback_token)
done = True
if done:
logging.info("Github operations successfully done!")
Expand Down
Loading

0 comments on commit 76b583c

Please sign in to comment.