Skip to content

Commit

Permalink
United grafana scripts (#27)
Browse files Browse the repository at this point in the history 
United grafana scripts

adapt gitea_info and failed_zuul to both clouds

Reviewed-by: Vladimir Vshivkov
  • Loading branch information
@YustinaKvr
YustinaKvr authored Aug 21, 2023
1 parent f58a4aa commit 3f0d598
Show file tree
Hide file tree
Showing 2 changed files with 100 additions and 65 deletions.
100 changes: 64 additions & 36 deletions failed_zuul.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@

db_host = os.getenv("DB_HOST")
db_port = os.getenv("DB_PORT")
db_name = os.getenv("DB_CSV") # here we're using main postgres db since we don't need orphan PRs
db_name = os.getenv("DB_ZUUL") # here we're using dedicated postgres db 'zuul' since Failed Zuul PRs panel should be placed on a same dashboard such as Open PRs
db_csv = os.getenv("DB_CSV") # here rtc service table is located
db_user = os.getenv("DB_USER")
db_password = os.getenv("DB_PASSWORD")

Expand Down Expand Up @@ -48,9 +49,9 @@ def connect_to_db(db_name):
return None


def create_prs_table(conn, cur, table_name):
def create_prs_table(conn_zuul, cur_zuul, table_name):
try:
cur.execute(
cur_zuul.execute(
f'''CREATE TABLE IF NOT EXISTS {table_name} (
id SERIAL PRIMARY KEY,
"Service Name" VARCHAR(255),
Expand All @@ -65,7 +66,7 @@ def create_prs_table(conn, cur, table_name):
"Parent PR Number" INT
);'''
)
conn.commit()
conn_zuul.commit()
print(f"Table {table_name} has been created successfully")
except psycopg2.Error as e:
print(f"Create table: an error occurred while trying to create a table {table_name} in the database: {e}")
Expand Down Expand Up @@ -145,7 +146,8 @@ def get_f_pr_commits(org, repo, f_pr_number, gitea_token):
print(f"Get failed PR commits: an error occurred while trying to get pull requests of {repo} repo for {org} org: {e}")


def get_failed_prs(org, repo, gitea_token, conn, cur, table_name):
def get_failed_prs(org, repo, gitea_token, conn_zuul, cur_zuul, table_name):

try:
if repo != "doc-exports" and repo != "dsf":
page = 1
Expand Down Expand Up @@ -176,14 +178,15 @@ def get_failed_prs(org, repo, gitea_token, conn, cur, table_name):
zuul_url, status, created_at, days_passed = get_f_pr_commits(org, repo, f_pr_number, gitea_token)
try:
if all(item is not None for item in [zuul_url, status, created_at, days_passed]):
cur.execute(f"""
cur_zuul.execute(f"""
INSERT INTO public.{table_name}
("Service Name", "Failed PR Title", "Failed PR URL", "Squad", "Failed PR State", "Zuul URL", "Zuul Check Status", "Days Passed", "Parent PR Number")
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
""",
(service_name, title, f_pr_url, squad, f_pr_state, zuul_url, status, days_passed, f_par_pr_num)
)
conn.commit()
conn_zuul.commit()
except Exception as e:
print(f"Failed PRs: an error occurred while inserting into {table_name} table: {e}")
else:
Expand All @@ -196,56 +199,81 @@ def get_failed_prs(org, repo, gitea_token, conn, cur, table_name):
print('Failed PRs: an error occurred:', e)


def update_squad_and_title(conn, cur, table_name, rtc):
print(f"Updating squads and titles with {rtc}...")
def update_squad_and_title(cur_dict, conn_table, cur_table, rtctable, opentable):
print(f"Updating squads and titles in {opentable}...")
try:
cur.execute(f"SELECT * FROM {table_name};")
failed_prs_rows = cur.fetchall()
cur_table.execute(f"SELECT * FROM {opentable};")
failed_prs_rows = cur_table.fetchall()

for row in failed_prs_rows:
cur.execute(
f"""UPDATE {table_name}
SET "Service Name" = rtc."Title", "Squad" = rtc."Category"
FROM {rtc} AS rtc
WHERE {table_name}."Service Name" = rtc."Repository"
AND {table_name}.id = %s;""",
(row[0],)
)
cur.execute(
f"""UPDATE {table_name}
SET "Squad" = 'Other'
WHERE {table_name}."Service Name" IN ('doc-exports', 'docs_on_docs', 'docsportal')
AND {table_name}.id = %s;""",
(row[0],)
service_name_index = 1
id_index = 0

cur_dict.execute(
f"""SELECT "Title", "Category"
FROM {rtctable}
WHERE "Repository" = %s;""",
(row[service_name_index],)
)
conn.commit()
rtc_row = cur_dict.fetchone()

if rtc_row:
cur_table.execute(
f"""UPDATE {opentable}
SET "Service Name" = %s, "Squad" = %s
WHERE id = %s;""",
(rtc_row[0], rtc_row[1], row[id_index])
)

if row[service_name_index] in ('doc-exports', 'docs_on_docs', 'docsportal'):
cur_table.execute(
f"""UPDATE {opentable}
SET "Squad" = 'Other'
WHERE id = %s;""",
(row[id_index],)
)

conn_table.commit()

except Exception as e:
print(f"Error updating squad and title: {e}")
conn.rollback()
conn_table.rollback()


def main(org, table_name, rtc):
check_env_variables()

conn = connect_to_db(db_name)
cur = conn.cursor()
conn_zuul = connect_to_db(db_name)
cur_zuul = conn_zuul.cursor()
conn_csv = connect_to_db(db_csv)
cur_csv = conn_csv.cursor()

cur.execute(f"DROP TABLE IF EXISTS {table_name}")
conn.commit()
cur_csv.execute(f"DROP TABLE IF EXISTS {table_name}")
conn_csv.commit()

create_prs_table(conn, cur, table_name)
create_prs_table(conn_zuul, cur_zuul, table_name)

repos = get_repos(org, gitea_token)

print("Gathering PRs info...")
for repo in repos:
get_failed_prs(org, repo, gitea_token, conn, cur, table_name)
get_failed_prs(org, repo, gitea_token, conn_zuul, cur_zuul, table_name)

update_squad_and_title(conn, cur, table_name, rtc)
update_squad_and_title(cur_csv, conn_zuul, cur_zuul, rtc, table_name)

cur.close()
conn.close()
cur_csv.close()
conn_csv.close()
cur_zuul.close()
conn_zuul.close()


if __name__ == "__main__":
org_string = "docs"
failed_table = "open_prs"
rtc_table = "repo_title_category"

main(org_string, failed_table, rtc_table)
main(f"{org_string}-swiss", f"{failed_table}_swiss", f"{rtc_table}_swiss")


if __name__ == "__main__":
Expand Down
65 changes: 36 additions & 29 deletions gitea_info.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -440,33 +440,45 @@ def get_github_open_prs(github_org, conn_csv, cur_csv, opentable, string):
print('Github PRs: an error occurred:', e)


def update_squad_and_title(conn_csv, cur_csv, rtctable, opentable):
print("Updating squads and titles...")
def update_squad_and_title(cur_csv, conn_table, cur_table, rtctable, opentable):
print(f"Updating squads and titles in {opentable}...")
try:
cur_csv.execute("SELECT * FROM open_prs;")
open_issues_rows = cur_csv.fetchall()
cur_table.execute(f"SELECT * FROM {opentable};") # Use Zuul cursor here
failed_prs_rows = cur_table.fetchall()

for row in failed_prs_rows:
service_name_index = 1
id_index = 0

for row in open_issues_rows:
cur_csv.execute(
f"""UPDATE {opentable}
SET "Service Name" = rtc."Title", "Squad" = rtc."Category"
FROM {rtctable} AS rtc
WHERE {opentable}."Service Name" = rtc."Repository"
AND {opentable}.id = %s;""",
(row[0],)
)
cur_csv.execute(
f"""UPDATE {opentable}
SET "Squad" = 'Other'
WHERE {opentable}."Service Name" IN ('doc-exports', 'docs_on_docs', 'docsportal')
AND {opentable}.id = %s;""",
(row[0],)
f"""SELECT "Title", "Category"
FROM {rtctable}
WHERE "Repository" = %s;""",
(str(row[service_name_index]),)
)
conn_csv.commit()
rtc_row = cur_csv.fetchone()

if rtc_row:
cur_table.execute(
f"""UPDATE {opentable}
SET "Service Name" = %s, "Squad" = %s
WHERE id = %s;""",
(rtc_row[0], rtc_row[1], row[id_index])
)

if row[service_name_index] in ('doc-exports', 'docs_on_docs', 'docsportal'):
cur_table.execute(
f"""UPDATE {opentable}
SET "Squad" = 'Other'
WHERE id = %s;""",
(row[id_index],)
)

conn_table.commit()

except Exception as e:
print(f"Error updating squad and title: {e}")
conn_csv.rollback()
conn_table.rollback()


def main(org, gh_org, rtctable, opentable, string):
Expand All @@ -475,7 +487,8 @@ def main(org, gh_org, rtctable, opentable, string):

conn_csv = connect_to_db(db_csv)
cur_csv = conn_csv.cursor()

conn_orph = connect_to_db(db_orph)
cur_orph = conn_orph.cursor()
g = Github(github_token)
github_org = g.get_organization(gh_org)

Expand All @@ -488,29 +501,23 @@ def main(org, gh_org, rtctable, opentable, string):
print("Gathering parent PRs...")
for repo in repos:
get_parent_pr(org, repo)

get_pull_requests(org, "doc-exports")

update_service_titles(cur_csv, rtctable)
add_squad_column(cur_csv, rtctable)

conn_orph = connect_to_db(db_orph)
cur_orph = conn_orph.cursor()

cur_orph.execute(f"DROP TABLE IF EXISTS {opentable}")
conn_orph.commit()

create_prs_table(conn_orph, cur_orph, opentable)
compare_csv_files(conn_csv, cur_csv, conn_orph, cur_orph, opentable)

csv_erase()

get_github_open_prs(github_org, conn_csv, cur_csv, opentable, string)
update_squad_and_title(conn_csv, cur_csv, rtctable, opentable)

update_squad_and_title(cur_csv, conn_orph, cur_orph, rtctable, opentable)

cur_csv.close()
conn_csv.close()

cur_orph.close()
conn_orph.close()

Expand Down

0 comments on commit 3f0d598

Please sign in to comment.