diff --git a/pyproject.toml b/pyproject.toml index 2679062..ea7ca74 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "rda_python_common" -version = "1.0.55" +version = "1.0.56" authors = [ { name="Zaihua Ji", email="zji@ucar.edu" }, ] @@ -30,3 +30,6 @@ dependencies = [ pythonpath = [ "src" ] + +[project.scripts] +pgpassword = "rda_python_common.pgpassword:main" diff --git a/requirements.txt b/requirements.txt index a1482d1..51dccc2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ psycopg2-binary==2.9.10 pytest==8.3.5 rda-python-globus unidecode +hvac diff --git a/src/rda_python_common/PgDBI.py b/src/rda_python_common/PgDBI.py index baa3781..fca9eba 100644 --- a/src/rda_python_common/PgDBI.py +++ b/src/rda_python_common/PgDBI.py @@ -15,6 +15,7 @@ import os import re import time +import hvac from datetime import datetime import psycopg2 as PgSQL from psycopg2.extras import execute_values @@ -41,6 +42,7 @@ } DBPASS = {} +DBBAOS = {} # hard coded db names for given schema names DBNAMES = { @@ -103,6 +105,7 @@ def SETPGDBI(name, value): SETPGDBI("VWNAME", PGDBI['DEFSC']) SETPGDBI("VWPORT", 0) SETPGDBI("VWSOCK", '') +SETPGDBI("BAOURL", 'https://bao.k8s.ucar.edu/') PGDBI['DBSHOST'] = PgLOG.get_short_host(PGDBI['DBHOST']) PGDBI['DEFSHOST'] = PgLOG.get_short_host(PGDBI['DEFHOST']) @@ -2228,29 +2231,76 @@ def pgname(str, sign = None): def get_pgpass_password(): if PGDBI['PWNAME']: return PGDBI['PWNAME'] + pwname = get_baopassword() + if not pwname: pwname = get_pgpassword() + + return pwname + +def get_pgpassword(): + if not DBPASS: read_pgpass() dbport = str(PGDBI['DBPORT']) if PGDBI['DBPORT'] else '5432' pwname = DBPASS.get((PGDBI['DBSHOST'], dbport, PGDBI['DBNAME'], PGDBI['LNNAME'])) if not pwname: pwname = DBPASS.get((PGDBI['DBHOST'], dbport, PGDBI['DBNAME'], PGDBI['LNNAME'])) - return pwname +def get_baopassword(): + + dbname = PGDBI['DBNAME'] + if dbname not in DBBAOS: read_openbao() + return DBBAOS[dbname].get(PGDBI['LNNAME']) + # # Reads the .pgpass file and returns a dictionary of credentials. # def read_pgpass(): + pgpass = PgLOG.PGLOG['DSSHOME'] + '/.pgpass' + if not op.isfile(pgpass): pgpass = PgLOG.PGLOG['GDEXHOME'] + '/.pgpass' try: - with open(PgLOG.PGLOG['DSSHOME'] + '/.pgpass', "r") as f: - for line in f: - line = line.strip() - if not line or line.startswith("#"): continue - dbhost, dbport, dbname, lnname, pwname = line.split(":") - DBPASS[(dbhost, dbport, dbname, lnname)] = pwname - except FileNotFoundError: - with open(PgLOG.PGLOG['GDEXHOME'] + '/.pgpass', "r") as f: + with open(pgpass, "r") as f: for line in f: line = line.strip() if not line or line.startswith("#"): continue dbhost, dbport, dbname, lnname, pwname = line.split(":") DBPASS[(dbhost, dbport, dbname, lnname)] = pwname + except Exception as e: + PgLOG.pglog(str(e), PGDBI['ERRLOG']) + +# +# Reads OpenBao secrets and returns a dictionary of credentials. +# +def read_openbao(): + + dbname = PGDBI['DBNAME'] + DBBAOS[dbname] = {} + url = 'https://bao.k8s.ucar.edu/' + baopath = { + 'ivaddb' : 'gdex/pgdb03', + 'ispddb' : 'gdex/pgdb03', + 'default' : 'gdex/pgdb01' + } + dbpath = baopath[dbname] if dbname in baopath else baopath['default'] + client = hvac.Client(url=PGDBI.get('BAOURL')) + client.token = PgLOG.PGLOG.get('BAOTOKEN') + try: + read_response = client.secrets.kv.v2.read_secret_version( + path=dbpath, + mount_point='kv', + raise_on_deleted_version=False + ) + except Exception as e: + return PgLOG.pglog(str(e), PGDBI['ERRLOG']) + + baos = read_response['data']['data'] + for key in baos: + ms = re.match(r'^(\w*)pass(\w*)$', key) + if not ms: continue + baoname = None + pre = ms.group(1) + suf = ms.group(2) + if pre: + baoname = 'metadata' if pre == 'meta' else pre + elif suf == 'word': + baoname = 'postgres' + if baoname: DBBAOS[dbname][baoname] = baos[key] diff --git a/src/rda_python_common/PgLOG.py b/src/rda_python_common/PgLOG.py index af2ce08..8e19d98 100644 --- a/src/rda_python_common/PgLOG.py +++ b/src/rda_python_common/PgLOG.py @@ -1332,6 +1332,7 @@ def set_common_pglog(): sm = "/usr/sbin/sendmail" if valid_command(sm): SETPGLOG("EMLSEND", f"{sm} -t") # send email command SETPGLOG("DBGLEVEL", '') # debug level + SETPGLOG("BAOTOKEN", 's.lh2t2kDjrqs3V8y2BU2zOocT') # OpenBao token SETPGLOG("DBGPATH", PGLOG['DSSDBHM']+"/log") # path to debug log file SETPGLOG("OBJCTBKT", "gdex-data") # default Bucket on Object Store SETPGLOG("BACKUPEP", "gdex-quasar") # default Globus Endpoint on Quasar diff --git a/src/rda_python_common/pgpassword.py b/src/rda_python_common/pgpassword.py new file mode 100644 index 0000000..c24b598 --- /dev/null +++ b/src/rda_python_common/pgpassword.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +# +################################################################################## +# +# Title: pgpassword +# Author: Zaihua Ji, zji@ucar.edu +# Date: 2025-10-27 +# Purpose: python script to retrieve passwords for postgrsql login to connect a +# gdex database from inside an python application +# +# Github: https://github.com/NCAR/rda-python-common.git +# +################################################################################## + +import os +import sys +import re +import pwd +import hvac +from . import PgLOG +from . import PgDBI + +DBFLDS = { + 'd' : 'dbname', + 'c' : 'scname', + 'h' : 'dbhost', + 'p' : 'dbport', + 'u' : 'lnname' +} + +DBINFO = { + 'dbname' : "", + 'scname' : "", + 'lnname' : "", + 'dbhost' : "", + 'dbport' : 5432 +} + +# +# main function to excecute this script +# +def main(): + + permit = False + aname = 'pgpassword' + argv = sys.argv[1:] + opt = None + dohelp = True + dbopt = False + + for arg in argv: + if re.match(r'^-\w+$', arg): + opt = arg[1:] + elif opt: + if opt == 'l': + PgDBI.PGDBI['BAOURL'] = arg + elif opt == 'k': + PgDBI.PGDBI['BAOTOKEN'] = arg + elif opt in DBFLDS: + dbopt = True + DBINFO[DBFLDS[opt]] = arg + else: + PgLOG.pglog(arg + ": Unknown option", PgLOG.LGEREX) + dohelp = False + else: + PgLOG.pglog(arg + ": Value provided without option", PgLOG.LGEREX) + + if dohelp: + print("Usage: pgpassword [-l OpenBaoURL] [-k TokenName] [-d DBNAME] \\") + print(" [-c SCHEMA] [-u USName] [-h DBHOST] [-p DBPORT]") + print(" -l OpenBao URL to retrieve passwords") + print(" -k OpenBao Token Name to retrieve passwords") + print(" -d PostgreSQL Database Name") + print(" -c PostgreSQL Schema Name") + print(" -u PostgreSQL Login User Name") + print(" -h PostgreSQL Server Host Name") + print(" -p PostgreSQL Port Number") + sys.exit(0) + + if dbopt: + PgDBI.default_scinfo(DBINFO['dbname'], DBINFO['scname'], DBINFO['dbhost'], + DBINFO['lnname'], None, DBINFO['dbport']) + + pwname = PgDBI.get_baopassword() + if not pwname: pwname = PgDBI.get_pgpassword() + print(pwname) + sys.exit(0) + +# +# call main() to start program +# +if __name__ == "__main__": main()