diff --git a/pyproject.toml b/pyproject.toml index cc99795..318ac3a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "rda_python_common" -version = "2.0.12" +version = "2.0.14" authors = [ { name="Zaihua Ji", email="zji@ucar.edu" }, ] diff --git a/src/rda_python_common/pg_cmd.py b/src/rda_python_common/pg_cmd.py index db1dfaa..39c2638 100644 --- a/src/rda_python_common/pg_cmd.py +++ b/src/rda_python_common/pg_cmd.py @@ -1,18 +1,13 @@ -# ############################################################################### -# -# Title : pg_cmd.py -# Author : Zaihua Ji, zji@ucar.edu -# Date : 08/25/2020 +# Title: pg_cmd.py +# Author: Zaihua Ji, zji@ucar.edu +# Date: 08/25/2020 # 2025-01-10 transferred to package rda_python_common from # https://github.com/NCAR/rda-shared-libraries.git -# Purpose : python library module for functions to record commands for delayed +# Purpose: python library module for functions to record commands for delayed # mode or command recovery -# -# Github : https://github.com/NCAR/rda-python-common.git -# +# Github: https://github.com/NCAR/rda-python-common.git ############################################################################### -# import os import re import sys @@ -25,18 +20,18 @@ def __init__(self): super().__init__() # initialize parent class # cached dscheck info self.DSCHK = {} - self.BOPTIONS = {"hostname" : None, "qoptions" : None, "modules" : None, "environments" : None} + self.BOPTIONS = {"hostname": None, "qoptions": None, "modules": None, "environments": None} self.BFIELDS = ', '.join(self.BOPTIONS) self.TRYLMTS = { - 'dsquasar' : 3, - 'dsarch' : 2, - 'default' : 1 + 'dsquasar': 3, + 'dsarch': 2, + 'default': 1 } self.DLYPTN = r'(^|\s)-(d|BP|BatchProcess|DelayedMode)(\s|$)' self.DLYOPT = { - 'dsarch' : ' -d', - 'dsupdt' : ' -d', - 'dsrqst' : ' -d' + 'dsarch': ' -d', + 'dsupdt': ' -d', + 'dsrqst': ' -d' } # params: dict array holding option values @@ -381,7 +376,7 @@ def get_dsrqst_counts(self, pgchk, logact = 0): fcnt = pgrec['fcount'] else: fcnt = 0 - pgrec = {'fcount' : 0} + pgrec = {'fcount': 0} if not fcnt: fcnt = self.pgget("wfrqst", "", cnd, logact) if fcnt and fcount != fcnt: fcount = fcnt if fcount: @@ -402,14 +397,14 @@ def get_dsrqst_counts(self, pgchk, logact = 0): # set dscheck fcount def set_dscheck_fcount(self, count, logact = 0): - record = {'fcount' : count, 'chktime' : int(time.time())} + record = {'fcount': count, 'chktime': int(time.time())} self.pgupdt("dscheck", record, self.DSCHK['chkcnd'], logact) self.DSCHK['fcount'] = count return self.DSCHK['dcount'] # return Done count # set dscheck dcount def set_dscheck_dcount(self, count, size, logact = 0): - record = {'dcount' : count, 'size' : size, 'chktime' : int(time.time())} + record = {'dcount': count, 'size': size, 'chktime': int(time.time())} self.pgupdt("dscheck", record, self.DSCHK['chkcnd'], logact) self.DSCHK['dcount'] = count self.DSCHK['size'] = size @@ -444,7 +439,7 @@ def record_dscheck_status(self, stat, logact = 0): if pgrec['pid'] != cpid or pgrec['lockhost'] != chost: return 0 # update dscheck status only if it is still locked by the current process - record = {'status' : stat, 'chktime' : int(time.time()), 'pid' : 0} + record = {'status': stat, 'chktime': int(time.time()), 'pid': 0} return self.pgupdt("dscheck", record, self.DSCHK['chkcnd'], logact) # get the number of tries to execute for a given cmd under dscheck control diff --git a/src/rda_python_common/pg_dbi.py b/src/rda_python_common/pg_dbi.py index 4080e97..e89a606 100644 --- a/src/rda_python_common/pg_dbi.py +++ b/src/rda_python_common/pg_dbi.py @@ -1,18 +1,13 @@ -# ############################################################################### -# -# Title : pg_dbi.py -- PostgreSQL DataBase Interface -# Author : Zaihua Ji, zji@ucar.edu -# Date : 06/07/2022 +# Title: pg_dbi.py -- PostgreSQL DataBase Interface +# Author: Zaihua Ji, zji@ucar.edu +# Date: 06/07/2022 # 2025-01-10 transferred to package rda_python_common from # https://github.com/NCAR/rda-shared-libraries.git # 2025-11-24 convert to class PgDBI -# Purpose : Python library module to handle query and manipulate PostgreSQL database -# -# Github : https://github.com/NCAR/rda-python-common.git -# +# Purpose: Python library module to handle query and manipulate PostgreSQL database +# Github: https://github.com/NCAR/rda-python-common.git ############################################################################### - import os import re import time @@ -49,22 +44,22 @@ def __init__(self): self.PGSIGNS = ['!', '<', '>', '<>'] self.CHCODE = 1042 # hard coded db ports for dbnames - self.DBPORTS = {'default' : 0} + self.DBPORTS = {'default': 0} self.DBPASS = {} self.DBBAOS = {} # hard coded db names for given schema names self.DBNAMES = { - 'ivaddb' : 'ivaddb', - 'cntldb' : 'ivaddb', - 'cdmsdb' : 'ivaddb', - 'ispddb' : 'ispddb', - 'obsua' : 'upadb', - 'default' : 'rdadb', + 'ivaddb': 'ivaddb', + 'cntldb': 'ivaddb', + 'cdmsdb': 'ivaddb', + 'ispddb': 'ispddb', + 'obsua': 'upadb', + 'default': 'rdadb', } # hard coded socket paths for machine_dbnames - self.DBSOCKS = {'default' : ''} + self.DBSOCKS = {'default': ''} # home path for check db on alter host - self.VIEWHOMES = {'default' : self.PGLOG['DSSDBHM']} + self.VIEWHOMES = {'default': self.PGLOG['DSSDBHM']} # add more to the list if used for names self.PGRES = ['end', 'window'] self.SETPGDBI('DEFDB', 'rdadb') @@ -397,8 +392,8 @@ def pgconnect(self, reconnect = 0, pgcnt = 0, autocommit = True): elif reconnect: reconnect = 0 # initial connection while True: - config = {'database' : self.PGDBI['DBNAME'], - 'user' : self.PGDBI['LNNAME']} + config = {'database': self.PGDBI['DBNAME'], + 'user': self.PGDBI['LNNAME']} if self.PGDBI['DBSHOST'] == self.PGLOG['HOSTNAME']: config['host'] = 'localhost' else: @@ -748,8 +743,8 @@ def pgmget(self, tablenames, fields, condition = None, logact = None): # tablenames: comma deliminated string of one or more tables # fields: comma deliminated string of one or more field names, - # cnddict: condition dict with field names : values - # return a dict(field names : values) upon success + # cnddict: condition dict with field names: values + # return a dict(field names: values) upon success # retrieve one records from tablenames condition dict def pghget(self, tablenames, fields, cnddict, logact = None): if logact is None: logact = self.PGDBI['ERRLOG'] @@ -794,8 +789,8 @@ def pghget(self, tablenames, fields, cnddict, logact = None): # tablenames: comma deliminated string of one or more tables # fields: comma deliminated string of one or more field names, - # cnddicts: condition dict with field names : value lists - # return a dict(field names : value lists) upon success + # cnddicts: condition dict with field names: value lists + # return a dict(field names: value lists) upon success # retrieve multiple records from tablenames for condition dict def pgmhget(self, tablenames, fields, cnddicts, logact = None): if logact is None: logact = self.PGDBI['ERRLOG'] @@ -870,7 +865,7 @@ def prepare_update(self, tablename, fields, condition = None, cndflds = None): # update one or multiple rows in tablename # tablename: update for one table name each call - # record: dict with field names : values + # record: dict with field names: values # condition: update conditions for where clause) # return number of rows undated upon success def pgupdt(self, tablename, record, condition, logact = None): @@ -904,8 +899,8 @@ def pgupdt(self, tablename, record, condition, logact = None): # update one or multiple records in tablename # tablename: update for one table name each call - # record: update values, dict with field names : values - # cnddict: condition dict with field names : values + # record: update values, dict with field names: values + # cnddict: condition dict with field names: values # return number of records updated upon success def pghupdt(self, tablename, record, cnddict, logact = None): if logact is None: logact = self.PGDBI['ERRLOG'] @@ -939,8 +934,8 @@ def pghupdt(self, tablename, record, cnddict, logact = None): # update multiple records in tablename # tablename: update for one table name each call - # records: update values, dict with field names : value lists - # cnddicts: condition dict with field names : value lists + # records: update values, dict with field names: value lists + # cnddicts: condition dict with field names: value lists # return number of records updated upon success def pgmupdt(self, tablename, records, cnddicts, logact = None): if logact is None: logact = self.PGDBI['ERRLOG'] @@ -1021,7 +1016,7 @@ def pgdel(self, tablename, condition, logact = None): # delete one or mutiple records in tablename according condition # tablename: delete for one table name each call - # cndict: delete condition dict for names : values + # cndict: delete condition dict for names: values # return number of records deleted upon success def pghdel(self, tablename, cnddict, logact = None): if logact is None: logact = self.PGDBI['ERRLOG'] @@ -1052,7 +1047,7 @@ def pghdel(self, tablename, cnddict, logact = None): # delete mutiple records in tablename according condition # tablename: delete for one table name each call - # cndicts: delete condition dict for names : value lists + # cndicts: delete condition dict for names: value lists # return number of records deleted upon success def pgmdel(self, tablename, cnddicts, logact = None): if logact is None: logact = self.PGDBI['ERRLOG'] @@ -1157,7 +1152,7 @@ def check_user_uid(self, userno, date = None): pgrec = self.pgget("dssdb.user", "uid", "userno = {}".format(userno), self.PGDBI['ERRLOG']) if pgrec: return pgrec['uid'] pgrec = self.ucar_user_info(userno) - if not pgrec: pgrec = {'userno' : userno, 'stat_flag' : 'M'} + if not pgrec: pgrec = {'userno': userno, 'stat_flag': 'M'} uid = self.pgadd("dssdb.user", pgrec, (self.PGDBI['EXITLG']|self.AUTOID)) if uid: self.pglog("{}: Scientist ID Added as user.uid = {}".format(userno, uid), self.LGWNEM) return uid @@ -1179,7 +1174,7 @@ def get_user_uid(self, logname, date = None): pgrec = self.pgget("dssdb.user", "uid", "logname = '{}'".format(logname), self.PGDBI['ERRLOG']) if pgrec: return pgrec['uid'] pgrec = self.ucar_user_info(0, logname) - if not pgrec: pgrec = {'logname' : logname, 'stat_flag' : 'M'} + if not pgrec: pgrec = {'logname': logname, 'stat_flag': 'M'} uid = self.pgadd("dssdb.user", pgrec, (self.PGDBI['EXITLG']|self.AUTOID)) if uid: self.pglog("{}: UCAR Login Name Added as user.uid = {}".format(logname, uid), self.LGWNEM) return uid @@ -1187,18 +1182,18 @@ def get_user_uid(self, logname, date = None): # get ucar user info for given userno (scientist number) or logname (Ucar login) def ucar_user_info(self, userno, logname = None): matches = { - 'upid' : "upid", - 'uid' : "userno", - 'username' : "logname", - 'lastName' : "lstname", - 'firstName' : "fstname", - 'active' : "stat_flag", - 'internalOrg' : "division", - 'externalOrg' : "org_name", - 'country' : "country", - 'forwardEmail' : "email", - 'email' : "ucaremail", - 'phone' : "phoneno" + 'upid': "upid", + 'uid': "userno", + 'username': "logname", + 'lastName': "lstname", + 'firstName': "fstname", + 'active': "stat_flag", + 'internalOrg': "division", + 'externalOrg': "org_name", + 'country': "country", + 'forwardEmail': "email", + 'email': "ucaremail", + 'phone': "phoneno" } buf = self.pgsystem("pgperson " + ("-uid {}".format(userno) if userno else "-username {}".format(logname)), self.LOGWRN, 20) if not buf: return None @@ -1250,13 +1245,13 @@ def ucar_user_info(self, userno, logname = None): # set country code for given coutry name or email address def set_country_code(self, email, country = None): codes = { - 'CHINA' : "P.R.CHINA", - 'ENGLAND' : "UNITED.KINGDOM", - 'FR' : "FRANCE", - 'KOREA' : "SOUTH.KOREA", - 'USSR' : "RUSSIA", - 'US' : "UNITED.STATES", - 'U.S.A.' : "UNITED.STATES" + 'CHINA': "P.R.CHINA", + 'ENGLAND': "UNITED.KINGDOM", + 'FR': "FRANCE", + 'KOREA': "SOUTH.KOREA", + 'USSR': "RUSSIA", + 'US': "UNITED.STATES", + 'U.S.A.': "UNITED.STATES" } if country: country = country.upper() @@ -1284,7 +1279,7 @@ def check_wuser_wuid(self, email, date = None): pgrec = self.pgget("wuser", "wuid", emcond, self.LOGERR) if pgrec: return pgrec['wuid'] # now add one in - record = {'email' : email} + record = {'email': email} # check again if a ruser is on file pgrec = self.pgget("ruser", "*", emcond + " AND end_date IS NULL", self.PGDBI['ERRLOG']) if not pgrec: pgrec = self.pgget("ruser", "*", emcond, self.PGDBI['ERRLOG']) @@ -1530,11 +1525,11 @@ def build_customized_email(self, table, field, condition, subject, logact = 0): # email: full user email address # get user real name from table ruser for a given email address - # opts == 1 : include email - # opts == 2 : include org_type - # opts == 4 : include country - # opts == 8 : include valid_email - # opts == 16 : include org + # opts == 1: include email + # opts == 2: include org_type + # opts == 4: include country + # opts == 8: include valid_email + # opts == 16: include org def get_ruser_names(self, email, opts = 0, date = None): fields = "lname lstname, fname fstname" if opts&1: fields += ", email" @@ -1859,9 +1854,9 @@ def read_openbao(self): self.DBBAOS[dbname] = {} url = 'https://bao.k8s.ucar.edu/' baopath = { - 'ivaddb' : 'gdex/pgdb03', - 'ispddb' : 'gdex/pgdb03', - 'default' : 'gdex/pgdb01' + 'ivaddb': 'gdex/pgdb03', + 'ispddb': 'gdex/pgdb03', + 'default': 'gdex/pgdb01' } dbpath = baopath[dbname] if dbname in baopath else baopath['default'] client = hvac.Client(url=self.PGDBI.get('BAOURL')) diff --git a/src/rda_python_common/pg_file.py b/src/rda_python_common/pg_file.py index 25e2cb1..eac0111 100644 --- a/src/rda_python_common/pg_file.py +++ b/src/rda_python_common/pg_file.py @@ -1,19 +1,14 @@ -# ############################################################################### -# -# Title : self.py -# Author : Zaihua Ji, zji@ucar.edu -# Date : 08/05/2020 +# Title: pg_file.py +# Author: Zaihua Ji, zji@ucar.edu +# Date: 08/05/2020 # 2025-01-10 transferred to package rda_python_common from # https://github.com/NCAR/rda-shared-libraries.git # 2025-12-01 convert to class PgFile -# Purpose : python library module to copy, move and delete data files locally +# Purpose: python library module to copy, move and delete data files locally # and remotely -# -# Github : https://github.com/NCAR/rda-python-common.git -# +# Github: https://github.com/NCAR/rda-python-common.git ############################################################################### -# import sys import os from os import path as op @@ -39,24 +34,24 @@ def __init__(self): super().__init__() # initialize parent class self.PGCMPS = { # extension Compress Uncompress ArchiveFormat - 'Z' : ['compress -f', 'uncompress -f', 'Z'], - 'zip' : ['zip', 'unzip', 'ZIP'], - 'gz' : ['gzip', 'gunzip', 'GZ'], - 'xz' : ['xz', 'unxz', 'XZ'], - 'bz2' : ['bzip2', 'bunzip2', 'BZ2'] + 'Z': ['compress -f', 'uncompress -f', 'Z'], + 'zip': ['zip', 'unzip', 'ZIP'], + 'gz': ['gzip', 'gunzip', 'GZ'], + 'xz': ['xz', 'unxz', 'XZ'], + 'bz2': ['bzip2', 'bunzip2', 'BZ2'] } self.CMPSTR = '|'.join(self.PGCMPS) self.PGTARS = { # extension Packing Unpacking ArchiveFormat - 'tar' : ['tar -cvf', 'tar -xvf', 'TAR'], - 'tar.Z' : ['tar -Zcvf', 'tar -xvf', 'TAR.Z'], - 'zip' : ['zip -v', 'unzip -v', 'ZIP'], - 'tgz' : ['tar -zcvf', 'tar -xvf', 'TGZ'], - 'tar.gz' : ['tar -zcvf', 'tar -xvf', 'TAR.GZ'], - 'txz' : ['tar -cvJf', 'tar -xvf', 'TXZ'], - 'tar.xz' : ['tar -cvJf', 'tar -xvf', 'TAR.XZ'], - 'tbz2' : ['tar -cvjf', 'tar -xvf', 'TBZ2'], - 'tar.bz2' : ['tar -cvjf', 'tar -xvf', 'TAR.BZ2'] + 'tar': ['tar -cvf', 'tar -xvf', 'TAR'], + 'tar.Z': ['tar -Zcvf', 'tar -xvf', 'TAR.Z'], + 'zip': ['zip -v', 'unzip -v', 'ZIP'], + 'tgz': ['tar -zcvf', 'tar -xvf', 'TGZ'], + 'tar.gz': ['tar -zcvf', 'tar -xvf', 'TAR.GZ'], + 'txz': ['tar -cvJf', 'tar -xvf', 'TXZ'], + 'tar.xz': ['tar -cvJf', 'tar -xvf', 'TAR.XZ'], + 'tbz2': ['tar -cvjf', 'tar -xvf', 'TBZ2'], + 'tar.bz2': ['tar -cvjf', 'tar -xvf', 'TAR.BZ2'] } self.TARSTR = '|'.join(self.PGTARS) self.DELDIRS = {} @@ -70,42 +65,42 @@ def __init__(self): self.DIRLVLS = 0 self.BFILES = {} # cache backup file names and dates for each bid # record how many errors happen for working with HPSS, local or remote machines - self.ECNTS = {'D' : 0, 'H' : 0, 'L' : 0, 'R' : 0, 'O' : 0, 'B' : 0} + self.ECNTS = {'D': 0, 'H': 0, 'L': 0, 'R': 0, 'O': 0, 'B': 0} # up limits for how many continuing errors allowed - self.ELMTS = {'D' : 20, 'H' : 20, 'L' : 20, 'R' : 20, 'O' : 10, 'B' : 10} + self.ELMTS = {'D': 20, 'H': 20, 'L': 20, 'R': 20, 'O': 10, 'B': 10} # down storage hostnames & paths self.DHOSTS = { - 'G' : self.PGLOG['GPFSNAME'], - 'O' : self.OHOST, - 'B' : self.BHOST, - 'D' : self.DHOST + 'G': self.PGLOG['GPFSNAME'], + 'O': self.OHOST, + 'B': self.BHOST, + 'D': self.DHOST } self.DPATHS = { - 'G' : self.PGLOG['DSSDATA'], - 'O' : self.PGLOG['OBJCTBKT'], - 'B' : '/' + self.PGLOG['DEFDSID'], # backup globus endpoint - 'D' : '/' + self.PGLOG['DEFDSID'] # disaster recovery globus endpoint + 'G': self.PGLOG['DSSDATA'], + 'O': self.PGLOG['OBJCTBKT'], + 'B': '/' + self.PGLOG['DEFDSID'], # backup globus endpoint + 'D': '/' + self.PGLOG['DEFDSID'] # disaster recovery globus endpoint } self.QSTATS = { - 'A' : 'ACTIVE', - 'I' : 'INACTIVE', - 'S' : 'SUCCEEDED', - 'F' : 'FAILED', + 'A': 'ACTIVE', + 'I': 'INACTIVE', + 'S': 'SUCCEEDED', + 'F': 'FAILED', } self.QPOINTS = { - 'L' : 'gdex-glade', - 'B' : 'gdex-quasar', - 'D' : 'gdex-quasar-drdata' + 'L': 'gdex-glade', + 'B': 'gdex-quasar', + 'D': 'gdex-quasar-drdata' } self.QHOSTS = { - 'gdex-glade' : self.LHOST, - 'gdex-quasar' : self.BHOST, - 'gdex-quasar-drdata' : self.DHOST + 'gdex-glade': self.LHOST, + 'gdex-quasar': self.BHOST, + 'gdex-quasar-drdata': self.DHOST } self.ENDPOINTS = { - 'gdex-glade' : "NCAR GDEX GLADE", - 'gdex-quasar' : "NCAR GDEX Quasar", - 'gdex-quasar-drdata' : "NCAR GDEX Quasar DRDATA" + 'gdex-glade': "NCAR GDEX GLADE", + 'gdex-quasar': "NCAR GDEX Quasar", + 'gdex-quasar-drdata': "NCAR GDEX Quasar DRDATA" } # reset the up limit for a specified error type @@ -339,7 +334,7 @@ def endpoint_copy_endpoint(self, tofile, fromfile, topoint, frompoint, logact = # submit a globus task and return a task id def submit_globus_task(self, cmd, endpoint, logact = 0, qstr = None): - task = {'id' : None, 'stat' : 'U'} + task = {'id': None, 'stat': 'U'} loop = reset = 0 while (loop-reset) < 2: buf = self.pgsystem(cmd, logact, self.CMDGLB, qstr) @@ -1465,7 +1460,7 @@ def check_object_path(self, path, bucket = None, logact = 0): # object store function to get file stat def object_file_stat(self, hash, uhash, opt): - info = {'isfile' : 1, 'data_size' : int(hash['Size']), 'fname' : op.basename(hash['Key'])} + info = {'isfile': 1, 'data_size': int(hash['Size']), 'fname': op.basename(hash['Key'])} if not opt: return info if opt&17: ms = re.match(r'^(\d+-\d+-\d+)\s+(\d+:\d+:\d+)', hash['LastModified']) diff --git a/src/rda_python_common/pg_lock.py b/src/rda_python_common/pg_lock.py index 4a45ae7..b2607c9 100644 --- a/src/rda_python_common/pg_lock.py +++ b/src/rda_python_common/pg_lock.py @@ -1,18 +1,13 @@ -# ############################################################################### -# -# Title : pg_lock.py -# Author : Zaihua Ji, zji@ucar.edu -# Date : 08/118/2020 +# Title: pg_lock.py +# Author: Zaihua Ji, zji@ucar.edu +# Date: 08/118/2020 # 2025-01-10 transferred to package rda_python_common from # https://github.com/NCAR/rda-shared-libraries.git # 2025-12-01 convert to class PgLock -# Purpose : python library module for functions to lock RDADB records -# -# Github : https://github.com/NCAR/rda-python-common.git -# +# Purpose: python library module for functions to lock RDADB records +# Github: https://github.com/NCAR/rda-python-common.git ############################################################################### -# import re import time from .pg_file import PgFile @@ -20,9 +15,8 @@ class PgLock(PgFile): def __init__(self): - super().__init__() # initialize parent class - self.DOLOCKS = {-2 : 'Force Unlock', -1 : 'Unlock', 0 : 'Unlock', 1 : 'Relock', 2 : 'Force Relock'} + self.DOLOCKS = {-2: 'Force Unlock', -1: 'Unlock', 0: 'Unlock', 1: 'Relock', 2: 'Force Relock'} def end_db_transaction(self, idx): if idx > 0: diff --git a/src/rda_python_common/pg_log.py b/src/rda_python_common/pg_log.py index c119a70..7061e6d 100644 --- a/src/rda_python_common/pg_log.py +++ b/src/rda_python_common/pg_log.py @@ -1,20 +1,15 @@ -# ############################################################################### -# -# Title : pg_log.py -- Module for logging messages. -# Author : Zaihua Ji, zji@ucar.edu -# Date : 03/02/2016 +# Title: pg_log.py -- Module for logging messages. +# Author: Zaihua Ji, zji@ucar.edu +# Date: 03/02/2016 # 2025-01-10 transferred to package rda_python_common from # https://github.com/NCAR/rda-shared-libraries.git # 2025-11-20 convert to class PgLOG -# Purpose : Python library module to log message and also do other things +# Purpose: Python library module to log message and also do other things # according to the value of logact, like display the error # message on screen and exit script -# -# Github : https://github.com/NCAR/rda-python-common.git -# +# Github: https://github.com/NCAR/rda-python-common.git ############################################################################### - import sys import os import re @@ -79,85 +74,82 @@ class PgLOG: def __init__(self): self.PGLOG = { # more defined in untaint_suid() with environment variables - 'EMLADDR' : '', - 'CCDADDR' : '', - 'SEPLINE' : "===========================================================\n", - 'TWOGBS' : 2147483648, - 'ONEGBS' : 1073741824, - 'MINSIZE' : 100, # minimal file size in bytes to be valid - 'LOGMASK' : (0xFFFFFF), # log mask to turn off certain log action bits - 'BCKGRND' : 0, # background process flag -b - 'ERRCNT' : 0, # record number of errors for email - 'ERRMSG' : '', # record error message for email - 'SUMMSG' : '', # record summary message for email - 'EMLMSG' : '', # record detail message for email - 'PRGMSG' : '', # record progressing message for email, replaced each time - 'GMTZ' : 0, # 0 - use local time, 1 - use greenwich mean time - 'NOLEAP' : 0, # 1 - skip 29 of Feburary while add days to date - 'GMTDIFF' : 6, # gmt is 6 hours ahead of us - 'CURUID' : None, # the login name who executes the program - 'SETUID' : '', # the login name for suid if it is different to the CURUID - 'FILEMODE': 0o664, # default 8-base file mode - 'EXECMODE': 0o775, # default 8-base executable file mode or directory mode - 'GDEXUSER' : "gdexdata", # common gdex user name - 'GDEXEMAIL' : "zji", # specialist to receipt email intead of common gdex user name - 'SUDOGDEX' : 0, # 1 to allow sudo to self.PGLOG['GDEXUSER'] - 'HOSTNAME' : '', # current host name the process in running on - 'OBJCTSTR' : "object", - 'BACKUPNM' : "quasar", - 'DRDATANM' : "drdata", - 'GPFSNAME' : "glade", - 'PBSNAME' : "PBS", - 'DSIDCHRS' : "d", - 'DOSHELL' : False, - 'NEWDSID' : True, - 'PUSGDIR' : None, - 'BCHHOSTS' : "PBS", - 'HOSTTYPE' : 'dav', # default HOSTTYPE - 'EMLMAX' : 256, # up limit of email line count - 'PGBATCH' : '', # current batch service name, SLURM or PBS - 'PGBINDIR' : '', - 'SLMTIME' : 604800, # max runtime for SLURM bath job, (7x24x60x60 seconds) - 'PBSTIME' : 86400, # max runtime for PBS bath job, (24x60x60 seconds) - 'MSSGRP' : None, # set if set to different HPSS group - 'GDEXGRP' : "decs", - 'EMLSEND' : None, # path to sendmail, None if not exists - 'DSCHECK' : None, # carry some cached dscheck information - 'PGDBBUF' : None, # reference to a connected database object - 'HPSSLMT' : 10, # up limit of HPSS streams - 'NOQUIT' : 0, # do not quit if this flag is set for daemons - 'DBRETRY' : 2, # db retry count after error - 'TIMEOUT' : 15, # default timeout (in seconds) for tosystem() - 'CMDTIME' : 120, # default command time (in seconds) for pgsystem() to record end time - 'SYSERR' : None, # cache the error message generated inside pgsystem() - 'ERR2STD' : [], # if non-empty reference to array of strings, change stderr to stdout if match - 'STD2ERR' : [], # if non-empty reference to array of strings, change stdout to stderr if match + 'EMLADDR': '', + 'CCDADDR': '', + 'SEPLINE': "===========================================================\n", + 'TWOGBS': 2147483648, + 'ONEGBS': 1073741824, + 'MINSIZE': 100, # minimal file size in bytes to be valid + 'LOGMASK': (0xFFFFFF), # log mask to turn off certain log action bits + 'BCKGRND': 0, # background process flag -b + 'ERRCNT': 0, # record number of errors for email + 'ERRMSG': '', # record error message for email + 'SUMMSG': '', # record summary message for email + 'EMLMSG': '', # record detail message for email + 'PRGMSG': '', # record progressing message for email, replaced each time + 'GMTZ': 0, # 0 - use local time, 1 - use greenwich mean time + 'NOLEAP': 0, # 1 - skip 29 of Feburary while add days to date + 'GMTDIFF': 6, # gmt is 6 hours ahead of us + 'CURUID': None, # the login name who executes the program + 'SETUID': '', # the login name for suid if it is different to the CURUID + 'FILEMODE': 0o664, # default 8-base file mode + 'EXECMODE': 0o775, # default 8-base executable file mode or directory mode + 'GDEXUSER': "gdexdata", # common gdex user name + 'GDEXEMAIL': "zji", # specialist to receipt email intead of common gdex user name + 'SUDOGDEX': 0, # 1 to allow sudo to self.PGLOG['GDEXUSER'] + 'HOSTNAME': '', # current host name the process in running on + 'OBJCTSTR': "object", + 'BACKUPNM': "quasar", + 'DRDATANM': "drdata", + 'GPFSNAME': "glade", + 'PBSNAME': "PBS", + 'DSIDCHRS': "d", + 'DOSHELL': False, + 'NEWDSID': True, + 'PUSGDIR': None, + 'BCHHOSTS': "PBS", + 'HOSTTYPE': 'dav', # default HOSTTYPE + 'EMLMAX': 256, # up limit of email line count + 'PGBATCH': '', # current batch service name, PBS + 'PGBINDIR': '', + 'PBSTIME': 86400, # max runtime for PBS bath job, (24x60x60 seconds) + 'MSSGRP': None, # set if set to different HPSS group + 'GDEXGRP': "decs", + 'EMLSEND': None, # path to sendmail, None if not exists + 'DSCHECK': None, # carry some cached dscheck information + 'PGDBBUF': None, # reference to a connected database object + 'NOQUIT': 0, # do not quit if this flag is set for daemons + 'DBRETRY': 2, # db retry count after error + 'TIMEOUT': 15, # default timeout (in seconds) for tosystem() + 'CMDTIME': 120, # default command time (in seconds) for pgsystem() to record end time + 'SYSERR': None, # cache the error message generated inside pgsystem() + 'ERR2STD': [], # if non-empty reference to array of strings, change stderr to stdout if match + 'STD2ERR': [], # if non-empty reference to array of strings, change stdout to stderr if match 'MISSFILE': "No such file or directory", - 'GITHUB' : "https://github.com" , # github server - 'EMLSRVR' : "ndir.ucar.edu", # UCAR email server and port - 'EMLPORT' : 25 + 'GITHUB': "https://github.com" , # github server + 'EMLSRVR': "ndir.ucar.edu", # UCAR email server and port + 'EMLPORT': 25 } self.PGLOG['RDAUSER'] = self.PGLOG['GDEXUSER'] self.PGLOG['RDAGRP'] = self.PGLOG['GDEXGRP'] self.PGLOG['RDAEMAIL'] = self.PGLOG['GDEXEMAIL'] self.PGLOG['SUDORDA'] = self.PGLOG['SUDOGDEX'] self.HOSTTYPES = { - 'rda' : 'dsg_mach', - 'casper' : 'dav', - 'crhtc' : 'dav', - 'cron' : 'dav', + 'rda': 'dsg_mach', + 'crlogin': 'dav', + 'casper': 'dav', + 'crhtc': 'dav', + 'cron': 'dav', } self.CPID = { - 'PID' : "", - 'CTM' : int(time.time()), - 'CMD' : "", - 'CPID' : "", + 'PID': "", + 'CTM': int(time.time()), + 'CMD': "", + 'CPID': "", } - self.BCHCMDS = {'PBS' : 'qsub'} + self.BCHCMDS = {'PBS': 'qsub'} # global dists to cashe information self.COMMANDS = {} - self.SLMHOSTS = [] - self.SLMSTATS = {} self.PBSHOSTS = [] self.PBSSTATS = {} # set additional common PGLOG values @@ -230,10 +222,10 @@ def get_email(self): def send_customized_email(self, logmsg, emlmsg, logact = None): if logact is None: logact = self.LOGWRN entries = { - 'fr' : ["From", 1, None], - 'to' : ["To", 1, None], - 'cc' : ["Cc", 0, ''], - 'sb' : ["Subject", 1, None] + 'fr': ["From", 1, None], + 'to': ["To", 1, None], + 'cc': ["Cc", 0, ''], + 'sb': ["Subject", 1, None] } if logmsg: logmsg += ': ' @@ -1067,10 +1059,10 @@ def set_common_pglog(self): if self.valid_command(sm): self.SETPGLOG("EMLSEND", f"{sm} -t") # send email command self.SETPGLOG("DBGLEVEL", '') # debug level self.SETPGLOG("BAOTOKEN", 's.lh2t2kDjrqs3V8y2BU2zOocT') # OpenBao token - self.SETPGLOG("DBGPATH", self.PGLOG['DSSDBHM']+"/log") # path to debug log file - self.SETPGLOG("OBJCTBKT", "gdex-data") # default Bucket on Object Store - self.SETPGLOG("BACKUPEP", "gdex-quasar") # default Globus Endpoint on Quasar - self.SETPGLOG("DRDATAEP", "gdex-quasar-drdata") # DRDATA Globus Endpoint on Quasar + self.SETPGLOG("DBGPATH", self.PGLOG['DSSDBHM']+"/log") # path to debug log file + self.SETPGLOG("OBJCTBKT", "gdex-data") # default Bucket on Object Store + self.SETPGLOG("BACKUPEP", "gdex-quasar") # default Globus Endpoint on Quasar + self.SETPGLOG("DRDATAEP", "gdex-quasar-drdata") # DRDATA Globus Endpoint on Quasar self.SETPGLOG("DBGFILE", "pgdss.dbg") # debug file name self.SETPGLOG("CNFPATH", self.PGLOG['DSSHOME']+"/config") # path to configuration files self.SETPGLOG("DSSURL", "https://gdex.ucar.edu") # current dss web URL @@ -1083,24 +1075,23 @@ def set_common_pglog(self): self.SETPGLOG("DSSWEB", self.PGLOG['LOCDATA']+"/web") self.SETPGLOG("DSWHOME", self.PGLOG['DSSWEB']+"/datasets") # datast web root path self.PGLOG['HOMEROOTS'] = "{}|{}".format(self.PGLOG['DSSHOME'], self.PGLOG['DSWHOME']) - self.SETPGLOG("DSSDATA", "/glade/campaign/collections/gdex") # dss data root path + self.SETPGLOG("DSSDATA", "/glade/campaign/collections/gdex") # dss data root path self.SETPGLOG("DSDHOME", self.PGLOG['DSSDATA']+"/data") # dataset data root path self.SETPGLOG("DECSHOME", self.PGLOG['DSSDATA']+"/decsdata") # dataset decsdata root path self.SETPGLOG("DSHHOME", self.PGLOG['DECSHOME']+"/helpfiles") # dataset help root path - self.SETPGLOG("GDEXWORK", "/lustre/desc1/gdex/work") # gdex work path + self.SETPGLOG("GDEXWORK", "/lustre/desc1/gdex/work") # gdex work path self.SETPGLOG("UPDTWKP", self.PGLOG['GDEXWORK']) # dsupdt work root path - self.SETPGLOG("TRANSFER", "/lustre/desc1/gdex/transfer") # gdex transfer path + self.SETPGLOG("TRANSFER", "/lustre/desc1/gdex/transfer") # gdex transfer path self.SETPGLOG("RQSTHOME", self.PGLOG['TRANSFER']+"/dsrqst") # dsrqst home - self.SETPGLOG("DSAHOME", "") # dataset data alternate root path - self.SETPGLOG("RQSTALTH", "") # alternate dsrqst path - self.SETPGLOG("GPFSHOST", "") # empty if writable to glade - self.SETPGLOG("PSQLHOST", "rda-db.ucar.edu") # host name for postgresql server - self.SETPGLOG("SLMHOSTS", "cheyenne:casper") # host names for SLURM server - self.SETPGLOG("PBSHOSTS", "cron:casper") # host names for PBS server - self.SETPGLOG("CHKHOSTS", "") # host names for dscheck daemon + self.SETPGLOG("DSAHOME", "") # dataset data alternate root path + self.SETPGLOG("RQSTALTH", "") # alternate dsrqst path + self.SETPGLOG("GPFSHOST", "") # empty if writable to glade + self.SETPGLOG("PSQLHOST", "rda-db.ucar.edu") # host name for postgresql server + self.SETPGLOG("PBSHOSTS", "cron:casper:crlogin") # host names for PBS server + self.SETPGLOG("CHKHOSTS", "") # host names for dscheck daemon self.SETPGLOG("PVIEWHOST", "pgdb02.k8s.ucar.edu") # host name for view only postgresql server self.SETPGLOG("PMISCHOST", "pgdb03.k8s.ucar.edu") # host name for misc postgresql server - self.SETPGLOG("FTPUPLD", self.PGLOG['TRANSFER']+"/rossby") # ftp upload path + self.SETPGLOG("FTPUPLD", self.PGLOG['TRANSFER']+"/rossby") # ftp upload path self.PGLOG['GPFSROOTS'] = "{}|{}|{}".format(self.PGLOG['DSDHOME'], self.PGLOG['UPDTWKP'], self.PGLOG['RQSTHOME']) if 'ECCODES_DEFINITION_PATH' not in os.environ: os.environ['ECCODES_DEFINITION_PATH'] = "/usr/local/share/eccodes/definitions" diff --git a/src/rda_python_common/pg_opt.py b/src/rda_python_common/pg_opt.py index d001a34..cc64c6e 100644 --- a/src/rda_python_common/pg_opt.py +++ b/src/rda_python_common/pg_opt.py @@ -1,20 +1,14 @@ -# ############################################################################### -# -# Title : pg_opt.py -# -# Author : Zaihua Ji, zji@ucar.edu -# Date : 08/26/2020 +# Title: pg_opt.py +# Author: Zaihua Ji, zji@ucar.edu +# Date: 08/26/2020 # 2025-01-10 transferred to package rda_python_common from # https://github.com/NCAR/rda-shared-libraries.git # 2025-12-01 convert to class PgOPT -# Purpose : python library module for holding global varaibles +# Purpose: python library module for holding global varaibles # functions for processing options and other global functions -# -# Github : https://github.com/NCAR/rda-pyhon-common.git -# +# Github: https://github.com/NCAR/rda-pyhon-common.git ############################################################################### -# import os import sys import re @@ -76,86 +70,86 @@ def __init__(self): self.OPTS = {} # global initial optional values self.PGOPT = { - 'ACTS' : 0, # carry current action bits - 'UACTS' : 0, # carry dsarch skip check UD action bits - 'CACT' : '', # current short action name - 'IFCNT' : 0, # 1 to read a single Input File at a time - 'ANAME' : '', # cache the application name if set - 'TABLE' : '', # table name the action is on - 'UID' : 0, # user.uid - 'MSET' : 'SA', # Action for multiple sets - 'WIDTH' : 128, # max column width - 'TXTBIT' : 64, # text field bit (0x1000) allow multiple lines - 'PEMAX' : 12, # max count of reuqest partition errors for auto reprocesses - 'PTMAX' : 24, # max number of partitions for a single request - 'REMAX' : 2, # max count of reuqest errors for auto reprocesses - 'RSMAX' : 100, # max count of gatherxml with options -R -S - 'RCNTL' : None, # placehold for a request control record - 'dcm' : "dcm", - 'sdp' : "sdp", - 'rcm' : "rcm", - 'scm' : "scm", - 'wpg' : "", - 'gatherxml' : "gatherxml", - 'cosconvert' : "cosconvert", - 'emllog' : self.LGWNEM, - 'emlerr' : self.LOGERR|self.EMEROL, - 'emerol' : self.LOGWRN|self.EMEROL, - 'emlsum' : self.LOGWRN|self.EMLSUM, - 'emlsep' : self.LGWNEM|self.SEPLIN, - 'wrnlog' : self.LOGWRN, - 'errlog' : self.LOGERR, - 'extlog' : self.LGEREX, - 'PTYPE' : "CPRV", - 'WDTYP' : "ADNU", - 'HFTYP' : "DS", - 'SDTYP' : "PORWUV", - 'GXTYP' : "DP" + 'ACTS': 0, # carry current action bits + 'UACTS': 0, # carry dsarch skip check UD action bits + 'CACT': '', # current short action name + 'IFCNT': 0, # 1 to read a single Input File at a time + 'ANAME': '', # cache the application name if set + 'TABLE': '', # table name the action is on + 'UID': 0, # user.uid + 'MSET': 'SA', # Action for multiple sets + 'WIDTH': 128, # max column width + 'TXTBIT': 64, # text field bit (0x1000) allow multiple lines + 'PEMAX': 12, # max count of reuqest partition errors for auto reprocesses + 'PTMAX': 24, # max number of partitions for a single request + 'REMAX': 2, # max count of reuqest errors for auto reprocesses + 'RSMAX': 100, # max count of gatherxml with options -R -S + 'RCNTL': None, # placehold for a request control record + 'dcm': "dcm", + 'sdp': "sdp", + 'rcm': "rcm", + 'scm': "scm", + 'wpg': "", + 'gatherxml': "gatherxml", + 'cosconvert': "cosconvert", + 'emllog': self.LGWNEM, + 'emlerr': self.LOGERR|self.EMEROL, + 'emerol': self.LOGWRN|self.EMEROL, + 'emlsum': self.LOGWRN|self.EMLSUM, + 'emlsep': self.LGWNEM|self.SEPLIN, + 'wrnlog': self.LOGWRN, + 'errlog': self.LOGERR, + 'extlog': self.LGEREX, + 'PTYPE': "CPRV", + 'WDTYP': "ADNU", + 'HFTYP': "DS", + 'SDTYP': "PORWUV", + 'GXTYP': "DP" } # global default parameters self.params = { - 'ES' : "<=>", - 'AO' : "", - 'DV' : "<:>" + 'ES': "<=>", + 'AO': "", + 'DV': "<:>" } self.WTYPE = { - 'A' : "ARCO", - 'D' : "DATA", - 'N' : "NCAR", - 'U' : "UNKNOWN", + 'A': "ARCO", + 'D': "DATA", + 'N': "NCAR", + 'U': "UNKNOWN", } self.HTYPE = { - 'D' : "DOCUMENT", - 'S' : "SOFTWARE", - 'U' : "UNKNOWN" + 'D': "DOCUMENT", + 'S': "SOFTWARE", + 'U': "UNKNOWN" } self.HPATH = { - 'D' : "docs", - 'S' : "software", - 'U' : "help" + 'D': "docs", + 'S': "software", + 'U': "help" } self.MTYPE = { - 'P' : "PRIMARY", - 'A' : "ARCHIVING", - 'V' : "VERSION", - 'W' : "WORKING", - 'R' : "ORIGINAL", - 'B' : "BACKUP", - 'O' : "OFFSITE", - 'C' : "CHRONOPOLIS", - 'U' : "UNKNOWN" + 'P': "PRIMARY", + 'A': "ARCHIVING", + 'V': "VERSION", + 'W': "WORKING", + 'R': "ORIGINAL", + 'B': "BACKUP", + 'O': "OFFSITE", + 'C': "CHRONOPOLIS", + 'U': "UNKNOWN" } self.STYPE = { - 'O' : "OFFLINE", - 'P' : "PRIMARY", - 'R' : "ORIGINAL", - 'V' : "VERSION", - 'W' : "WORKING", - 'U' : "UNKNOWN" + 'O': "OFFLINE", + 'P': "PRIMARY", + 'R': "ORIGINAL", + 'V': "VERSION", + 'W': "WORKING", + 'U': "UNKNOWN" } self.BTYPE = { - 'B' : "BACKUPONLY", - 'D' : "BACKDRDATA", + 'B': "BACKUPONLY", + 'D': "BACKDRDATA", } # process and parsing input information @@ -1224,7 +1218,7 @@ def get_control_frequency(frequency): val = "invalid frequency '{}', unit must be (Y,M,W,D,H)".format(frequency) return (None, unit) freq = [0]*7 # initialize the frequence list - uidx = {'Y' : 0, 'D' : 2, 'H' : 3, 'N' : 4, 'S' : 5} + uidx = {'Y': 0, 'D': 2, 'H': 3, 'N': 4, 'S': 5} if unit == 'M': freq[1] = val if nf: freq[6] = nf # number of fractions in a month @@ -1273,18 +1267,18 @@ def append_format_string(sformat, sfmt, chkend = 0): @staticmethod def request_type(rtype, idx = 0): RTYPE = { - 'C' : ["Customized Data", 0], - 'D' : ["CDP Link", 0], - 'M' : ["Delayed Mode Data", 1], - 'N' : ["NCARDAP(THREDDS) Data Server", 0], - 'Q' : ["Database Query", 0], - 'R' : ["Realtime Data", 0], - 'S' : ["Subset Data", 0], - 'T' : ["Subset/Format-Conversion Data", 0], - 'F' : ["Format Conversion Data", 1], # web - 'A' : ["Archive Format Conversion", 1], # web - 'P' : ["Plot Chart", 0], - 'U' : ["Data", 0] + 'C': ["Customized Data", 0], + 'D': ["CDP Link", 0], + 'M': ["Delayed Mode Data", 1], + 'N': ["NCARDAP(THREDDS) Data Server", 0], + 'Q': ["Database Query", 0], + 'R': ["Realtime Data", 0], + 'S': ["Subset Data", 0], + 'T': ["Subset/Format-Conversion Data", 0], + 'F': ["Format Conversion Data", 1], # web + 'A': ["Archive Format Conversion", 1], # web + 'P': ["Plot Chart", 0], + 'U': ["Data", 0] } if rtype not in RTYPE: rtype = 'U' return RTYPE[rtype][idx] diff --git a/src/rda_python_common/pg_sig.py b/src/rda_python_common/pg_sig.py index 827f5f9..fdad7c0 100644 --- a/src/rda_python_common/pg_sig.py +++ b/src/rda_python_common/pg_sig.py @@ -1,19 +1,13 @@ -# ############################################################################### -# -# Title : pg_sig.py -# -# Author : Zaihua Ji, zji@ucar.edu -# Date : 08/05/2020 +# Title: pg_sig.py +# Author: Zaihua Ji, zji@ucar.edu +# Date: 08/05/2020 # 2025-01-10 transferred to package rda_python_common from # https://github.com/NCAR/rda-shared-libraries.git # 2025-11-20 convert to class PgSIG -# Purpose : python library module for start and control daemon process -# -# Github : https://github.com/NCAR/rda-python-common.git -# +# Purpose: python library module for start and control daemon process +# Github: https://github.com/NCAR/rda-python-common.git ############################################################################### -# import os import re import sys @@ -31,26 +25,26 @@ def __init__(self): self.CPIDS = {} # allow upto 'mproc' processes at one time for daemon self.CBIDS = {} # allow upto 'bproc' background processes at one time for each child self.SDUMP = { - 'DEF' : '/dev/null', - 'ERR' : '', - 'OUT' : '' + 'DEF': '/dev/null', + 'ERR': '', + 'OUT': '' } self.PGSIG = { - 'QUIT' : 0, # 1 if QUIT signal received, quit server if no child - 'MPROC' : 1, # default number of multiple processes - 'BPROC' : 1, # default number of multiple background processes - 'ETIME' : 20, # default error waiting time (in seconds) - 'WTIME' : 120, # default waiting time (in seconds) - 'DTIME' : 600, # the daemon record refresh time (in seconds) - 'RTIME' : 2400, # the web rda config unlocking and unconfigured system down waiting time (in seconds) - 'CTIME' : 4800, # the lock cleaning & configued system down waiting time (in seconds) - 'PPID' : -1, # 1 - server, (> 1) - child, 0 - non-daemon mode - 'PID' : 0, # current process ID - 'DNAME' : '', # daemon name - 'DSTR' : '', # string for daemon with user login name - 'MTIME' : 0, # maximum daemon running time in seconds, 0 for unlimited - 'STIME' : 0, # time the daemon is started - 'STRTM' : '', # string format of 'STIME' + 'QUIT': 0, # 1 if QUIT signal received, quit server if no child + 'MPROC': 1, # default number of multiple processes + 'BPROC': 1, # default number of multiple background processes + 'ETIME': 20, # default error waiting time (in seconds) + 'WTIME': 120, # default waiting time (in seconds) + 'DTIME': 600, # the daemon record refresh time (in seconds) + 'RTIME': 2400, # the web rda config unlocking and unconfigured system down waiting time (in seconds) + 'CTIME': 4800, # the lock cleaning & configued system down waiting time (in seconds) + 'PPID': -1, # 1 - server, (> 1) - child, 0 - non-daemon mode + 'PID': 0, # current process ID + 'DNAME': '', # daemon name + 'DSTR': '', # string for daemon with user login name + 'MTIME': 0, # maximum daemon running time in seconds, 0 for unlimited + 'STIME': 0, # time the daemon is started + 'STRTM': '', # string format of 'STIME' } # add users for starting this daemon diff --git a/src/rda_python_common/pg_split.py b/src/rda_python_common/pg_split.py index 1aa715e..744592d 100644 --- a/src/rda_python_common/pg_split.py +++ b/src/rda_python_common/pg_split.py @@ -1,33 +1,26 @@ -# ############################################################################### -# -# Title : pg_split.py -- PostgreSQL DataBase Interface foe table wfile -# Author : Zaihua Ji, zji@ucar.edu -# Date : 09/010/2024 +# Title: pg_split.py -- PostgreSQL DataBase Interface foe table wfile +# Author: Zaihua Ji, zji@ucar.edu +# Date: 09/010/2024 # 2025-01-10 transferred to package rda_python_common from # https://github.com/NCAR/rda-shared-libraries.git # 2025-12-01 convert to class PgSplit -# Purpose : Python library module to handle query and manipulate table wfile -# -# Github : https://github.com/NCAR/rda-python-common.git -# +# Purpose: Python library module to handle query and manipulate table wfile +# Github: https://github.com/NCAR/rda-python-common.git ############################################################################### - import os import re from os import path as op -from .pg_dbi import PgDBI from .pg_util import PgUtil -class PgSplit(PgDBI): +class PgSplit(PgUtil): def __init__(self): super().__init__() # initialize parent class # compare wfile records between tables wfile and wfile_dNNNNNN, # and return the records need to be added, modified and deleted - @staticmethod - def compare_wfile(wfrecs, dsrecs): + def compare_wfile(self, wfrecs, dsrecs): flds = dsrecs.keys() flen = len(flds) arecs = dict(zip(flds, [[]]*flen)) @@ -39,15 +32,15 @@ def compare_wfile(wfrecs, dsrecs): i = j = 0 while i < wfcnt and j < dscnt: if i > pi: - wfrec = PgUtil.onerecord(wfrecs, i) + wfrec = self.onerecord(wfrecs, i) wwid = wfrec['wid'] pi = i if j > pj: - dsrec = PgUtil.onerecord(dsrecs, j) + dsrec = self.onerecord(dsrecs, j) dwid = dsrec['wid'] pj = j if wwid == dwid: - mrec = PgSplit.compare_one_record(flds, wfrec, dsrec) + mrec = self.compare_one_record(flds, wfrec, dsrec) if mrec: mrecs[wwid] = mrec i += 1 j += 1 @@ -109,8 +102,8 @@ def get_dsid_condition(dsid, condition): # insert one record into wfile and/or wfile_dsid def pgadd_wfile(self, dsid, wfrec, logact = None, getid = None): if logact is None: logact = self.LOGERR - record = {'wfile' : wfrec['wfile'], - 'dsid' : (wfrec['dsid'] if 'dsid' in wfrec else dsid)} + record = {'wfile': wfrec['wfile'], + 'dsid': (wfrec['dsid'] if 'dsid' in wfrec else dsid)} wret = self.pgadd('wfile', record, logact, 'wid') if wret: record = self.wfile2wdsid(wfrec, wret) @@ -123,8 +116,8 @@ def pgadd_wfile(self, dsid, wfrec, logact = None, getid = None): # insert multiple records into wfile and/or wfile_dsid def pgmadd_wfile(self, dsid, wfrecs, logact = None, getid = None): if logact is None: logact = self.LOGERR - records = {'wfile' : wfrecs['wfile'], - 'dsid' : (wfrecs['dsid'] if 'dsid' in wfrecs else [dsid]*len(wfrecs['wfile']))} + records = {'wfile': wfrecs['wfile'], + 'dsid': (wfrecs['dsid'] if 'dsid' in wfrecs else [dsid]*len(wfrecs['wfile']))} wret = self.pgmadd('wfile', records, logact, 'wid') wcnt = wret if isinstance(wret, int) else len(wret) if wcnt: diff --git a/src/rda_python_common/pg_util.py b/src/rda_python_common/pg_util.py index a674c1f..4c4d52c 100644 --- a/src/rda_python_common/pg_util.py +++ b/src/rda_python_common/pg_util.py @@ -1,18 +1,13 @@ -# ############################################################################### -# -# Title : pg_util.py -- module for misc utilities. -# Author : Zaihua Ji, zji@ucar.edu -# Date : 07/27/2020 +# Title: pg_util.py -- module for misc utilities. +# Author: Zaihua Ji, zji@ucar.edu +# Date: 07/27/2020 # 2025-01-10 transferred to package rda_python_common from # https://github.com/NCAR/rda-shared-libraries.git # 2025-11-20 convert to class PgUtil -# Purpose : python library module for global misc utilities -# -# Github : https://github.com/NCAR/rda-python-common.git -# +# Purpose: python library module for global misc utilities +# Github: https://github.com/NCAR/rda-python-common.git ############################################################################### -# import os import re import time @@ -27,15 +22,15 @@ class PgUtil(PgLOG): def __init__(self): super().__init__() # initialize parent class self.DATEFMTS = { - 'C' : '(CC|C)', # century - 'Y' : '(YYYY|YY00|YYY|YY|YEAR|YR|Y)', # YYY means decade - 'Q' : '(QQ|Q)', # quarter - 'M' : '(Month|Mon|MM|M)', # numeric or string month - 'W' : '(Week|Www|W)', # string or numeric weedday - 'D' : '(DDD|DD|D)', # days in year or month - 'H' : '(HHH|HH|H)', # hours in month or day - 'N' : '(NNNN|NN|N)', # minutes in day or hour - 'S' : '(SSSS|SS|S)' # seconds in hour or minute + 'C': '(CC|C)', # century + 'Y': '(YYYY|YY00|YYY|YY|YEAR|YR|Y)', # YYY means decade + 'Q': '(QQ|Q)', # quarter + 'M': '(Month|Mon|MM|M)', # numeric or string month + 'W': '(Week|Www|W)', # string or numeric weedday + 'D': '(DDD|DD|D)', # days in year or month + 'H': '(HHH|HH|H)', # hours in month or day + 'N': '(NNNN|NN|N)', # minutes in day or hour + 'S': '(SSSS|SS|S)' # seconds in hour or minute } self.MONTHS = [ "january", "february", "march", "april", "may", "june", @@ -650,7 +645,7 @@ def find_dataset_id(self, idstr, flag = 'B', logact = 0): ms = re.search(r'(^|\W)(ds\d\d\d(\.|)\d)($|\D)', idstr) if not ms: ms = re.search(r'(^|\W)(\d\d\d\.\d)($|\D)', idstr) if ms: return ms.group(2) - if logact: self.pglog("{} : No valid dsid found for flag {}".format(idstr, flag), logact) + if logact: self.pglog("{}: No valid dsid found for flag {}".format(idstr, flag), logact) return None # find and convert all found dsids according to old/new dsids diff --git a/src/rda_python_common/pgpassword.py b/src/rda_python_common/pgpassword.py index cafa647..edfcc0c 100644 --- a/src/rda_python_common/pgpassword.py +++ b/src/rda_python_common/pgpassword.py @@ -1,18 +1,13 @@ #!/usr/bin/env python3 -# ################################################################################## -# # Title: pgpassword # Author: Zaihua Ji, zji@ucar.edu # Date: 2025-10-27 # 2025-12-02 convert to class PgPassword # Purpose: python script to retrieve passwords for postgrsql login to connect a # gdex database from inside an python application -# # Github: https://github.com/NCAR/rda-python-common.git -# ################################################################################## - import sys import re from .pg_dbi import PgDBI @@ -22,18 +17,18 @@ class PgPassword(PgDBI): def __init__(self): super().__init__() # initialize parent class self.DBFLDS = { - 'd' : 'dbname', - 'c' : 'scname', - 'h' : 'dbhost', - 'p' : 'dbport', - 'u' : 'lnname' + 'd': 'dbname', + 'c': 'scname', + 'h': 'dbhost', + 'p': 'dbport', + 'u': 'lnname' } self.DBINFO = { - 'dbname' : "", - 'scname' : "", - 'lnname' : "", - 'dbhost' : "", - 'dbport' : 5432 + 'dbname': "", + 'scname': "", + 'lnname': "", + 'dbhost': "", + 'dbport': 5432 } self.dbopt = False self.password = ''