Skip to content

Hua work common #82

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
zaihuaji merged 4 commits into from
Jan 8, 2026
Merged

Hua work common #82

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
925a694
remove empty lines in headers
Dec 31, 2025
0deeaed
add crlogin to PBSHOSTS
Jan 8, 2026
00860e9
remove spaces before :
Jan 8, 2026
9210e40
save
Jan 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"

[project]
name = "rda_python_common"
version = "2.0.12"
version = "2.0.14"
authors = [
{ name="Zaihua Ji", email="zji@ucar.edu" },
]
Expand Down
37 changes: 16 additions & 21 deletions src/rda_python_common/pg_cmd.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,18 +1,13 @@
#
###############################################################################
#
# Title : pg_cmd.py
# Author : Zaihua Ji, zji@ucar.edu
# Date : 08/25/2020
# Title: pg_cmd.py
# Author: Zaihua Ji, zji@ucar.edu
# Date: 08/25/2020
# 2025-01-10 transferred to package rda_python_common from
# https://github.com/NCAR/rda-shared-libraries.git
# Purpose : python library module for functions to record commands for delayed
# Purpose: python library module for functions to record commands for delayed
# mode or command recovery
#
# Github : https://github.com/NCAR/rda-python-common.git
#
# Github: https://github.com/NCAR/rda-python-common.git
###############################################################################
#
import os
import re
import sys
Expand All @@ -25,18 +20,18 @@ def __init__(self):
super().__init__() # initialize parent class
# cached dscheck info
self.DSCHK = {}
self.BOPTIONS = {"hostname" : None, "qoptions" : None, "modules" : None, "environments" : None}
self.BOPTIONS = {"hostname": None, "qoptions": None, "modules": None, "environments": None}
self.BFIELDS = ', '.join(self.BOPTIONS)
self.TRYLMTS = {
'dsquasar' : 3,
'dsarch' : 2,
'default' : 1
'dsquasar': 3,
'dsarch': 2,
'default': 1
}
self.DLYPTN = r'(^|\s)-(d|BP|BatchProcess|DelayedMode)(\s|$)'
self.DLYOPT = {
'dsarch' : ' -d',
'dsupdt' : ' -d',
'dsrqst' : ' -d'
'dsarch': ' -d',
'dsupdt': ' -d',
'dsrqst': ' -d'
}

# params: dict array holding option values
Expand Down Expand Up @@ -381,7 +376,7 @@ def get_dsrqst_counts(self, pgchk, logact = 0):
fcnt = pgrec['fcount']
else:
fcnt = 0
pgrec = {'fcount' : 0}
pgrec = {'fcount': 0}
if not fcnt: fcnt = self.pgget("wfrqst", "", cnd, logact)
if fcnt and fcount != fcnt: fcount = fcnt
if fcount:
Expand All @@ -402,14 +397,14 @@ def get_dsrqst_counts(self, pgchk, logact = 0):

# set dscheck fcount
def set_dscheck_fcount(self, count, logact = 0):
record = {'fcount' : count, 'chktime' : int(time.time())}
record = {'fcount': count, 'chktime': int(time.time())}
self.pgupdt("dscheck", record, self.DSCHK['chkcnd'], logact)
self.DSCHK['fcount'] = count
return self.DSCHK['dcount'] # return Done count

# set dscheck dcount
def set_dscheck_dcount(self, count, size, logact = 0):
record = {'dcount' : count, 'size' : size, 'chktime' : int(time.time())}
record = {'dcount': count, 'size': size, 'chktime': int(time.time())}
self.pgupdt("dscheck", record, self.DSCHK['chkcnd'], logact)
self.DSCHK['dcount'] = count
self.DSCHK['size'] = size
Expand Down Expand Up @@ -444,7 +439,7 @@ def record_dscheck_status(self, stat, logact = 0):
if pgrec['pid'] != cpid or pgrec['lockhost'] != chost: return 0

# update dscheck status only if it is still locked by the current process
record = {'status' : stat, 'chktime' : int(time.time()), 'pid' : 0}
record = {'status': stat, 'chktime': int(time.time()), 'pid': 0}
return self.pgupdt("dscheck", record, self.DSCHK['chkcnd'], logact)

# get the number of tries to execute for a given cmd under dscheck control
Expand Down
119 changes: 57 additions & 62 deletions src/rda_python_common/pg_dbi.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,18 +1,13 @@
#
###############################################################################
#
# Title : pg_dbi.py -- PostgreSQL DataBase Interface
# Author : Zaihua Ji, zji@ucar.edu
# Date : 06/07/2022
# Title: pg_dbi.py -- PostgreSQL DataBase Interface
# Author: Zaihua Ji, zji@ucar.edu
# Date: 06/07/2022
# 2025-01-10 transferred to package rda_python_common from
# https://github.com/NCAR/rda-shared-libraries.git
# 2025-11-24 convert to class PgDBI
# Purpose : Python library module to handle query and manipulate PostgreSQL database
#
# Github : https://github.com/NCAR/rda-python-common.git
#
# Purpose: Python library module to handle query and manipulate PostgreSQL database
# Github: https://github.com/NCAR/rda-python-common.git
###############################################################################

import os
import re
import time
Expand Down Expand Up @@ -49,22 +44,22 @@ def __init__(self):
self.PGSIGNS = ['!', '<', '>', '<>']
self.CHCODE = 1042
# hard coded db ports for dbnames
self.DBPORTS = {'default' : 0}
self.DBPORTS = {'default': 0}
self.DBPASS = {}
self.DBBAOS = {}
# hard coded db names for given schema names
self.DBNAMES = {
'ivaddb' : 'ivaddb',
'cntldb' : 'ivaddb',
'cdmsdb' : 'ivaddb',
'ispddb' : 'ispddb',
'obsua' : 'upadb',
'default' : 'rdadb',
'ivaddb': 'ivaddb',
'cntldb': 'ivaddb',
'cdmsdb': 'ivaddb',
'ispddb': 'ispddb',
'obsua': 'upadb',
'default': 'rdadb',
}
# hard coded socket paths for machine_dbnames
self.DBSOCKS = {'default' : ''}
self.DBSOCKS = {'default': ''}
# home path for check db on alter host
self.VIEWHOMES = {'default' : self.PGLOG['DSSDBHM']}
self.VIEWHOMES = {'default': self.PGLOG['DSSDBHM']}
# add more to the list if used for names
self.PGRES = ['end', 'window']
self.SETPGDBI('DEFDB', 'rdadb')
Expand Down Expand Up @@ -397,8 +392,8 @@ def pgconnect(self, reconnect = 0, pgcnt = 0, autocommit = True):
elif reconnect:
reconnect = 0 # initial connection
while True:
config = {'database' : self.PGDBI['DBNAME'],
'user' : self.PGDBI['LNNAME']}
config = {'database': self.PGDBI['DBNAME'],
'user': self.PGDBI['LNNAME']}
if self.PGDBI['DBSHOST'] == self.PGLOG['HOSTNAME']:
config['host'] = 'localhost'
else:
Expand Down Expand Up @@ -748,8 +743,8 @@ def pgmget(self, tablenames, fields, condition = None, logact = None):

# tablenames: comma deliminated string of one or more tables
# fields: comma deliminated string of one or more field names,
# cnddict: condition dict with field names : values
# return a dict(field names : values) upon success
# cnddict: condition dict with field names: values
# return a dict(field names: values) upon success
# retrieve one records from tablenames condition dict
def pghget(self, tablenames, fields, cnddict, logact = None):
if logact is None: logact = self.PGDBI['ERRLOG']
Expand Down Expand Up @@ -794,8 +789,8 @@ def pghget(self, tablenames, fields, cnddict, logact = None):

# tablenames: comma deliminated string of one or more tables
# fields: comma deliminated string of one or more field names,
# cnddicts: condition dict with field names : value lists
# return a dict(field names : value lists) upon success
# cnddicts: condition dict with field names: value lists
# return a dict(field names: value lists) upon success
# retrieve multiple records from tablenames for condition dict
def pgmhget(self, tablenames, fields, cnddicts, logact = None):
if logact is None: logact = self.PGDBI['ERRLOG']
Expand Down Expand Up @@ -870,7 +865,7 @@ def prepare_update(self, tablename, fields, condition = None, cndflds = None):

# update one or multiple rows in tablename
# tablename: update for one table name each call
# record: dict with field names : values
# record: dict with field names: values
# condition: update conditions for where clause)
# return number of rows undated upon success
def pgupdt(self, tablename, record, condition, logact = None):
Expand Down Expand Up @@ -904,8 +899,8 @@ def pgupdt(self, tablename, record, condition, logact = None):

# update one or multiple records in tablename
# tablename: update for one table name each call
# record: update values, dict with field names : values
# cnddict: condition dict with field names : values
# record: update values, dict with field names: values
# cnddict: condition dict with field names: values
# return number of records updated upon success
def pghupdt(self, tablename, record, cnddict, logact = None):
if logact is None: logact = self.PGDBI['ERRLOG']
Expand Down Expand Up @@ -939,8 +934,8 @@ def pghupdt(self, tablename, record, cnddict, logact = None):

# update multiple records in tablename
# tablename: update for one table name each call
# records: update values, dict with field names : value lists
# cnddicts: condition dict with field names : value lists
# records: update values, dict with field names: value lists
# cnddicts: condition dict with field names: value lists
# return number of records updated upon success
def pgmupdt(self, tablename, records, cnddicts, logact = None):
if logact is None: logact = self.PGDBI['ERRLOG']
Expand Down Expand Up @@ -1021,7 +1016,7 @@ def pgdel(self, tablename, condition, logact = None):

# delete one or mutiple records in tablename according condition
# tablename: delete for one table name each call
# cndict: delete condition dict for names : values
# cndict: delete condition dict for names: values
# return number of records deleted upon success
def pghdel(self, tablename, cnddict, logact = None):
if logact is None: logact = self.PGDBI['ERRLOG']
Expand Down Expand Up @@ -1052,7 +1047,7 @@ def pghdel(self, tablename, cnddict, logact = None):

# delete mutiple records in tablename according condition
# tablename: delete for one table name each call
# cndicts: delete condition dict for names : value lists
# cndicts: delete condition dict for names: value lists
# return number of records deleted upon success
def pgmdel(self, tablename, cnddicts, logact = None):
if logact is None: logact = self.PGDBI['ERRLOG']
Expand Down Expand Up @@ -1157,7 +1152,7 @@ def check_user_uid(self, userno, date = None):
pgrec = self.pgget("dssdb.user", "uid", "userno = {}".format(userno), self.PGDBI['ERRLOG'])
if pgrec: return pgrec['uid']
pgrec = self.ucar_user_info(userno)
if not pgrec: pgrec = {'userno' : userno, 'stat_flag' : 'M'}
if not pgrec: pgrec = {'userno': userno, 'stat_flag': 'M'}
uid = self.pgadd("dssdb.user", pgrec, (self.PGDBI['EXITLG']|self.AUTOID))
if uid: self.pglog("{}: Scientist ID Added as user.uid = {}".format(userno, uid), self.LGWNEM)
return uid
Expand All @@ -1179,26 +1174,26 @@ def get_user_uid(self, logname, date = None):
pgrec = self.pgget("dssdb.user", "uid", "logname = '{}'".format(logname), self.PGDBI['ERRLOG'])
if pgrec: return pgrec['uid']
pgrec = self.ucar_user_info(0, logname)
if not pgrec: pgrec = {'logname' : logname, 'stat_flag' : 'M'}
if not pgrec: pgrec = {'logname': logname, 'stat_flag': 'M'}
uid = self.pgadd("dssdb.user", pgrec, (self.PGDBI['EXITLG']|self.AUTOID))
if uid: self.pglog("{}: UCAR Login Name Added as user.uid = {}".format(logname, uid), self.LGWNEM)
return uid

# get ucar user info for given userno (scientist number) or logname (Ucar login)
def ucar_user_info(self, userno, logname = None):
matches = {
'upid' : "upid",
'uid' : "userno",
'username' : "logname",
'lastName' : "lstname",
'firstName' : "fstname",
'active' : "stat_flag",
'internalOrg' : "division",
'externalOrg' : "org_name",
'country' : "country",
'forwardEmail' : "email",
'email' : "ucaremail",
'phone' : "phoneno"
'upid': "upid",
'uid': "userno",
'username': "logname",
'lastName': "lstname",
'firstName': "fstname",
'active': "stat_flag",
'internalOrg': "division",
'externalOrg': "org_name",
'country': "country",
'forwardEmail': "email",
'email': "ucaremail",
'phone': "phoneno"
}
buf = self.pgsystem("pgperson " + ("-uid {}".format(userno) if userno else "-username {}".format(logname)), self.LOGWRN, 20)
if not buf: return None
Expand Down Expand Up @@ -1250,13 +1245,13 @@ def ucar_user_info(self, userno, logname = None):
# set country code for given coutry name or email address
def set_country_code(self, email, country = None):
codes = {
'CHINA' : "P.R.CHINA",
'ENGLAND' : "UNITED.KINGDOM",
'FR' : "FRANCE",
'KOREA' : "SOUTH.KOREA",
'USSR' : "RUSSIA",
'US' : "UNITED.STATES",
'U.S.A.' : "UNITED.STATES"
'CHINA': "P.R.CHINA",
'ENGLAND': "UNITED.KINGDOM",
'FR': "FRANCE",
'KOREA': "SOUTH.KOREA",
'USSR': "RUSSIA",
'US': "UNITED.STATES",
'U.S.A.': "UNITED.STATES"
}
if country:
country = country.upper()
Expand Down Expand Up @@ -1284,7 +1279,7 @@ def check_wuser_wuid(self, email, date = None):
pgrec = self.pgget("wuser", "wuid", emcond, self.LOGERR)
if pgrec: return pgrec['wuid']
# now add one in
record = {'email' : email}
record = {'email': email}
# check again if a ruser is on file
pgrec = self.pgget("ruser", "*", emcond + " AND end_date IS NULL", self.PGDBI['ERRLOG'])
if not pgrec: pgrec = self.pgget("ruser", "*", emcond, self.PGDBI['ERRLOG'])
Expand Down Expand Up @@ -1530,11 +1525,11 @@ def build_customized_email(self, table, field, condition, subject, logact = 0):

# email: full user email address
# get user real name from table ruser for a given email address
# opts == 1 : include email
# opts == 2 : include org_type
# opts == 4 : include country
# opts == 8 : include valid_email
# opts == 16 : include org
# opts == 1: include email
# opts == 2: include org_type
# opts == 4: include country
# opts == 8: include valid_email
# opts == 16: include org
def get_ruser_names(self, email, opts = 0, date = None):
fields = "lname lstname, fname fstname"
if opts&1: fields += ", email"
Expand Down Expand Up @@ -1859,9 +1854,9 @@ def read_openbao(self):
self.DBBAOS[dbname] = {}
url = 'https://bao.k8s.ucar.edu/'
baopath = {
'ivaddb' : 'gdex/pgdb03',
'ispddb' : 'gdex/pgdb03',
'default' : 'gdex/pgdb01'
'ivaddb': 'gdex/pgdb03',
'ispddb': 'gdex/pgdb03',
'default': 'gdex/pgdb01'
}
dbpath = baopath[dbname] if dbname in baopath else baopath['default']
client = hvac.Client(url=self.PGDBI.get('BAOURL'))
Expand Down
Loading