Add qmonitor

Author: Scott Wales

Diff for: qmonitor

@@ -0,0 +1,97 @@
+#!/g/data/hh5/public/apps/nci_scripts/python-analysis3
+# Copyright 2020 Scott Wales
+# author: Scott Wales <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Monitor a PBS queue job
+"""
+
+import argparse
+import tqdm
+import subprocess
+import json
+import time
+import pandas
+import importlib
+from qtools import qstat, decode_bytes
+
+
+def metrics(stat):
+    cpu = 0
+    mem = 0
+    wall = 0
+
+    if stat['job_state'] == 'R':
+        mem_request = decode_bytes(stat['Resource_List']['mem'])
+        mem_used = decode_bytes(stat['resources_used']['mem'])
+        mem = mem_used / mem_request * 100
+
+        cpu_time = pandas.to_timedelta(stat['resources_used']['cput'])
+        wall_used = pandas.to_timedelta(stat['resources_used']['walltime'])
+        ncpus = stat['Resource_List']['ncpus']
+
+        cpu = cpu_time / wall_used / ncpus * 100
+
+        wall = wall_used.seconds
+
+    return cpu, mem, wall
+
+
+def monitor(jobid):
+    jobid, stat = list(qstat([jobid]).items())[0]
+    c, m, w = metrics(stat)
+
+    wall_request = pandas.to_timedelta(stat['Resource_List']['walltime']).seconds
+
+    print(f"{jobid} monitor")
+
+    bar_format = '{l_bar}{bar}'
+    cpu = tqdm.tqdm(unit="%", desc="CPU", total=100.0, position=0, bar_format=bar_format, leave=True)
+    mem = tqdm.tqdm(unit="%", desc="MEM", total=100.0, position=1, bar_format=bar_format, leave=True)
+    wall = tqdm.tqdm(unit='s', desc="TIME", total=wall_request, position=2, bar_format=bar_format, leave=True)
+
+    while stat["job_state"] in ["Q", "R"]:
+        c, m, w = metrics(stat)
+
+        cpu.reset()
+        cpu.update(c)
+        cpu.refresh()
+
+        mem.reset()
+        mem.update(m)
+        mem.refresh()
+
+        wall.reset()
+        wall.update(w)
+        wall.refresh()
+
+        time.sleep(10)
+        stat = list(qstat([jobid]).values())[0]
+
+    cpu.close()
+    mem.close()
+    wall.close()
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("jobid")
+    args = parser.parse_args()
+
+    monitor(args.jobid)
+
+
+if __name__ == "__main__":
+    main()

Diff for: qtools.py

@@ -0,0 +1,86 @@
+#!/g/data/hh5/public/apps/nci_scripts/python-analysis3
+# Copyright 2020 Scott Wales
+# author: Scott Wales <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import typing as T
+import pandas
+import re
+import subprocess
+import json
+
+def decode_bytes(s):
+    """
+    Convert a formatted size to number of bytes
+    """
+    if pandas.isnull(s):
+        return s
+
+    scales = {
+        'k': 1024,
+        }
+    if not s.endswith('b'):
+        raise Exception(f"{s} doesn't look like a size")
+
+    scale = 1
+    s = s[:-1]
+
+    if not s[-1].isdigit():
+        scale = scales[s[-1]]
+        s = s[:-1]
+
+    return int(s) * scale
+
+
+def clean_qstat_json(stream):
+    """
+    Clean up the improperly escaped JSON returned by qstat
+    """
+    string_entry_re = re.compile(r'^\s*"(?P<key>.+)":"(?P<value>.+)"(?P<comma>,?)$')
+
+    lines = []
+
+    for line in stream.splitlines():
+        match = string_entry_re.match(line)
+        if match is not None:
+            fixed_value = json.dumps(match.group('value'))
+            line = f'"{match.group("key")}":{fixed_value}{match.group("comma")}'
+
+        lines.append(line)
+
+    return json.loads(''.join(lines))
+
+
+def qstat(jobids: T.List[str], show_finished: bool=False):
+    """
+    Returns the information from qstat
+    """
+
+    extra_args = []
+    if show_finished:
+        extra_args.append('-x')
+
+    subp = subprocess.run(
+        ["/opt/pbs/default/bin/qstat", *extra_args, "-f", "-F", "json", *jobids],
+        text=True,
+        check=True,
+        stdout=subprocess.PIPE,
+    )
+
+    jobs = clean_qstat_json(subp.stdout)["Jobs"]
+
+    return jobs
+
+

Diff for: uqstat

@@ -8,6 +8,7 @@ import re
 import sys
 import pymunge
 import requests
+from qtools import *
 
 # Base charge rates in SU/hour for 1 cpu with minimal memory
 charge_rates = {
@@ -37,25 +38,6 @@ base_mem = {
     'normalsl': 192/32,
     }
 
-def decode_bytes(s):
-    if pandas.isnull(s):
-        return s
-
-    scales = {
-        'k': 1024,
-        }
-    if not s.endswith('b'):
-        raise Exception(f"{s} doesn't look like a size")
-
-    scale = 1
-    s = s[:-1]
-
-    if not s[-1].isdigit():
-        scale = scales[s[-1]]
-        s = s[:-1]
-
-    return int(s) * scale
-
 
 def maybe_get(x, key):
     if pandas.isnull(x):
@@ -72,25 +54,6 @@ def get_list(s, lst, key):
     except AttributeError:
         return None
 
-
-def clean_qstat_json(stream):
-    """
-    Clean up the improperly escaped JSON returned by qstat
-    """
-    string_entry_re = re.compile(r'^\s*"(?P<key>.+)":"(?P<value>.+)"(?P<comma>,?)$')
-
-    lines = []
-
-    for line in stream.splitlines():
-        match = string_entry_re.match(line)
-        if match is not None:
-            fixed_value = json.dumps(match.group('value'))
-            line = f'"{match.group("key")}":{fixed_value}{match.group("comma")}'
-
-        lines.append(line)
-
-    return json.loads(''.join(lines))
-
 def qstat_df(historical=False):
     command= ['/opt/pbs/default/bin/qstat','-f','-F','json']