Merge branch 'release/0.12.1'

Author: dan-blanchard

gridmap/job.py

@@ -508,7 +508,7 @@ def send_error_mail(job):
     if job.heart_beat:
         body_text += "Last memory usage: {}\n".format(job.heart_beat["memory"])
         body_text += "Last cpu load: {}\n".format(job.heart_beat["cpu_load"][0])
-        body_text += ("Last process state: " +
+        body_text += ("Process was running at last check: " +
                       "{}\n\n").format(job.heart_beat["cpu_load"][1])
 
     body_text += "Host: {}\n\n".format(job.host_name)
@@ -523,10 +523,8 @@ def send_error_mail(job):
     msg.attach(body_msg)
 
     # attach log file
-    if job.heart_beat and os.path.exists(job.heart_beat["log_file"]):
-        log_file_fn = job.heart_beat['log_file']
-        with open(log_file_fn, "rb") as log_file:
-            log_file_attachement = MIMEText(log_file.read())
+    if job.heart_beat and "log_file" in job.heart_beat:
+        log_file_attachement = MIMEText(job.heart_beat['log_file'])
         log_file_attachement.add_header('Content-Disposition', 'attachment',
                                         filename='{}_log.txt'.format(job.id))
         msg.attach(log_file_attachement)
@@ -602,8 +600,9 @@ def handle_resubmit(session_id, job, temp_dir='/scratch/'):
 
     if job.num_resubmits < NUM_RESUBMITS:
         logger = logging.getLogger(__name__)
-        logger.warning("Looks like job died an unnatural death, resubmitting" +
-                       "(previous resubmits = %i)", job.num_resubmits)
+        logger.warning("Looks like job %s (%s) died an unnatural death, " +
+                       "resubmitting (previous resubmits = %i)", job.name,
+                       job.id, job.num_resubmits)
 
         # remove node from white_list
         node_name = '{}@{}'.format(job.queue, job.host_name)

gridmap/runner.py

@@ -115,9 +115,12 @@ def get_memory_usage(pid, heart_pid):
     """
     process = psutil.Process(pid)
     mem_total = float(process.get_memory_info()[0])
-    mem_total += sum(float(p.get_memory_info()[0]) for p in
-                     process.get_children(recursive=True)
-                     if process.pid != heart_pid)
+    for p in process.get_children(recursive=True):
+        if p.is_running() and p.pid != heart_pid:
+            try:
+                mem_total += float(p.get_memory_info()[0])
+            except psutil.NoSuchProcess:
+                continue
     return mem_total / (1024.0 ** 2.0)
 
 
@@ -139,10 +142,15 @@ def get_cpu_load(pid, heart_pid):
     running = process.status not in _SLEEP_STATUSES
     num_procs = 1
     for p in process.get_children(recursive=True):
-        if p.pid != heart_pid:
-            cpu_sum += float(p.get_cpu_percent())
-            running = running or (p.status not in _SLEEP_STATUSES)
-            num_procs += 1
+        # Make sure this process hasn't exited before querying its status
+        if p.is_running() and p.pid != heart_pid:
+            try:
+                cpu_sum += float(p.get_cpu_percent())
+                running = running or (p.status not in _SLEEP_STATUSES)
+            except psutil.NoSuchProcess:
+                continue
+            else:
+                num_procs += 1
     return cpu_sum / num_procs, running
 
 

gridmap/version.py

@@ -28,5 +28,5 @@
 :organization: ETS
 '''
 
-__version__ = '0.12.0'
+__version__ = '0.12.1'
 VERSION = tuple(int(x) for x in __version__.split('.'))

gridmap/web.py

@@ -41,6 +41,7 @@
 
 from gridmap.data import clean_path
 from gridmap.runner import _send_zmq_msg
+from gridmap.version import __version__
 
 
 class WebMonitor(object):
@@ -148,6 +149,8 @@ def main(argv=None):
     parser.add_argument('-p', '--port',
                         help='Port for server to listen on.', type=int,
                         default=8076)
+    parser.add_argument('--version', action='version',
+                        version='%(prog)s {0}'.format(__version__))
     args = parser.parse_args(argv)
 
     # Make warnings from built-in warnings module get formatted more nicely