# -*- coding: utf-8 -*-
#!/usr/bin/python
##-------------------------------------------------------------------
## @copyright 2017 DennyZhang.com
## Licensed under MIT 
##   https://raw.githubusercontent.com/DennyZhang/devops_public/master/LICENSE
##
## File : check_out_of_memory.py
## Author : Denny <denny@dennyzhang.com>
## Description : Detect whether OOM(Out Of Memory) has happened in the previous several hours
## --
## Created : <2017-02-28>
## Updated: Time-stamp: <2017-03-14 15:25:13>
##-------------------------------------------------------------------
# Check: http://www.dennyzhang.com/monitor_oom/
import argparse
import platform
import sys
import subprocess
import time

NAGIOS_OK_ERROR=0
NAGIOS_EXIT_ERROR=2

def get_time_seconds_from_dmsg(dmsg_entry):
    # From: [Sat Mar 11 00:19:44 2017] java invoked oom-killer: gfp_mask=0x26000c0, order=2, oom_score_adj=-17
    # To: 1489191584
    l = dmsg_entry.split("] ")
    date = l[0][1:]
    return int(time.mktime(time.strptime(date,'%a %b %d %H:%M:%S %Y')))

def get_oom_entry():
    oom_list = []
    # Sample output
    '''
root@bematech-es-1:~# dmesg -T | grep -i oom
[Sat Mar 11 00:19:44 2017] java invoked oom-killer: gfp_mask=0x26000c0, order=2, oom_score_adj=-17
[Sat Mar 11 00:19:44 2017]  [<ffffffff81188b35>] oom_kill_process+0x205/0x3d0
[Sat Mar 11 00:19:44 2017] [ pid ]   uid  tgid total_vm      rss nr_ptes nr_pmds swapents oom_score_adj name
    '''
    command = 'dmesg -T | grep -i oom'
    p = subprocess.Popen(command, shell=True, stderr=subprocess.PIPE)
    while True:
        out = p.stderr.read(1)
        if out == '' and p.poll() != None:
            break
        if out != '':
            # sys.stdout.write(out)
            # sys.stdout.flush()
            oom_list.append(out)
    return oom_list

def filter_entry_by_datetime(oom_list, hours_to_check):
    ret_list = []
    seconds_per_hour = 3600
    current_seconds = int(round(time.time()))
    for entry in oom_list:
        entry_seconds = get_time_seconds_from_dmsg(entry)
        if current_seconds <= entry_seconds + hours_to_check * seconds_per_hour:
            ret_list.append(entry)
    return ret_list

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--hours_to_check', required=False, default=4, type=int, \
                        help="Only check oom entries happen within previous several hours")
    l = parser.parse_args()
    hours_to_check = l.hours_to_check

    # Check OS release
    if platform.linux_distribution()[0] != 'Ubuntu':
        print "ERROR: current only support Ubuntu OS."
        sys.exit(NAGIOS_EXIT_ERROR)

    oom_list = get_oom_entry()
    matched_oom_list = filter_entry_by_datetime(oom_list, hours_to_check)
    if len(matched_oom_list) == 0:
        print "OK: No OOM has happened in previous %d hours." % (hours_to_check)
    else:
        print "ERROR: OOM has happened in previous %d hours.\n%s" % \
            (hours_to_check, "\n".join(matched_oom_list))
        sys.exit(NAGIOS_EXIT_ERROR)

    sys.exit(NAGIOS_OK_ERROR)
## File : check_out_of_memory.py ends

@@ -487,13 +487,13 @@
  15:
-25:13
+31:35
 %3E%0A##
@@ -658,25 +658,23 @@
 %0Aimport 
-subproces
+command
 s%0Aimport
@@ -1044,26 +1044,8 @@
 ():%0A
-    oom_list = %5B%5D%0A
     
@@ -1454,325 +1454,89 @@
     
-p = subprocess.Popen(command, shell=True, stderr=subprocess.PIPE)%0A    while True:%0A        out = p.stderr.read(1)%0A        if o
+_status, outp
 ut =
-= '' and p.poll() != None:%0A            break%0A        if out != '':%0A            # sys.stdout.write(out)%0A            # sys.stdout.flush()%0A            oom_list.append(out)%0A    return oom_list
+ commands.getstatusoutput(command)%0A    return output.split(%22%5Cn%22)
 %0A%0Ade
@@ -1762,24 +1762,116 @@
 dmsg(entry)%0A
+        # print %22current_seconds: %25d, entry_seconds: %25d%22 %25 (current_seconds, entry_seconds)%0A
         if c
@@ -2129,17 +2129,18 @@
 default=
-4
+10
 , type=i

change default time