~carstengrohmann/OOMAnalyser

41a51a7e2ad63307a0f38b5626d1b16255a3c84c — Carsten Grohmann a month ago 1abf455
Rework removal of unused information

The columns left to the oom are not used. With this change, the
algorithm becomes more robust and can better handle known special
cases like "kernel:<processname>".
3 files changed, 100 insertions(+), 48 deletions(-)

M OOMAnalyser.html
M OOMAnalyser.py
M test.py
M OOMAnalyser.html => OOMAnalyser.html +1 -0
@@ 818,6 818,7 @@ function read_and_display_file(file) {
        <li>Improve SVG chart colour palette</li>
        <li>Add Selenium based unit tests</li>
        <li>Fix to allow process names with spaces</li>
        <li>Rework removal of unused information</li>
        <li>...</li>
    </ol>


M OOMAnalyser.py => OOMAnalyser.py +21 -17
@@ 201,8 201,9 @@ class OOMEntity(object):
            return

        oom_lines = self._remove_non_oom_lines(oom_lines)
        oom_lines = self._strip_needless_columns(oom_lines)
        oom_lines = self._remove_kernel_colon(oom_lines)
        oom_lines = self._rsyslog_unescape_lf(oom_lines)
        oom_lines = self._strip_needless_columns(oom_lines)

        self.lines = oom_lines
        self.text = '\n'.join(oom_lines)


@@ 223,21 224,13 @@ class OOMEntity(object):
        columns = first_line.split(" ")

        # Examples:
        # [11686.888109] sed invoked oom-killer: gfp_mask=0x201da, order=0, oom_adj=0, oom_score_adj=0
        # Apr 01 14:13:32 mysrv kernel: sed invoked OOM-killer: gfp_mask=0x201da, order=0
        # Apr 01 14:13:32 mysrv kernel: [11686.888109] sed invoked oom-killer: gfp_mask=0x84d0, order=0, oom_adj=0, oom_score_adj=0
        # [11686.888109] CPU: 4 PID: 29481 Comm: sed Not tainted 3.10.0-514.6.1.el7.x86_64 #1
        # Apr 01 14:13:32 mysrv kernel: CPU: 4 PID: 29481 Comm: sed Not tainted 3.10.0-514.6.1.el7.x86_64 #1
        # Apr 01 14:13:32 mysrv kernel: [11686.888109] CPU: 4 PID: 29481 Comm: sed Not tainted 3.10.0-514.6.1.el7.x86_64 #1
        try:
            # strip all incl. "kernel:"
            if 'kernel:' in first_line:
                to_strip = columns.index("kernel:")
                # increase to include "kernel:"
                to_strip += 1

            # check if next column is a timestamp like "[11686.888109]" and remove it too
            rec = re.compile('\[\d+\.\d+\]')
            if rec.match(columns[to_strip]):
                # increase to include timestamp
                to_strip += 1
            # strip all excl. "CPU:"
            if 'CPU:' in first_line:
                to_strip = columns.index("CPU:")
        except ValueError:
            pass



@@ 283,6 276,16 @@ class OOMEntity(object):

        return lines

    def _remove_kernel_colon(self, oom_lines):
        """
        Remove the "kernel:" pattern w/o leading and tailing spaces.

        Some OOM messages don't have a space between "kernel:" and the process name. _strip_needless_columns() will
        fail in such cases. Therefore the pattern is removed.
        """
        oom_lines = [i.replace('kernel:', '') for i in oom_lines]
        return oom_lines

    def _strip_needless_columns(self, oom_lines):
        """
        Remove needless columns at the start of every line.


@@ 291,7 294,7 @@ class OOMEntity(object):
        syslog priority/facility.
        """
        stripped_lines = []
        cols_to_strip = self._number_of_columns_to_strip(oom_lines[0])
        cols_to_strip = self._number_of_columns_to_strip(oom_lines[2])

        for line in oom_lines:
            # remove empty lines


@@ 412,7 415,7 @@ class OOMAnalyser(object):
    REC_SWAP = re.compile(
        r'^(?P<swap_cache_pages>\d+) pages in swap cache'
        r'(?:\n)'
        r'^Swap cache stats: add \d+, delete \d+, find \d+/\d+'
        r'^Swap cache stats: add \d+, delete \d+, find \d+\/\d+'
        r'(?:\n)'
        r'^Free swap  = (?P<swap_free_kb>\d+)kB'
        r'(?:\n)'


@@ 989,6 992,7 @@ Killed process 6576 (mysqld) total-vm:33914892kB, anon-rss:20629004kB, file-rss:
    """SVG graphics with one black triangle DOWN for sorting"""

    def __init__(self):
        self.oom = None
        self.set_HTML_defaults()
        self.update_toc()


M test.py => test.py +78 -31
@@ 18,6 18,7 @@
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

import http.server
import logging
import os
import socketserver
import threading


@@ 101,13 102,21 @@ class TestInBrowser(TestBase):

    def assert_on_warn(self):
        notify_box = self.driver.find_element_by_id('notify_box')
        with self.assertRaises(NoSuchElementException):
            notify_box.find_element_by_class_name('js-notify_box__msg--warning')
        try:
            warning = notify_box.find_element_by_class_name('js-notify_box__msg--warning')
        except NoSuchElementException:
            pass
        else:
            self.fail('Unexpected warning message: %s' % warning.text)

    def assert_on_error(self):
        notify_box = self.driver.find_element_by_id('notify_box')
        with self.assertRaises(NoSuchElementException):
            notify_box.find_element_by_class_name('js-notify_box__msg--error')
        try:
            error = notify_box.find_element_by_class_name('js-notify_box__msg--error')
        except NoSuchElementException:
            pass
        else:
            self.fail('Unexpected error message: %s' % error.text)

        for event in self.driver.get_log('browser'):
            # ignore favicon.ico errors


@@ 115,13 124,23 @@ class TestInBrowser(TestBase):
                continue
            self.fail('Error on browser console reported: %s' % event)

    def assert_on_warn_error(self):
        self.assert_on_warn()
        self.assert_on_error()

    def click_analyse(self):
        analyse = self.driver.find_element_by_xpath('//button[text()="Analyse"]')
        analyse.click()

    def click_reset(self):
        # OOMAnalyser.OOMDisplayInstance.reset_form()
        reset = self.driver.find_element_by_xpath('//button[text()="Reset"]')
        reset.click()
        if reset.is_displayed():
            reset.click()
        else:
            new_analysis = self.driver.find_element_by_xpath('//a[contains(text(), "Step 1 - Enter your OOM message")]')
            # new_analysis = self.driver.find_element_by_link_text('Run a new analysis')
            new_analysis.click()
        self.assert_on_warn_error()

    def analyse_oom(self, text):


@@ 141,9 160,28 @@ class TestInBrowser(TestBase):

        self.click_analyse()

    def assert_on_warn_error(self):
        self.assert_on_warn()
        self.assert_on_error()
    def check_results(self):
        """Check the results of the analysis of the default example"""
        self.assert_on_warn_error()
        h3_summary = self.driver.find_element_by_xpath('//h3[text()="Summary"]')
        self.assertTrue(h3_summary.is_displayed(), "Analysis details incl. <h3>Summary</h3> should be displayed")

        trigger_proc_name = self.driver.find_element_by_class_name('trigger_proc_name')
        self.assertEqual(trigger_proc_name.text, 'sed', 'Unexpected trigger process name')
        trigger_proc_pid = self.driver.find_element_by_class_name('trigger_proc_pid')
        self.assertEqual(trigger_proc_pid.text, '29481', 'Unexpected trigger process pid')

        killed_proc_score = self.driver.find_element_by_class_name('killed_proc_score')
        self.assertEqual(killed_proc_score.text, '651', 'Unexpected OOM score of killed process')

        swap_cache_kb = self.driver.find_element_by_class_name('swap_cache_kb')
        self.assertEqual(swap_cache_kb.text, '45368 kBytes')
        swap_used_kb = self.driver.find_element_by_class_name('swap_used_kb')
        self.assertEqual(swap_used_kb.text, '8343236 kBytes')
        swap_free_kb = self.driver.find_element_by_class_name('swap_free_kb')
        self.assertEqual(swap_free_kb.text, '0 kBytes')
        swap_total_kb = self.driver.find_element_by_class_name('swap_total_kb')
        self.assertEqual(swap_total_kb.text, '8388604 kBytes')

    def test_001_load_page(self):
        """Test if the page is loading"""


@@ 166,32 204,14 @@ class TestInBrowser(TestBase):
        self.assertFalse(h3_summary.is_displayed(), "Analysis details incl. <h3>Summary</h3> should be not displayed")

        self.click_analyse()

        self.assert_on_warn_error()
        self.assertTrue(h3_summary.is_displayed(), "Analysis details incl. <h3>Summary</h3> should be displayed")

        trigger_proc_name = self.driver.find_element_by_class_name('trigger_proc_name')
        self.assertEqual(trigger_proc_name.text, 'sed', 'Unexpected trigger process name')
        trigger_proc_pid = self.driver.find_element_by_class_name('trigger_proc_pid')
        self.assertEqual(trigger_proc_pid.text, '29481', 'Unexpected trigger process pid')

        killed_proc_score = self.driver.find_element_by_class_name('killed_proc_score')
        self.assertEqual(killed_proc_score.text, '651', 'Unexpected OOM score of killed process')

        swap_cache_kb = self.driver.find_element_by_class_name('swap_cache_kb')
        self.assertEqual(swap_cache_kb.text, '45368 kBytes')
        swap_used_kb = self.driver.find_element_by_class_name('swap_used_kb')
        self.assertEqual(swap_used_kb.text, '8343236 kBytes')
        swap_free_kb = self.driver.find_element_by_class_name('swap_free_kb')
        self.assertEqual(swap_free_kb.text, '0 kBytes')
        swap_total_kb = self.driver.find_element_by_class_name('swap_total_kb')
        self.assertEqual(swap_total_kb.text, '8388604 kBytes')
        self.check_results()

    def test_004_begin_but_no_end(self):
        """Test incomplete OOM text - just the beginning"""
        example = """\
sed invoked oom-killer: gfp_mask=0x201da, order=0, oom_score_adj=0
sed cpuset=/ mems_allowed=0-1
CPU: 4 PID: 29481 Comm: sed Not tainted 3.10.0-514.6.1.el7.x86_64 #1
        """
        self.analyse_oom(example)



@@ 239,6 259,33 @@ Killed process 6576 (java) total-vm:33914892kB, anon-rss:20629004kB, file-rss:0k
        h3_summary = self.driver.find_element_by_xpath('//h3[text()="Summary"]')
        self.assertTrue(h3_summary.is_displayed(), "Analysis details incl. <h3>Summary</h3> should be displayed")

    def test_removal_of_leading_but_useless_columns(self):
        """Test removal of leading but useless columns"""
        self.analyse_oom(OOMAnalyser.OOMDisplay.example)
        self.check_results()
        self.click_reset()
        for prefix in ["[11686.888109] ",
                       "Apr 01 14:13:32 mysrv: ",
                       "Apr 01 14:13:32 mysrv kernel: ",
                       "Apr 01 14:13:32 mysrv <kern.warning> kernel: ",
                       "Apr 01 14:13:32 mysrv kernel: [11686.888109] ",
                       "kernel:",
                       "Apr 01 14:13:32 mysrv <kern.warning> kernel:",
                       ]:
            lines = OOMAnalyser.OOMDisplay.example.split('\n')
            lines = ["{}{}".format(prefix, line) for line in lines]
            oom_text = "\n".join(lines)
            self.analyse_oom(oom_text)

            try:
                self.check_results()
            except AssertionError:
                logging.error('prefix %s', prefix)
                import pdb; pdb.set_trace()
                oom = OOMAnalyser.OOMEntity(oom_text)
                print(oom.text)
            self.click_reset()


class TestPython(TestBase):



@@ 272,9 319,9 @@ class TestPython(TestBase):
        """Test stripping useless / leading columns"""
        oom_entity = OOMAnalyser.OOMEntity(OOMAnalyser.OOMDisplay.example)
        for pos, line in [
            (1, '[11686.888109] sed invoked oom-killer: gfp_mask=0x201da, order=0, oom_adj=0, oom_score_adj=0'),
            (5, 'Apr 01 14:13:32 mysrv kernel: sed invoked OOM-killer: gfp_mask=0x201da, order=0'),
            (6, 'Apr 01 14:13:32 mysrv kernel: [11686.888109] sed invoked oom-killer: gfp_mask=0x84d0, order=0, oom_adj=0, oom_score_adj=0'),
            (1, '[11686.888109] CPU: 4 PID: 29481 Comm: sed Not tainted 3.10.0-514.6.1.el7.x86_64 #1'),
            (5, 'Apr 01 14:13:32 mysrv kernel: CPU: 4 PID: 29481 Comm: sed Not tainted 3.10.0-514.6.1.el7.x86_64 #1'),
            (6, 'Apr 01 14:13:32 mysrv kernel: [11686.888109] CPU: 4 PID: 29481 Comm: sed Not tainted 3.10.0-514.6.1.el7.x86_64 #1'),
        ]:
            to_strip = oom_entity._number_of_columns_to_strip(line)
            self.assertEqual(to_strip, pos, 'Calc wrong number of columns to strip for "%s": got: %d, expect: %d' % (