~carstengrohmann/OOMAnalyser

6528b96adb0c31a6fbce9904275f902234f571a9 — Carsten Grohmann a month ago dd8b806
Restructure regex for better error messages
2 files changed, 121 insertions(+), 131 deletions(-)

M OOMAnalyser.py
M test.py
M OOMAnalyser.py => OOMAnalyser.py +112 -121
@@ 353,115 353,115 @@ class OOMEntity(object):
class OOMAnalyser(object):
    """Analyse an OOM object and calculate additional values"""

    REC_INVOKED_OOMKILLER = re.compile(
        r'^(?P<trigger_proc_name>[\S ]+) invoked oom-killer: '
        r'gfp_mask=(?P<trigger_proc_gfp_mask>0x[a-z0-9]+)(\((?P<trigger_proc_gfp_flags>[A-Z_|]+)\))?, '
        r'(nodemask=(?P<trigger_proc_nodemask>([\d,-]+|\(null\))), )?'
        r'order=(?P<trigger_proc_order>\d+), '
        r'oom_score_adj=(?P<trigger_proc_oomscore>\d+)',
        re.MULTILINE)

    REC_PID_KERNELVERSION = re.compile(
        r'^CPU: \d+ PID: (?P<trigger_proc_pid>\d+) '
        r'Comm: .* (Not tainted|Tainted:.*) '
        r'(?P<kernel_version>\d[\w.-]+) #\d',
        re.MULTILINE
    )

    # split caused by a limited number of iterations during converting PY regex into JS regex
    REC_MEMINFO_1 = re.compile(
        # head line
        r'^Mem-Info:.*'

        # first line break
        r'(?:\n)'

        # first line (starting with a space)
        r'^active_anon:(?P<active_anon_pages>\d+) inactive_anon:(?P<inactive_anon_pages>\d+) '
        r'isolated_anon:(?P<isolated_anon_pages>\d+)'

        # next line break
        r'(?:\n)'

        # remaining lines (with leading space)
        r'^ active_file:(?P<active_file_pages>\d+) inactive_file:(?P<inactive_file_pages>\d+) '
        r'isolated_file:(?P<isolated_file_pages>\d+)'

        # next line break
        r'(?:\n)'

        r'^ unevictable:(?P<unevictable_pages>\d+) dirty:(?P<dirty_pages>\d+) writeback:(?P<writeback_pages>\d+) '
        r'unstable:(?P<unstable_pages>\d+)'

        # # next line break
        # r'(?:\n)'
        #
        , re.MULTILINE
    )

    REC_MEMINFO_2 = re.compile(
        r'^ slab_reclaimable:(?P<slab_reclaimable_pages>\d+) slab_unreclaimable:(?P<slab_unreclaimable_pages>\d+)'
        r'(?:\n)'
        r'^ mapped:(?P<mapped_pages>\d+) shmem:(?P<shmem_pages>\d+) pagetables:(?P<pagetables_pages>\d+) '
        r'bounce:(?P<bounce_pages>\d+)'
        r'(?:\n)'
        r'^ free:(?P<free_pages>\d+) free_pcp:(?P<free_pcp_pages>\d+) free_cma:(?P<free_cma_pages>\d+)',
        re.MULTILINE
    )

    REC_MEM_NODEINFO = re.compile(r'(^Node \d+ (DMA|Normal|hugepages).*(:?\n))+', re.MULTILINE)

    REC_PAGECACHE = re.compile(r'^(?P<pagecache_total_pages>\d+) total pagecache pages.*$', re.MULTILINE)

    REC_SWAP = re.compile(
        r'^(?P<swap_cache_pages>\d+) pages in swap cache'
        r'(?:\n)'
        r'^Swap cache stats: add \d+, delete \d+, find \d+\/\d+'
        r'(?:\n)'
        r'^Free swap  = (?P<swap_free_kb>\d+)kB'
        r'(?:\n)'
        r'^Total swap = (?P<swap_total_kb>\d+)kB',
        re.MULTILINE)

    REC_PAGEINFO = re.compile(
        r'^(?P<ram_pages>\d+) pages RAM'
        r'('
        r'(?:\n)'
        r'^(?P<highmem_pages>\d+) pages HighMem/MovableOnly'
        r')?'
        r'(?:\n)'
        r'^(?P<reserved_pages>\d+) pages reserved'
        r'('
        r'(?:\n)'
        r'^(?P<cma_pages>\d+) pages cma reserved'
        r')?'
        r'('
        r'(?:\n)'
        r'^(?P<pagetablecache_pages>\d+) pages in pagetable cache'
        r')?'
        r'('
        r'(?:\n)'
        r'^(?P<hwpoisoned_pages>\d+) pages hwpoisoned'
        r')?',
        re.MULTILINE)
    EXTRACT_PATTERN = {
        'invoked oom-killer': (
            r'^(?P<trigger_proc_name>[\S ]+) invoked oom-killer: '
            r'gfp_mask=(?P<trigger_proc_gfp_mask>0x[a-z0-9]+)(\((?P<trigger_proc_gfp_flags>[A-Z_|]+)\))?, '
            r'(nodemask=(?P<trigger_proc_nodemask>([\d,-]+|\(null\))), )?'
            r'order=(?P<trigger_proc_order>\d+), '
            r'oom_score_adj=(?P<trigger_proc_oomscore>\d+)',
            True,
        ),
        'Trigger process and kernel version': (
            r'^CPU: \d+ PID: (?P<trigger_proc_pid>\d+) '
            r'Comm: .* (Not tainted|Tainted:.*) '
            r'(?P<kernel_version>\d[\w.-]+) #\d',
            True,
        ),

        # split caused by a limited number of iterations during converting PY regex into JS regex
        'Mem-Info (part 1)': (
            r'^Mem-Info:.*'
            r'(?:\n)'

            # first line (starting w/o a space)
            r'^active_anon:(?P<active_anon_pages>\d+) inactive_anon:(?P<inactive_anon_pages>\d+) '
            r'isolated_anon:(?P<isolated_anon_pages>\d+)'
            r'(?:\n)'

            # remaining lines (w/ leading space)
            r'^ active_file:(?P<active_file_pages>\d+) inactive_file:(?P<inactive_file_pages>\d+) '
            r'isolated_file:(?P<isolated_file_pages>\d+)'
            r'(?:\n)'

            r'^ unevictable:(?P<unevictable_pages>\d+) dirty:(?P<dirty_pages>\d+) writeback:(?P<writeback_pages>\d+) '
            r'unstable:(?P<unstable_pages>\d+)',
            True,
        ),
        'Mem-Info (part 2)': (
            r'^ slab_reclaimable:(?P<slab_reclaimable_pages>\d+) slab_unreclaimable:(?P<slab_unreclaimable_pages>\d+)'
            r'(?:\n)'
            r'^ mapped:(?P<mapped_pages>\d+) shmem:(?P<shmem_pages>\d+) pagetables:(?P<pagetables_pages>\d+) '
            r'bounce:(?P<bounce_pages>\d+)'
            r'(?:\n)'
            r'^ free:(?P<free_pages>\d+) free_pcp:(?P<free_pcp_pages>\d+) free_cma:(?P<free_cma_pages>\d+)',
            True,
        ),
        'Memory node information': (
             r'(^Node \d+ (DMA|Normal|hugepages).*(:?\n))+',
            False,
        ),
        'Page cache': (
            r'^(?P<pagecache_total_pages>\d+) total pagecache pages.*$',
            True,
        ),
        'Swap usage information': (
            r'^(?P<swap_cache_pages>\d+) pages in swap cache'
            r'(?:\n)'
            r'^Swap cache stats: add \d+, delete \d+, find \d+\/\d+'
            r'(?:\n)'
            r'^Free swap  = (?P<swap_free_kb>\d+)kB'
            r'(?:\n)'
            r'^Total swap = (?P<swap_total_kb>\d+)kB',
            True,
        ),
        'Page information': (
            r'^(?P<ram_pages>\d+) pages RAM'
            r'('
            r'(?:\n)'
            r'^(?P<highmem_pages>\d+) pages HighMem/MovableOnly'
            r')?'
            r'(?:\n)'
            r'^(?P<reserved_pages>\d+) pages reserved'
            r'('
            r'(?:\n)'
            r'^(?P<cma_pages>\d+) pages cma reserved'
            r')?'
            r'('
            r'(?:\n)'
            r'^(?P<pagetablecache_pages>\d+) pages in pagetable cache'
            r')?'
            r'('
            r'(?:\n)'
            r'^(?P<hwpoisoned_pages>\d+) pages hwpoisoned'
            r')?',
            True,
        ),
        'Process killed by OOM': (
            r'^Out of memory: Kill process (?P<killed_proc_pid>\d+) \((?P<killed_proc_name>[\S ]+)\) '
            r'score (?P<killed_proc_score>\d+) or sacrifice child',
            True,
        ),
        'Details of process killed by OOM': (
            r'^Killed process \d+ \(.*\)'
            r'(, UID \d+,)?'
            r' total-vm:(?P<killed_proc_total_vm_kb>\d+)kB, anon-rss:(?P<killed_proc_anon_rss_kb>\d+)kB, '
            r'file-rss:(?P<killed_proc_file_rss_kb>\d+)kB, shmem-rss:(?P<killed_proc_shmem_rss_kb>\d+)kB.*',
            True,
        ),
    }
    """
    RE pattern to extract information from OOM.
    
    The first item is the RE pattern and the second is whether it is mandatory to find this pattern.
    
    :type: dict(tuple(str, bool))
    """

    REC_PROCESS_LINE = re.compile(
        r'^\[(?P<pid>[ \d]+)\]\s+(?P<uid>\d+)\s+(?P<tgid>\d+)\s+(?P<total_vm_pages>\d+)\s+(?P<rss_pages>\d+)\s+'
        r'(?P<nr_ptes_pages>\d+)\s+(?P<swapents_pages>\d+)\s+(?P<oom_score_adj>-?\d+)\s+(?P<name>.+)\s*')

    REC_OOM_KILL_PROCESS = re.compile(
        r'^Out of memory: Kill process (?P<killed_proc_pid>\d+) \((?P<killed_proc_name>[\S ]+)\) '
        r'score (?P<killed_proc_score>\d+) or sacrifice child',
        re.MULTILINE
    )

    REC_KILLED_PROCESS = re.compile(
        r'^Killed process \d+ \(.*\)'
        r'(, UID \d+,)?'
        r' total-vm:(?P<killed_proc_total_vm_kb>\d+)kB, anon-rss:(?P<killed_proc_anon_rss_kb>\d+)kB, '
        r'file-rss:(?P<killed_proc_file_rss_kb>\d+)kB, shmem-rss:(?P<killed_proc_shmem_rss_kb>\d+)kB.*',
        re.MULTILINE)

    lines = []
    """All lines of an OOM without leading timestamps"""



@@ 500,26 500,17 @@ class OOMAnalyser(object):
        """Extract details from OOM message text"""

        self.results = {}

        for rec in [self.REC_INVOKED_OOMKILLER,
                    self.REC_KILLED_PROCESS,
                    self.REC_MEMINFO_1,
                    self.REC_MEMINFO_2,
                    self.REC_OOM_KILL_PROCESS,
                    self.REC_PAGECACHE,
                    self.REC_PAGEINFO,
                    self.REC_PID_KERNELVERSION,
                    self.REC_SWAP,
                    ]:
        # __pragma__ ('jsiter')
        for k in self.EXTRACT_PATTERN:
            pattern, is_mandatory = self.EXTRACT_PATTERN[k]
            rec = re.compile(pattern, re.MULTILINE)
            match = rec.search(self.oom_entity.text)
            if match:
                self.results.update(match.groupdict())
            else:
                warning('No match for regex: "{}"'.format(rec.pattern))

        match = self.REC_MEM_NODEINFO.search(self.oom_entity.text)
        if match:
            self.results['mem_node_info'] = match.group()
            elif is_mandatory:
                error('Failed to extract information from OOM text. The regular expression "{}" (pattern "{}") '
                      'does not find anything. This will cause subsequent errors.'.format(k, pattern))
        # __pragma__ ('nojsiter')

        self.results['hardware_info'] = self._extract_block_from_next_pos('Hardware name:')


M test.py => test.py +9 -10
@@ 20,6 20,7 @@
import http.server
import logging
import os
import re
import socketserver
import threading
import unittest


@@ 286,28 287,26 @@ class TestPython(TestBase):
    def test_001_trigger_proc_space(self):
        """Test RE to find name of trigger process"""
        first = self.get_first_line(OOMAnalyser.OOMDisplay.example)
        rec = OOMAnalyser.OOMAnalyser.REC_INVOKED_OOMKILLER
        pattern = OOMAnalyser.OOMAnalyser.EXTRACT_PATTERN['invoked oom-killer'][0]
        rec = re.compile(pattern, re.MULTILINE)
        match = rec.search(first)
        self.assertTrue(match, 'Error: re.search(REC_INVOKED_OOMKILLER) failed for simple '
                               'process name')
        self.assertTrue(match, "Error: re.search('invoked oom-killer') failed for simple process name")

        first = first.replace('sed', 'VM Monitoring Task')
        match = rec.search(first)
        self.assertTrue(match, 'Error: re.search(REC_INVOKED_OOMKILLER) failed for process name '
                               'with space')
        self.assertTrue(match, "Error: re.search('invoked oom-killer') failed for process name with space")

    def test_002_killed_proc_space(self):
        """Test RE to find name of killed process"""
        last = self.get_last_line(OOMAnalyser.OOMDisplay.example)
        rec = OOMAnalyser.OOMAnalyser.REC_OOM_KILL_PROCESS
        pattern = OOMAnalyser.OOMAnalyser.EXTRACT_PATTERN['Process killed by OOM'][0]
        rec = re.compile(pattern, re.MULTILINE)
        match = rec.search(last)
        self.assertTrue(match, 'Error: re.search(REC_OOM_KILL_PROCESS) failed for simple '
                               'process name')
        self.assertTrue(match, "Error: re.search('Process killed by OOM') failed for simple process name")

        last = last.replace('sed', 'VM Monitoring Task')
        match = rec.search(last)
        self.assertTrue(match, 'Error: re.search(REC_OOM_KILL_PROCESS) failed for process name '
                               'with space')
        self.assertTrue(match, "Error: re.search('Process killed by OOM') failed for process name with space")

    def test_003_OOMEntity_number_of_columns_to_strip(self):
        """Test stripping useless / leading columns"""