diff --git a/.coveragerc b/.coveragerc index 8af9056b0c6..501ba333762 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,13 +1,3 @@ [report] omit = */synapse/tests/test_* - -; Uncomment this section to enable code coverage of storm files in the -; storm_dirs directory listed below. This is disabled by default right now -; because it's pretty intensive and imposes a large perf hit on the already slow -; tests. -;[run] -;plugins = synapse.utils.stormcov - -[synapse.utils.stormcov] -storm_dirs = synapse/assets/storm diff --git a/synapse/lib/parser.py b/synapse/lib/parser.py index 4bb0a1f88d9..eede9b54349 100644 --- a/synapse/lib/parser.py +++ b/synapse/lib/parser.py @@ -271,8 +271,6 @@ def funccall(self, meta, kids): argkids = [] kwargkids = [] kwnames = set() - indx = 1 - kcnt = len(kids) todo = collections.deque(kids) diff --git a/synapse/tests/files/stormcov/argvquery.storm b/synapse/tests/files/stormcov/argvquery.storm index 3d2314179a3..d2020c4cd96 100644 --- a/synapse/tests/files/stormcov/argvquery.storm +++ b/synapse/tests/files/stormcov/argvquery.storm @@ -6,3 +6,5 @@ function foo() { } {} } yield $foo() +// coverage: 1, 2, 3, 4, 6, 8 +// lines: 1, 2, 3, 4, 8 diff --git a/synapse/tests/files/stormcov/argvquery2.storm b/synapse/tests/files/stormcov/argvquery2.storm new file mode 100644 index 00000000000..4d18aecf835 --- /dev/null +++ b/synapse/tests/files/stormcov/argvquery2.storm @@ -0,0 +1,4 @@ + +tee { inet:fqdn=argvquery2.storm } +// coverage: 2 +// lines: 2 diff --git a/synapse/tests/files/stormcov/argvquery3.storm b/synapse/tests/files/stormcov/argvquery3.storm new file mode 100644 index 00000000000..ce5893acd4a --- /dev/null +++ b/synapse/tests/files/stormcov/argvquery3.storm @@ -0,0 +1,6 @@ + +tee { + inet:fqdn=argvquery3.storm +} +// coverage: 2, 3 +// lines: 2, 3 diff --git a/synapse/tests/files/stormcov/argvquery4.storm b/synapse/tests/files/stormcov/argvquery4.storm new file mode 100644 index 00000000000..650a2a93a1c --- /dev/null +++ b/synapse/tests/files/stormcov/argvquery4.storm @@ -0,0 +1,7 @@ + +tee +{ + inet:fqdn=argvquery4.storm +} +// coverage: 2, 3, 4 +// lines: 2, 4 diff --git a/synapse/tests/files/stormcov/argvquery5.storm b/synapse/tests/files/stormcov/argvquery5.storm new file mode 100644 index 00000000000..de715cc6085 --- /dev/null +++ b/synapse/tests/files/stormcov/argvquery5.storm @@ -0,0 +1,11 @@ + +tee +{ + + + inet:fqdn=argvquery5.storm + + +} +// coverage: 2, 3, 6 +// lines: 2, 6 diff --git a/synapse/tests/files/stormcov/badstorm.storm b/synapse/tests/files/stormcov/badstorm.storm new file mode 100644 index 00000000000..93d51406d63 --- /dev/null +++ b/synapse/tests/files/stormcov/badstorm.storm @@ -0,0 +1 @@ +[{}] diff --git a/synapse/tests/files/stormcov/dupesubs.storm b/synapse/tests/files/stormcov/dupesubs.storm index 9b082696240..872374a6963 100644 --- a/synapse/tests/files/stormcov/dupesubs.storm +++ b/synapse/tests/files/stormcov/dupesubs.storm @@ -1,12 +1,28 @@ -$eq = ${ [inet:fqdn=bar.com] } -$eq = ${ [inet:fqdn=bar.com] } +$eq = ${ + [inet:fqdn=bar.com] +} +$eq = ${ + [inet:fqdn=bar.com] +} tee { [inet:fqdn=foo.com] } { [inet:fqdn=baz.com ] } $eq - +| tee { - [inet:fqdn=argv.com] + if (false) { + [inet:fqdn=argv.com] + } } { [inet:fqdn=argv.com] } +| +tee +{ + [ inet:fqdn=dup00.com ] +} +{ + [ inet:fqdn=dup00.com ] +} +// coverage: 1, 2, 4, 7, 8, 9, 10, 12, 13, 16, 17, 20, 21, 22, 24 +// lines: 1, 2, 4, 5, 7, 8, 9, 10, 12, 13, 14, 17, 20, 22, 25 diff --git a/synapse/tests/files/stormcov/dupewarn.storm b/synapse/tests/files/stormcov/dupewarn.storm new file mode 100644 index 00000000000..f8f17b77936 --- /dev/null +++ b/synapse/tests/files/stormcov/dupewarn.storm @@ -0,0 +1,39 @@ +tee ${ inet:fqdn=01.embed.dupewarn.com } ${ inet:fqdn=01.embed.dupewarn.com } +| +tee ${ + [ inet:fqdn=00.embed.dupewarn.com ] +} +${ + + + [ inet:fqdn=00.embed.dupewarn.com ] + + +} +| +tee +${ inet:fqdn=02.embed.dupewarn.com } +${ + inet:fqdn=02.embed.dupewarn.com +} +// embddups: 1:1, 6:3, 16:15 +| +tee { + [ inet:fqdn=00.argv.dupewarn.com ] +} +{ + + + [ inet:fqdn=00.argv.dupewarn.com ] + + +} +| +tee { inet:fqdn=01.argv.dupewarn.com } { inet:fqdn=01.argv.dupewarn.com } +| +tee +{ inet:fqdn=02.argv.dupewarn.com } +{ + inet:fqdn=02.argv.dupewarn.com +} +// argvdups: 24:21, 32:32, 36:35 diff --git a/synapse/tests/files/stormcov/embedquery.storm b/synapse/tests/files/stormcov/embedquery.storm new file mode 100644 index 00000000000..d3a515267d8 --- /dev/null +++ b/synapse/tests/files/stormcov/embedquery.storm @@ -0,0 +1,21 @@ +$eq1 = ${ + [ inet:fqdn=vertex.link ] +} +$eq2 = ${ + if (true) { + [ inet:fqdn=vertex.link ] + } +} +$eq3 = ${ + if (false) { + [ inet:fqdn=vertex.link ] + } +} +$eq4 = ${ [ inet:fqdn=vtx.lk ] } +$eq5 = ${ + + [ inet:fqdn=v.vtx.lk ] +} +tee $eq1 $eq2 $eq3 $eq4 $eq5 +// coverage: 1, 2, 4, 5, 6, 9, 10, 14, 15, 17, 19 +// lines: 1, 2, 4, 5, 6, 9, 10, 11, 14, 15, 17, 19 diff --git a/synapse/tests/files/stormcov/embedquery2.storm b/synapse/tests/files/stormcov/embedquery2.storm new file mode 100644 index 00000000000..e838cec22ee --- /dev/null +++ b/synapse/tests/files/stormcov/embedquery2.storm @@ -0,0 +1,4 @@ + +tee ${ inet:fqdn=embedquery2.storm } +// coverage: 2 +// lines: 2 diff --git a/synapse/tests/files/stormcov/embedquery3.storm b/synapse/tests/files/stormcov/embedquery3.storm new file mode 100644 index 00000000000..31cf550fed3 --- /dev/null +++ b/synapse/tests/files/stormcov/embedquery3.storm @@ -0,0 +1,6 @@ + +tee ${ + inet:fqdn=embedquery3.storm +} +// coverage: 2, 3 +// lines: 2, 3 diff --git a/synapse/tests/files/stormcov/embedquery4.storm b/synapse/tests/files/stormcov/embedquery4.storm new file mode 100644 index 00000000000..1bb1402d543 --- /dev/null +++ b/synapse/tests/files/stormcov/embedquery4.storm @@ -0,0 +1,7 @@ + +tee +${ + inet:fqdn=embedquery4.storm +} +// coverage: 2, 3, 4 +// lines: 2, 4 diff --git a/synapse/tests/files/stormcov/embedquery5.storm b/synapse/tests/files/stormcov/embedquery5.storm new file mode 100644 index 00000000000..e97963568d6 --- /dev/null +++ b/synapse/tests/files/stormcov/embedquery5.storm @@ -0,0 +1,11 @@ + +tee +${ + + + inet:fqdn=embedquery5.storm + + +} +// coverage: 2, 3, 6 +// lines: 2, 6 diff --git a/synapse/tests/files/stormcov/lookup.storm b/synapse/tests/files/stormcov/lookup.storm index 23172dc849b..28a733d1eb9 100644 --- a/synapse/tests/files/stormcov/lookup.storm +++ b/synapse/tests/files/stormcov/lookup.storm @@ -4,3 +4,4 @@ function foo() { } yield $foo() yield $foo() +// lines: 1, 2, 3, 5, 6 diff --git a/synapse/tests/files/stormcov/pivot.storm b/synapse/tests/files/stormcov/pivot.storm index ced65f58b6e..d264208afcb 100644 --- a/synapse/tests/files/stormcov/pivot.storm +++ b/synapse/tests/files/stormcov/pivot.storm @@ -1,2 +1,4 @@ [inet:fqdn=vertex.link] -> inet:fqdn +// coverage: 1, 2 +// lines: 1, 2 diff --git a/synapse/tests/files/stormcov/pragma-nocov.storm b/synapse/tests/files/stormcov/pragma-nocov.storm index ce8cd72153f..171930d09f5 100644 --- a/synapse/tests/files/stormcov/pragma-nocov.storm +++ b/synapse/tests/files/stormcov/pragma-nocov.storm @@ -16,3 +16,5 @@ function foo() { } // pragma: no cover stop } yield $foo() +// coverage: 1, 2, 3, 18 +// lines: 1, 2, 3, 12, 18 diff --git a/synapse/tests/files/stormcov/spin.storm b/synapse/tests/files/stormcov/spin.storm index fade24bdfb2..54c3f07e697 100644 --- a/synapse/tests/files/stormcov/spin.storm +++ b/synapse/tests/files/stormcov/spin.storm @@ -1,3 +1,5 @@ [inet:fqdn=vertex.link] spin +// coverage: 2, 3 +// lines: 2, 3 diff --git a/synapse/tests/files/stormcov/stormctrl.storm b/synapse/tests/files/stormcov/stormctrl.storm index e6570e51858..b0323a501ad 100644 --- a/synapse/tests/files/stormcov/stormctrl.storm +++ b/synapse/tests/files/stormcov/stormctrl.storm @@ -4,3 +4,5 @@ function foo() { } } yield $foo() +// coverage: 1, 2, 3, 6 +// lines: 1, 2, 3, 6 diff --git a/synapse/tests/test_utils_stormcov.py b/synapse/tests/test_utils_stormcov.py index 046ca7c6c3c..255cd58b2e6 100644 --- a/synapse/tests/test_utils_stormcov.py +++ b/synapse/tests/test_utils_stormcov.py @@ -1,12 +1,10 @@ +import os import logging -import inspect -import unittest.mock as mock +import pathlib +import argparse -from coverage.exceptions import NoSource - -import synapse.lib.ast as s_ast -import synapse.lib.view as s_view -import synapse.lib.stormctrl as s_stormctrl +import regex +import coverage.exceptions import synapse.tests.files as s_files import synapse.tests.utils as s_utils @@ -15,134 +13,197 @@ logger = logging.getLogger(__name__) -class TestUtilsStormcov(s_utils.SynTest): - async def test_basics(self): - - opts = {"storm_dirs": "synapse/tests/files/stormcov"} - s_stormcov.coverage_init(mock.MagicMock(), opts) - plugin = s_stormcov.StormPlugin(opts) - - reporter = plugin.file_reporter(s_files.getAssetPath('stormcov/stormctrl.storm')) - self.eq(s_files.getAssetStr('stormcov/stormctrl.storm'), reporter.source()) - self.eq(reporter.lines(), {1, 2, 3, 6}) - self.eq(reporter.translate_lines({1, 2}), {1, 2}) - - # no cover, no cover start, and no cover stop - reporter = plugin.file_reporter(s_files.getAssetPath('stormcov/pragma-nocov.storm')) - self.eq(reporter.lines(), {1, 2, 3, 12, 18}) - self.eq(reporter.excluded_lines(), {6, 8, 9, 10, 14, 15, 16}) - - # We no longer do whitespace transformations of lines. - reporter = plugin.file_reporter(s_files.getAssetPath('stormcov/spin.storm')) - self.eq(reporter.translate_lines({1, 2}), {1, 2}) - - with self.raises(NoSource): - reporter = plugin.file_reporter('newp') - reporter.source() - - stormtracer = plugin.file_tracer('synapse/lib/ast.py') - self.true(stormtracer.has_dynamic_source_filename()) - self.none(stormtracer.dynamic_source_filename(None, inspect.currentframe())) +class StormcovConfig: + ''' + Helper class to simulate pytest config + ''' + def __init__(self, stormcov=False, stormdirs='', stormexts='storm', stormcov_append=False, stormcov_basedir=s_stormcov.PACKAGE_DIR): + self.option = argparse.Namespace( + stormcov=stormcov, + stormdirs=stormdirs, + stormexts=stormexts, + stormcov_append=stormcov_append, + stormcov_basedir=pathlib.Path(stormcov_basedir) + ) - ctrltracer = plugin.file_tracer('synapse/lib/stormctrl.py') - self.true(ctrltracer.has_dynamic_source_filename()) - self.none(ctrltracer.dynamic_source_filename(None, inspect.currentframe())) - - pivotracer = plugin.file_tracer('synapse/lib/view.py') - self.true(pivotracer.has_dynamic_source_filename()) - self.none(pivotracer.dynamic_source_filename(None, inspect.currentframe())) - - self.none(plugin.file_tracer('newp')) +class TestUtilsStormcov(s_utils.SynTest): + async def test_stormcov_basics(self): + basedir = pathlib.Path(s_files.ASSETS) + stormdir = str(basedir / 'stormcov') + opts = StormcovConfig(stormdirs=stormdir, stormcov_basedir=basedir) + + stormfiles = [ + s_files.getAssetPath('stormcov/argvquery.storm'), + s_files.getAssetPath('stormcov/argvquery2.storm'), + s_files.getAssetPath('stormcov/argvquery3.storm'), + s_files.getAssetPath('stormcov/argvquery4.storm'), + s_files.getAssetPath('stormcov/argvquery5.storm'), + s_files.getAssetPath('stormcov/dupesubs.storm'), + s_files.getAssetPath('stormcov/dupewarn.storm'), + s_files.getAssetPath('stormcov/embedquery.storm'), + s_files.getAssetPath('stormcov/embedquery2.storm'), + s_files.getAssetPath('stormcov/embedquery3.storm'), + s_files.getAssetPath('stormcov/embedquery4.storm'), + s_files.getAssetPath('stormcov/embedquery5.storm'), + s_files.getAssetPath('stormcov/lookup.storm'), + s_files.getAssetPath('stormcov/pivot.storm'), + s_files.getAssetPath('stormcov/pragma-nocov.storm'), + s_files.getAssetPath('stormcov/spin.storm'), + s_files.getAssetPath('stormcov/stormctrl.storm'), + ] + + stormcov = s_stormcov.StormcovPlugin(opts) + with self.getLoggerStream('synapse.utils.stormcov') as stream: + stormcov.find_storm_files(stormdir) + + badstorm = s_files.getAssetPath('stormcov/badstorm.storm') + stream.expect(f'Skipping invalid storm file: {badstorm}') + + self.sorteq(list(stormcov.guid_map.values()), stormfiles) + stormcov.reset() + + stormcov.discover_stormdirs('') + self.sorteq(list(stormcov.guid_map.values()), stormfiles) + stormcov.reset() + + stormcov.stormdirs = [] + stormcov.discover_stormdirs([pathlib.Path(stormdir)]) + self.sorteq(list(stormcov.guid_map.values()), stormfiles) + stormcov.reset() + + async def test_stormcov_dups(self): + basedir = pathlib.Path(s_files.ASSETS) + stormdir = str(basedir / 'stormcov') + opts = StormcovConfig(stormdirs=stormdir, stormcov_basedir=basedir) + + stormcov = s_stormcov.StormcovPlugin(opts) + with self.getLoggerStream('synapse.utils.stormcov') as stream: + stormcov.find_storm_files(stormdir) + + dupewarn = s_files.getAssetPath('stormcov/dupewarn.storm') + self.isin(dupewarn, list(stormcov.guid_map.values())) + + storm = s_files.getAssetStr('stormcov/dupewarn.storm') + + # Read argvquery duplicate pairs + argvdups = regex.search(r'\/\/ argvdups:.*?$', storm, flags=regex.M) + self.nn(argvdups, msg='dupewarn.storm requires a "// argvdups: #:#, #:#, #:#" line') + argvpairs = regex.findall(r'\d+:\d+', argvdups.group()) + + # Read embedquery duplicate pairs + embddups = regex.search(r'\/\/ embddups:.*?$', storm, flags=regex.M) + self.nn(embddups, msg='dupewarn.storm requires a "// embddups: #:#, #:#, #:#" line') + embdpairs = regex.findall(r'\d+:\d+', embddups.group()) + + def splitpair(x): + return list(map(int, x.split(':'))) + + argvexp = map(splitpair, argvpairs) + embdexp = map(splitpair, embdpairs) + + for last, first in embdexp: + stream.expect(f'Duplicate embedquery in {dupewarn} at line {last}, coverage will be reported on first instance in {dupewarn} at line {first}') + + for last, first in argvexp: + stream.expect(f'Duplicate argvquery in {dupewarn} at line {last}, coverage will be reported on first instance in {dupewarn} at line {first}') + + async def test_stormcov_coverage(self): + basedir = pathlib.Path(s_files.ASSETS) + stormdir = str(basedir / 'stormcov') + opts = StormcovConfig(stormdirs=stormdir, stormcov_basedir=basedir) + + stormcov = s_stormcov.StormcovPlugin(opts) + stormcov.find_storm_files(stormdir) async with self.getTestCore() as core: - orig = s_stormctrl.StormCtrlFlow.__init__ - def __init__(self, item=None): - frame = inspect.currentframe() - assert 'stormctrl.storm' in ctrltracer.dynamic_source_filename(None, frame) - assert (3, 3) == ctrltracer.line_number_range(frame) - orig(self, item=item) - - with mock.patch('synapse.lib.stormctrl.StormCtrlFlow.__init__', __init__): - await core.nodes(s_files.getAssetStr('stormcov/stormctrl.storm')) - - def __init__(self, item=None): - frame = inspect.currentframe() - assert 'argvquery.storm' in ctrltracer.dynamic_source_filename(None, frame) - assert (4, 4) == ctrltracer.line_number_range(frame) - orig(self, item=item) - - with mock.patch('synapse.lib.stormctrl.StormCtrlFlow.__init__', __init__): - await core.stormlist(s_files.getAssetStr('stormcov/argvquery.storm')) - - def __init__(self, item=None): - frame = inspect.currentframe() - assert ctrltracer.dynamic_source_filename(None, frame) is None - orig(self, item=item) - - with mock.patch('synapse.lib.stormctrl.StormCtrlFlow.__init__', __init__): - await core.stormlist(s_files.getAssetStr('stormcov/lookup.storm'), opts={'mode': 'lookup'}) + async def check_cov(filename): + storm = s_files.getAssetStr(filename) + stormcov._start_sysmon() + await core.stormlist(storm) + stormcov._stop_sysmon() + + coverage = regex.search(r'\/\/ coverage:.*?$', storm, flags=regex.M) + self.nn(coverage, msg='Stormcov sample files require a "// coverage: #, #, #" line') + + linenums = regex.findall(r'\d+', coverage.group()) + expected = set(map(int, linenums)) + + self.eq(dict(stormcov.lines_hit), {s_files.getAssetPath(filename): expected}) + + stormcov.reset() + + await check_cov('stormcov/argvquery.storm') + await check_cov('stormcov/argvquery2.storm') + await check_cov('stormcov/argvquery3.storm') + await check_cov('stormcov/argvquery4.storm') + await check_cov('stormcov/argvquery5.storm') + await check_cov('stormcov/dupesubs.storm') + await check_cov('stormcov/embedquery.storm') + await check_cov('stormcov/embedquery2.storm'), + await check_cov('stormcov/embedquery3.storm'), + await check_cov('stormcov/embedquery4.storm'), + await check_cov('stormcov/embedquery5.storm'), + await check_cov('stormcov/pivot.storm') + await check_cov('stormcov/pragma-nocov.storm') + await check_cov('stormcov/spin.storm') + await check_cov('stormcov/stormctrl.storm') + + async def test_stormcov_lookup(self): + basedir = s_files.ASSETS + stormdir = os.path.join(basedir, 'stormcov') + opts = StormcovConfig(stormdirs=stormdir, stormcov_basedir=basedir) + + stormcov = s_stormcov.StormcovPlugin(opts) + stormcov.find_storm_files(stormdir) - orig = s_view.View.nodesByPropValu - async def nodesByPropValu(self, full, cmpr, valu, norm=True): - frame = inspect.currentframe() - if pivotracer.dynamic_source_filename(None, frame) is not None: - assert (2, 2) == pivotracer.line_number_range(frame) - - async for item in orig(self, full, cmpr, valu): - yield item - - with mock.patch('synapse.lib.view.View.nodesByPropValu', nodesByPropValu): - await core.nodes(s_files.getAssetStr('stormcov/pivot.storm')) - - async def pullone(genr): - gotone = None - async for gotone in genr: - break - - async def pullgenr(): - frame = inspect.currentframe() - assert 'spin.storm' in stormtracer.dynamic_source_filename(None, frame) - assert (3, 3) == stormtracer.line_number_range(frame) - - if gotone is None: - return - - yield gotone - async for item in genr: - yield item - - return pullgenr(), gotone is None - - with mock.patch('synapse.lib.ast.pullone', pullone): - await core.nodes(s_files.getAssetStr('stormcov/spin.storm')) - - async def pullone(genr): - gotone = None - async for gotone in genr: - break - - async def pullgenr(): - frame = inspect.currentframe() - assert stormtracer.dynamic_source_filename(None, frame) is None - - if gotone is None: - return - - yield gotone - async for item in genr: - yield item - - return pullgenr(), gotone is None - - with mock.patch('synapse.lib.ast.pullone', pullone): - await core.nodes(s_files.getAssetStr('stormcov/pivot.storm')) - - orig = s_ast.Const.compute - async def compute(self, runt, valu): - frame = inspect.currentframe() - assert 'pivot.storm' in stormtracer.dynamic_source_filename(None, frame) - assert stormtracer.line_number_range(frame) in ((1, 1), (2, 2)) - return await orig(self, runt, valu) + async with self.getTestCore() as core: + await core.nodes('[ inet:fqdn=vertex.link ]') + stormcov._start_sysmon() + await core.stormlist(s_files.getAssetStr('stormcov/lookup.storm'), opts={'mode': 'lookup'}) + stormcov._stop_sysmon() + + # No coverage for lookup mode + self.len(0, dict(stormcov.lines_hit)) + + async def test_stormcov_stormreporter(self): + parser = s_stormcov.get_parser() + + async def check_lines(filename): + storm = s_files.getAssetStr(filename) + reporter = s_stormcov.StormReporter(s_files.getAssetPath(filename), parser) + + lines = regex.search(r'\/\/ lines:.*?$', storm, flags=regex.M) + self.nn(lines, msg='Stormcov sample files require a "// lines: #, #, #" line') + + linenums = regex.findall(r'\d+', lines.group()) + expected = set(map(int, linenums)) + + self.eq(reporter.lines(), expected) + + await check_lines('stormcov/argvquery.storm') + await check_lines('stormcov/argvquery2.storm') + await check_lines('stormcov/argvquery3.storm') + await check_lines('stormcov/argvquery4.storm') + await check_lines('stormcov/argvquery5.storm') + await check_lines('stormcov/dupesubs.storm') + await check_lines('stormcov/embedquery.storm'), + await check_lines('stormcov/embedquery2.storm'), + await check_lines('stormcov/embedquery3.storm'), + await check_lines('stormcov/embedquery4.storm'), + await check_lines('stormcov/embedquery5.storm'), + await check_lines('stormcov/lookup.storm') + await check_lines('stormcov/pivot.storm') + await check_lines('stormcov/pragma-nocov.storm') + await check_lines('stormcov/spin.storm') + await check_lines('stormcov/stormctrl.storm') + + # Non-existent file + reporter = s_stormcov.StormReporter('newp', parser) + with self.raises(coverage.exceptions.NoSource) as exc: + reporter.source() + self.eq(str(exc.exception), "Couldn't read newp: [Errno 2] No such file or directory: 'newp'") - with mock.patch('synapse.lib.ast.Const.compute', compute): - await core.nodes(s_files.getAssetStr('stormcov/pivot.storm')) + async def test_stormcov_stormreporter_plugin(self): + plugin = s_stormcov.StormReporterPlugin() + reporter = plugin.file_reporter(s_files.getAssetPath('stormcov/pivot.storm')) + self.eq(reporter.lines(), {1, 2}) diff --git a/synapse/utils/stormcov.py b/synapse/utils/stormcov.py new file mode 100644 index 00000000000..869920a680b --- /dev/null +++ b/synapse/utils/stormcov.py @@ -0,0 +1,464 @@ +import os +import sys +import inspect +import logging +import pathlib +import collections + +import lark +import regex +import pytest +import coverage + +from coverage.exceptions import NoSource + +import synapse.data as s_data +import synapse.common as s_common + +logger = logging.getLogger(__name__) + +PACKAGE_DIR = pathlib.Path('packages').absolute() + +def pytest_addoption(parser): # pragma: no cover + # NB: no coverage since this is a pytest hook + """Add options to control storm coverage.""" + + group = parser.getgroup('stormcov', 'storm coverage reporting') + + group.addoption( + '--stormcov', + action='store_true', + default=False, + dest='stormcov', + help='Enable stormcov. Default: False', + ) + + group.addoption( + '--storm-dirs', + action='store', + dest='stormdirs', + help='Comma separated list of paths to search for Storm files. Default: autodiscover stormdirs based on executed tests.', + ) + + group.addoption( + '--storm-exts', + action='store', + default='storm', + dest='stormexts', + help='Comma separated list of file extensions containing Storm. Default: storm', + ) + + group.addoption( + '--stormcov-append', + action='store_true', + default=False, + dest='stormcov_append', + help='Append to existing coverage reports instead of erasing.', + ) + + group.addoption( + '--stormcov-basedir', + default=PACKAGE_DIR, + type=pathlib.Path, + help='The base package directory. Useful for monorepo environments.', + ) + +DISABLE = sys.monitoring.DISABLE + +def pytest_configure(config): # pragma: no cover + # NB: no coverage since this is a pytest hook + if not config.option.stormcov and not (config.option.stormdirs or config.option.stormcov_append): + return + + config.pluginmanager.register(StormcovPlugin(config), 'stormcov') + +def get_parser(): + grammar = s_data.getLark('storm') + return lark.Lark(grammar, start='query', regex=True, parser='lalr', keep_all_tokens=True, + maybe_placeholders=False, propagate_positions=True) + +class StormcovPlugin: + + PARSE_METHODS = {'compute', 'once', 'lift', 'getPivsOut', 'getPivsIn'} + + def __init__(self, config): + # toolid=4 is not defined in sys.monitoring so there's less of a chance + # that we interfere with another tool, namely coveragepy. + self.toolid = 4 + self._prevcb = None + self._freetool = False + + self.isworker = os.environ.get("PYTEST_XDIST_WORKER") is not None + + # xdist detection logic from here: + # https://github.com/pytest-dev/pytest-cov/blob/be3366838e41a9cbefa714636a39c5c2f6d5f588/src/pytest_cov/plugin.py#L235C19-L235C139 + self.iscontroller = ( + getattr(config.option, 'numprocesses', False) or + getattr(config.option, 'distload', False) or + getattr(config.option, 'dist', 'no') != 'no' + ) and not self.isworker + + self.config = config + self.handlers = { + 'ast.py': self.handle_ast, + 'view.py': self.handle_view, + 'stormctrl.py': self.handle_stormctrl, + } + + self.node_map = {} + self.text_map = {} + self.guid_map = {} + self.subq_map = {} + self.lines_hit = collections.defaultdict(set) + + self.freg = regex.compile(r'.*synapse/lib/(ast.py|view.py|stormctrl.py)$') + + self.parser = get_parser() + + opts = config.option + + self.append = config.option.stormcov_append + self.stormdirs = config.option.stormdirs + self.basedir = config.option.stormcov_basedir + self.extensions = [e.strip() for e in opts.stormexts.split(',')] + + def reset(self): + self.lines_hit = collections.defaultdict(set) + + def find_storm_files(self, dirn): + for path in self.find_executable_files(dirn): + with open(path, 'r') as f: + apth = os.path.abspath(path) + + try: + tree = self.parser.parse(f.read()) + except lark.exceptions.UnexpectedToken: + logger.warning('Skipping invalid storm file: %s', apth) + continue + + self.find_subqueries(tree, apth) + + guid = s_common.guid(str(tree)) + self.guid_map[guid] = apth + + def find_executable_files(self, src_dir): + rx = r"^[^#~!$@%^&*()+=,]+\.(" + "|".join(self.extensions) + r")$" + for (dirpath, dirnames, filenames) in os.walk(src_dir): + for filename in filenames: + if regex.search(rx, filename): + path = os.path.join(dirpath, filename) + yield path + + def find_subqueries(self, tree, path): + for rule in ('argvquery', 'embedquery'): + for node in tree.find_data(rule): + + subq = node.children[1] + if subq.meta.empty: + continue + + subg = s_common.guid(str(subq)) + + line = node.meta.line - 1 + rline = node.meta.line + if rule == 'argvquery': + line = subq.meta.line - 1 + + if subg in self.subq_map: + (pname, _, pline) = self.subq_map[subg] + logger.warning(f'Duplicate {rule} in {path} at line {rline}, coverage will ' + f'be reported on first instance in {pname} at line {pline}') + continue + + self.subq_map[subg] = (path, line, rline) + + def _start_sysmon(self): + if sys.monitoring.get_tool(self.toolid) is None: + sys.monitoring.use_tool_id(self.toolid, 'pytest-stormcov') + self._freetool = True + + sys.monitoring.set_events(self.toolid, sys.monitoring.events.PY_START) + self._prevcb = sys.monitoring.register_callback(self.toolid, sys.monitoring.events.PY_START, self.sysmon_py_start) + + def _stop_sysmon(self): + if self._prevcb: # pragma: no cover + sys.monitoring.register_callback(self.toolid, sys.monitoring.events.PY_START, self._prevcb) + return + + if self._freetool: + sys.monitoring.free_tool_id(self.toolid) + self._freetool = False + + @pytest.hookimpl(wrapper=True) + def pytest_runtestloop(self, session): # pragma: no cover + # NB: no coverage since this is a pytest hook + + self.cov = coverage.Coverage.current() + + if self.cov is None: + self.cov = coverage.Coverage() + + if not self.append: + self.cov.erase() + + self._start_sysmon() + yield + self._stop_sysmon() + + self.cov.load() + + data = self.cov.get_data() + + # Bail if there's no coverage. This could be because of a xdist worker + # that didn't have any work + if not self.lines_hit: + return + + # Add our stormcov data + data.add_lines(dict(self.lines_hit)) + data.touch_files(self.guid_map.values(), 'synapse.utils.stormcov.StormReporterPlugin') + + # Save the coverage data + data.write() + + def discover_stormdirs(self, testpaths: list[pathlib.Path]): + # If a specific set of directories were specified, use that + if self.stormdirs: + for dirn in self.stormdirs.split(','): + self.find_storm_files(dirn) + return + + stormdirs = set() + + # Iterate through the tests, get their path, and add the containing storm package directory + for testpath in testpaths: + if testpath.is_relative_to(self.basedir): + stormdir = str(self.basedir / testpath.relative_to(self.basedir).parts[0]) + stormdirs.add(stormdir) + + for dirn in stormdirs: + self.find_storm_files(dirn) + + @pytest.hookimpl(wrapper=True) + def pytest_collection_modifyitems(self, config, items): # pragma: no cover + # NB: no coverage since this is a pytest hook + # Note: If using xdist, this function executes on each worker node + testpaths = [item.path for item in items] + self.discover_stormdirs(testpaths) + yield + + @pytest.hookimpl(wrapper=True) + def pytest_xdist_node_collection_finished(self, node, ids): # pragma: no cover + # NB: no coverage since this is a pytest hook + # Note: This hook allows the xdist controller to get a list of test ids so we can build a list of storm dirs to present stormterm coverage + testpaths = [pathlib.Path(testid.split('::')[0]).absolute() for testid in ids] + self.discover_stormdirs(testpaths) + yield + + def pytest_terminal_summary(self, terminalreporter, exitstatus, config): # pragma: no cover + # NB: no coverage since this is a pytest hook + if self.isworker: + return + + if not self.iscontroller and not self.lines_hit: + return + + self.cov.report(skip_covered=False, skip_empty=False) + + def sysmon_py_start(self, code, instruction_offset): # pragma: no cover + # NB: no coverage since this runs inside of the sys.monitoring callback + if (fname := self.freg.match(code.co_filename)): + return self.handlers[fname.group(1)](code) + return DISABLE + + def handle_ast(self, code, frame=None): # pragma: no cover + # NB: no coverage since this runs inside of the sys.monitoring callback + if frame is None: + if code.co_name == 'pullgenr': + frame = inspect.currentframe().f_back.f_back + + if frame.f_back.f_code.co_name != 'execStormCmd': + return + + frame = frame.f_back.f_back + + elif code.co_name not in self.PARSE_METHODS: + return DISABLE + + else: + frame = inspect.currentframe().f_back.f_back + + realnode = frame.f_locals.get('self') + if hasattr(realnode, '_coverage_hit'): + return + + realnode._coverage_hit = True + + node = realnode + while hasattr(node, 'parent'): + node = node.parent + + nodeid = id(node) + + info = self.node_map.get(nodeid, s_common.novalu) + if info is None: + return + + if info is not s_common.novalu: + self.mark_lines(frame, info) + return + + if node.__class__.__name__ != 'Query': + self.node_map[nodeid] = None + return + + info = self.text_map.get(node.text, s_common.novalu) + if info is not s_common.novalu: + self.mark_lines(frame, info) + return + + tree = self.parser.parse(node.text) + guid = s_common.guid(str(tree)) + + subq = self.subq_map.get(guid) + if subq is not None: + (filename, offs, _) = subq + else: + filename = self.guid_map.get(guid) + offs = 0 + + if filename is None: + self.node_map[nodeid] = None + return + + self.node_map[nodeid] = (filename, offs) + self.text_map[node.text] = (filename, offs) + self.mark_lines(frame, (filename, offs)) + + def mark_lines(self, frame, info): # pragma: no cover + # NB: no coverage since this runs inside of the sys.monitoring callback + astn = frame.f_locals.get('self') + fname, offs = info + strt = astn.astinfo.sline + if astn.astinfo.isterm: + fini = astn.astinfo.eline + else: + fini = strt + + self.lines_hit[fname].update(range(strt + offs, fini + offs + 1)) + + PIVOT_METHODS = {'nodesByPropValu', 'nodesByPropArray', 'nodesByTag', 'getNodeByNdef'} + def handle_view(self, code): # pragma: no cover + # NB: no coverage since this runs inside of the sys.monitoring callback + if code.co_name not in self.PIVOT_METHODS: + return DISABLE + + frame = inspect.currentframe().f_back.f_back + if frame.f_code.co_name != 'run': + return + return self.handle_ast(code, frame=frame) + + def handle_stormctrl(self, code): # pragma: no cover + # NB: no coverage since this runs inside of the sys.monitoring callback + if code.co_name != '__init__': + return DISABLE + return self.handle_ast(code, frame=inspect.currentframe().f_back.f_back.f_back) + +TOKENS = [ + 'ABSPROP', + 'ABSPROPNOUNIV', + 'PROPS', + 'UNIVNAME', + 'EXPRUNIVNAME', + 'RELNAME', + 'EXPRRELNAME', + 'ALLTAGS', + 'BREAK', + 'CONTINUE', + 'CMDNAME', + 'TAGMATCH', + 'NONQUOTEWORD', + 'VARTOKN', + 'EXPRVARTOKN', + 'NUMBER', + 'HEXNUMBER', + 'OCTNUMBER', + 'BOOL', + 'EXPRTIMES', + '_EMIT', + '_STOP', + '_RETURN', +] + +class StormReporter(coverage.FileReporter): + def __init__(self, filename, parser): + super().__init__(filename) + + self._parser = parser + self._source = None + + def source(self): + if self._source is None: + try: + with open(self.filename, 'r') as f: + self._source = f.read() + + except (OSError, UnicodeError) as exc: + raise NoSource(f"Couldn't read {self.filename}: {exc}") + return self._source + + def lines(self): + source_lines = set() + + tree = self._parser.parse(self.source()) + + for token in tree.scan_values(lambda v: isinstance(v, lark.lexer.Token)): + if token.type in TOKENS: + source_lines.add(token.line) + + return source_lines - self.excluded_lines() + + def excluded_lines(self): + excluded_lines = set() + + pragma = 'pragma: no cover' + start = 'pragma: no cover start' + stop = 'pragma: no cover stop' + + lines = self.source().splitlines() + nocov = [(lineno + 1, text) for (lineno, text) in enumerate(lines) if pragma in text] + + block = None + for (lineno, text) in nocov: + if stop in text: + if block is not None: + # End a multi-line block + excluded_lines |= set(range(block, lineno + 1)) + block = None + continue + + if start in text: + if block is None: + # Start a multi-line block + block = lineno + continue + + if pragma in text: + excluded_lines.add(lineno) + + return excluded_lines + +# StormReporterPlugin and coverage_init below are both to support the command +# line coverage tools being able to interpret coverage results generated by +# stormcov +class StormReporterPlugin(coverage.CoveragePlugin): + def __init__(self): + self.parser = get_parser() + + def file_reporter(self, filename): + return StormReporter(filename, self.parser) + +def coverage_init(reg, options): # pragma: no cover + # NB: no coverage since this is the coverage plugin entrypoint + plugin = StormReporterPlugin() + reg.add_file_tracer(plugin) diff --git a/synapse/utils/stormcov/__init__.py b/synapse/utils/stormcov/__init__.py deleted file mode 100644 index 6c27fd9f448..00000000000 --- a/synapse/utils/stormcov/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from .plugin import StormPlugin - -def coverage_init(reg, options): - plugin = StormPlugin(options) - reg.add_configurer(plugin) - reg.add_file_tracer(plugin) diff --git a/synapse/utils/stormcov/plugin.py b/synapse/utils/stormcov/plugin.py deleted file mode 100644 index 63826160681..00000000000 --- a/synapse/utils/stormcov/plugin.py +++ /dev/null @@ -1,273 +0,0 @@ -import os -import lark -import regex -import logging -import coverage - -from coverage.exceptions import NoSource - -import synapse.data as s_data -import synapse.common as s_common - -logger = logging.getLogger(__name__) - -class StormPlugin(coverage.CoveragePlugin, coverage.FileTracer): - - def __init__(self, options): - extensions = options.get('storm_extensions', 'storm') - self.extensions = [e.strip() for e in extensions.split(',')] - - grammar = s_data.getLark('storm') - self.parser = lark.Lark(grammar, start='query', debug=True, regex=True, - parser='lalr', keep_all_tokens=True, maybe_placeholders=False, - propagate_positions=True) - - self.node_map = {} - self.text_map = {} - self.guid_map = {} - self.subq_map = {} - - dirs = options.get('storm_dirs', '.') - if dirs: - self.stormdirs = [d.strip() for d in dirs.split(',')] - for dirn in self.stormdirs: - self.find_storm_files(dirn) - - def find_storm_files(self, dirn): - for path in self.find_executable_files(dirn): - with open(path, 'r') as f: - apth = os.path.abspath(path) - - tree = self.parser.parse(f.read()) - self.find_subqueries(tree, apth) - - guid = s_common.guid(str(tree)) - self.guid_map[guid] = apth - - def find_subqueries(self, tree, path): - for rule in ('argvquery', 'embedquery'): - for node in tree.find_data(rule): - - subq = node.children[1] - if subq.meta.empty: - continue - - subg = s_common.guid(str(subq)) - line = (subq.meta.line - 1) - - if subg in self.subq_map: - (pname, pline) = self.subq_map[subg] - logger.warning(f'Duplicate {rule} in {path} at line {line + 1}, coverage will ' - f'be reported on first instance in {pname} at line {pline + 1}') - continue - - self.subq_map[subg] = (path, subq.meta.line - 1) - - def file_tracer(self, filename): - if filename.endswith('synapse/lib/ast.py'): - return self - - if filename.endswith('synapse/lib/stormctrl.py'): - return StormCtrlTracer(self) - - if filename.endswith('synapse/lib/view.py'): - return PivotTracer(self) - - return None - - def file_reporter(self, filename): - return StormReporter(filename, self.parser) - - def find_executable_files(self, src_dir): - rx = r"^[^#~!$@%^&*()+=,]+\.(" + "|".join(self.extensions) + r")$" - for (dirpath, dirnames, filenames) in os.walk(src_dir): - for filename in filenames: - if regex.search(rx, filename): - path = os.path.join(dirpath, filename) - yield path - - def has_dynamic_source_filename(self): - return True - - PARSE_METHODS = {'compute', 'once', 'lift', 'getPivsOut', 'getPivsIn'} - - def dynamic_source_filename(self, filename, frame, force=False): - - if frame.f_code.co_name == 'pullgenr': - if frame.f_back.f_code.co_name != 'execStormCmd': - return None - frame = frame.f_back.f_back - - elif frame.f_code.co_name not in self.PARSE_METHODS and not force: - return None - - realnode = frame.f_locals.get('self') - node = realnode - while hasattr(node, 'parent'): - node = node.parent - - nodeid = id(node) - - info = self.node_map.get(nodeid, s_common.novalu) - if info is None: - return - - if info is not s_common.novalu: - realnode._coverage_offs = info[1] - return info[0] - - if not node.__class__.__name__ == 'Query': - self.node_map[nodeid] = None - return - - info = self.text_map.get(node.text, s_common.novalu) - if info is not s_common.novalu: - realnode._coverage_offs = info[1] - return info[0] - - tree = self.parser.parse(node.text) - guid = s_common.guid(str(tree)) - - subq = self.subq_map.get(guid) - if subq is not None: - (filename, offs) = subq - else: - filename = self.guid_map.get(guid) - offs = 0 - - realnode._coverage_offs = offs - - self.node_map[nodeid] = (filename, offs) - self.text_map[node.text] = (filename, offs) - return filename - - def line_number_range(self, frame): - if frame.f_code.co_name == 'pullgenr': - frame = frame.f_back.f_back - - astn = frame.f_locals.get('self') - - offs = astn._coverage_offs - strt = astn.astinfo.sline - if astn.astinfo.isterm: - fini = astn.astinfo.eline - else: - fini = strt - - return (strt + offs, fini + offs) - -class StormCtrlTracer(coverage.FileTracer): - def __init__(self, parent): - self.parent = parent - - def has_dynamic_source_filename(self): - return True - - def dynamic_source_filename(self, filename, frame): - if frame.f_code.co_name != '__init__': - return None - return self.parent.dynamic_source_filename(None, frame.f_back, force=True) - - def line_number_range(self, frame): - return self.parent.line_number_range(frame.f_back) - -class PivotTracer(coverage.FileTracer): - def __init__(self, parent): - self.parent = parent - - def has_dynamic_source_filename(self): - return True - - PARSE_METHODS = {'nodesByPropValu', 'nodesByPropArray', 'nodesByTag', 'getNodeByNdef'} - - def dynamic_source_filename(self, filename, frame): - if frame.f_code.co_name not in self.PARSE_METHODS or frame.f_back.f_code.co_name != 'run': - return None - return self.parent.dynamic_source_filename(None, frame.f_back, force=True) - - def line_number_range(self, frame): - return self.parent.line_number_range(frame.f_back) - -TOKENS = [ - 'ABSPROP', - 'ABSPROPNOUNIV', - 'PROPS', - 'UNIVNAME', - 'EXPRUNIVNAME', - 'RELNAME', - 'EXPRRELNAME', - 'ALLTAGS', - 'BREAK', - 'CONTINUE', - 'CMDNAME', - 'TAGMATCH', - 'NONQUOTEWORD', - 'VARTOKN', - 'EXPRVARTOKN', - 'NUMBER', - 'HEXNUMBER', - 'OCTNUMBER', - 'BOOL', - 'EXPRTIMES', - '_EMIT', - '_STOP', - '_RETURN', -] - -class StormReporter(coverage.FileReporter): - def __init__(self, filename, parser): - super().__init__(filename) - - self._parser = parser - self._source = None - - def source(self): - if self._source is None: - try: - with open(self.filename, 'r') as f: - self._source = f.read() - - except (OSError, UnicodeError) as exc: - raise NoSource(f"Couldn't read {self.filename}: {exc}") - return self._source - - def lines(self): - source_lines = set() - - tree = self._parser.parse(self.source()) - - for token in tree.scan_values(lambda v: isinstance(v, lark.lexer.Token)): - if token.type in TOKENS: - source_lines.add(token.line) - - return source_lines - self.excluded_lines() - - def excluded_lines(self): - excluded_lines = set() - - pragma = 'pragma: no cover' - start = 'pragma: no cover start' - stop = 'pragma: no cover stop' - - lines = self.source().splitlines() - nocov = [(lineno + 1, text) for (lineno, text) in enumerate(lines) if pragma in text] - - block = None - for (lineno, text) in nocov: - if stop in text: - if block is not None: - # End a multi-line block - excluded_lines |= set(range(block, lineno + 1)) - block = None - continue - - if start in text: - if block is None: - # Start a multi-line block - block = lineno - continue - - if pragma in text: - excluded_lines.add(lineno) - - return excluded_lines