From b441d1ecaccfcd83c3b6ea844b091e87419c5a7e Mon Sep 17 00:00:00 2001 From: Tycho Date: Thu, 11 Jan 2024 13:43:28 +0100 Subject: [PATCH 01/19] Move replies to a thread --- matterbot.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/matterbot.py b/matterbot.py index 92fdf606..c806ed6d 100755 --- a/matterbot.py +++ b/matterbot.py @@ -74,7 +74,7 @@ async def handle_message(self, message: dict): except json.JSONDecodeError as e: print(('ERROR'), e) - async def send_message(self, channel, text): + async def send_message(self, channel, text, postid=None): try: channelname = channel.lower() log.info('Channel:' + channelname + ' <- Message: (' + str(len(text)) + ' chars)') @@ -98,6 +98,7 @@ async def send_message(self, channel, text): for block in blocks: self.mmDriver.posts.create_post(options={'channel_id': channel, 'message': block, + 'root_id': postid }) except: raise @@ -111,6 +112,7 @@ async def handle_post(self, data: dict): return post = json.loads(data['post']) userid = post['user_id'] + postid = post['id'] channelinfo = self.mmDriver.channels.get_channel(post['channel_id']) userchannels = [i['name'] for i in self.mmDriver.channels.get_channels_for_user(userid, self.my_team_id)] channelname = channelinfo['name'] @@ -170,7 +172,7 @@ async def handle_post(self, data: dict): if args: text += '\n**Arguments**: `' + args + '`' if len(text)>0: - await self.send_message(channelid, text) + await self.send_message(channelid, text, postid) except NameError: await self.send_message(channelid, 'No additional help available for the `' + module + '` module.') # Normal command @@ -195,7 +197,7 @@ async def handle_post(self, data: dict): results.append(executor.submit(self.commands[task]['process'], command, channelname, username, params, files, self.mmDriver)) except Exception as e: text = 'An error occurred within module: '+task+': '+str(type(e))+': '+e - await self.send_message(channelid, text) + await self.send_message(channelid, text, postid) for _ in concurrent.futures.as_completed(results): result = _.result() if result and 'messages' in result: @@ -220,9 +222,9 @@ async def handle_post(self, data: dict): 'file_ids': file_ids, }) else: - await self.send_message(channelid, text) + await self.send_message(channelid, text, postid) else: - await self.send_message(channelid, text) + await self.send_message(channelid, text, postid) except Exception as e: text = 'A Python error occurred: '+str(type(e))+': '+str(e) await self.send_message(channelid, text) From 168c762a403c845a31be60e0a04bcf2b615106f5 Mon Sep 17 00:00:00 2001 From: Tycho Date: Fri, 12 Jan 2024 16:00:00 +0100 Subject: [PATCH 02/19] Poc of formatted messsages. Formatting done in command module. --- commands/cyberthreat/command.py | 112 +++++++++++++++++++++++++++----- 1 file changed, 94 insertions(+), 18 deletions(-) diff --git a/commands/cyberthreat/command.py b/commands/cyberthreat/command.py index 39864fe6..cfb4ba7b 100755 --- a/commands/cyberthreat/command.py +++ b/commands/cyberthreat/command.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 import re -import requests +import logging import tldextract from pathlib import Path from datetime import datetime @@ -31,26 +31,87 @@ for actor in results['results']: actorlist[actor['name']]=actor +""" +Input data in the format + +{ + "source":"provider", + "responses": [ + { + "paragraph":"subtitle", + "preamble":"introduction to source", + "data": [ + {"category":"category", "datapoint":"datapoint", "stixtype":"ipv4-addr", "value":"value"}, + {"category":"category", "datapoint":"datapoint", "value":"value"} + ] + } + ] +} + +Converts to a message text and possibly an attachment. +The text can have multiple paragraph with a short introduction + +""" + +def formatdata(data): + logging.debug(f"working with {data}") + text = f"### {data['source']}\n" + if 'intro' in data: + text += f"{data['intro']}\n" + for response in data['responses']: + text += f"#### {response['paragraph']}\n" + if 'preamble' in response: + text += response['preamble'] + text +="\n\n" + if 'data' in response: + text += "|Category | Datapoint | Value|\n| :- | -: | :- |\n" + logging.warning(f"{response['data']}") + for line in response['data']: + text += f"|{line['category']}|{line['datapoint']}|{line['value']}\n" + logging.debug(f"Text to return: {text}") + return text + + + def process(command, channel, username, params, files, conn): filters = '&'.join(settings.APIURL['cyberthreat']['filters']) if len(params)>0: - params = params[0].replace('[', '').replace(']', '').replace('hxxp','http').lower() - intro = f"cyberthreat.nl *Hosting Intelligence* API search for `{params}`:" + params = params[0].replace('[', '').replace(']', '').replace('hxxp','http').lower().strip() + data = { "source":"cyberthreat hosting intelligence", + "responses": [] +} + + data['intro'] = f"cyberthreat.nl *Hosting Intelligence* API search for `{params}`:" listitem = '`\n- `' try: - if params in actorlist: text = f"**{actorlist[params]['name'].capitalize()}**\n{actorlist[params]['description']}" + data['responses'].append({}) + data['responses'][0]['paragraph'] = "Bulletproof hosting provider" + data['responses'][0]['preamble'] = actorlist[params]['description'] + data['responses'][0]['data'] = list() + data['responses'][0]['data'].append({"category":"Actor", "datapoint":"name", "stixtype":"", "value":actorlist[params]['name']}) + data['responses'][0]['data'].append({"category":"Actor", "datapoint":"type", "stixtype":"", "value":actorlist[params]['type']}) + elif re.search(r"^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\.){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])(\:[0-65535]*)?$", params): results = cyberthreat.wget('addresses/'+params+'?'+filters) for address in results: last_seen = datetime.strptime(address['last_seen'], '%Y-%m-%dT%H:%M:%S.%f%z') - text=f"IPv4 address `{params}` {settings.confidence_tabel[address['credibility']]['level']} used by the actor **{address['actor'].capitalize()}**.\n" - text+=f"Last seen: {last_seen.strftime('%Y-%m-%d')}" + data['responses'].append(dict()) + data['responses'][0]['paragraph'] = "IP Lookup" + data['responses'][0]['preamble']=f"IPv4 address `{params}` {settings.confidence_tabel[address['credibility']]['level']} used by the actor **{address['actor'].capitalize()}**.\n" + data['responses'][0]['data'] = list() + data['responses'][0]['data'].append({"category":"Indicator", "datapoint":"ipv4 address", "stixtype":"ipv4-addr", "value":params}) + data['responses'][0]['data'].append({"category":"Indicator", "datapoint":"last seen", "stixtype":"", "value":last_seen.strftime('%Y-%m-%d')}) + data['responses'][0]['data'].append({"category":"Indicator", "datapoint":"actor", "stixtype":"", "value":address['actor'].capitalize() }) + data['responses'][0]['data'].append({"category":"Indicator", "datapoint":"actor type", "stixtype":"", "value":address['type'].capitalize() }) + data['responses'][0]['data'].append({"category":"Indicator", "datapoint":"credibility", "stixtype":"", "value": settings.confidence_tabel[address['credibility']]['short_description']}) + + elif params: extract = tldextract.extract(params) extracted_domain = extract.registered_domain @@ -75,6 +136,7 @@ def process(command, channel, username, params, files, conn): fqdnlist[domain]['actor'] = result.get('actor') fqdnlist[domain]['type'] = result.get('type') + logging.warning(f"fqdn list: {fqdnlist}") if len(fqdnlist): text='The domainname ' @@ -83,23 +145,37 @@ def process(command, channel, username, params, files, conn): """ for domain in fqdnlist: text+=f"`{domain}` {settings.confidence_tabel[fqdnlist[domain]['credibility']]['level']} hosted on the {fqdnlist[domain]['type']} network of actor **{fqdnlist[domain]['actor'].capitalize()}**.\n" - text+=f"Last seen: {fqdnlist[domain]['last_seen'].strftime('%Y-%m-%d')}.\n" - if len(fqdnlist[domain]['subdomains']): - text+=f"We have found the following subdomains: \n- `{listitem.join(fqdnlist[domain]['subdomains'])}`." - + + data['responses'].append({}) + data['responses'][0]['paragraph'] = "Domain search" + data['responses'][0]['preamble'] = text + data['responses'][0]['data'] = list() + data['responses'][0]['data'].append({"category":"Hosting", "datapoint":"domain", "stixtype":"", "value":domain}) + data['responses'][0]['data'].append({"category":"Hosting", "datapoint":"actor", "stixtype":"", "value":fqdnlist[domain]['actor']}) + data['responses'][0]['data'].append({"category":"Hosting", "datapoint":"credibility", "stixtype":"", "value":settings.confidence_tabel[fqdnlist[domain]['credibility']]['short_description']}) + data['responses'][0]['data'].append({"category":"Hosting", "datapoint":"last seen", "stixtype":"", "value":fqdnlist[domain]['last_seen'].strftime('%Y-%m-%d')}) + for item in fqdnlist[domain]['subdomains']: + data['responses'][0]['data'].append({"category":"Domain", "datapoint":"fqdn", "stixtype":"", "value":item}) + + else: """ In case the params doesnt even look like a valid domain name. """ return - if 'text' in locals(): - return {'messages': [ - {'text': intro + '\n' + text}, - ]} - #else: - # return {'messages': [ - # {'text': 'cyberthreat API searched for `%s` without result' % (params.strip(),)} - # ]} + if len(data['responses']): + text = formatdata(data) + logging.warning(f"Returned text: {text}") + return {'messages': [ + {'text': text} + ] } + + else: + return {'messages': [ + {'text': 'cyberthreat API searched for `%s` without result' % (params.strip(),)} + ]} except Exception as e: + raise e return {'messages': [ {'text': 'An error occurred searching cyberthreat for `%s`:\nError: `%s`' % (params, e)}, ]} + From 4da7cf518396f008a1bedebcb043edb4dfc6ec5f Mon Sep 17 00:00:00 2001 From: Tycho Date: Fri, 12 Jan 2024 16:02:51 +0100 Subject: [PATCH 03/19] Quick fix if settings.py is not found. Also allow some bots in all channels. --- matterbot.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/matterbot.py b/matterbot.py index c806ed6d..032bbd96 100755 --- a/matterbot.py +++ b/matterbot.py @@ -10,9 +10,9 @@ import os import sys import tempfile - import configargparse from mattermostdriver import Driver +from pathlib import Path class TokenAuth(): @@ -46,15 +46,19 @@ def __init__(self): sys.path.append(modulepath) for root, dirs, files in os.walk(modulepath): for module in fnmatch.filter(files, "command.py"): - module_name = root.split('/')[-1].lower() - module = importlib.import_module(module_name + '.' + 'command') - defaults = importlib.import_module(module_name + '.' + 'defaults') - overridesettings = importlib.import_module(module_name + '.' + 'settings') - if hasattr(defaults, 'HELP'): - HELP = defaults.HELP - if hasattr(overridesettings, 'HELP'): - HELP = overridesettings.HELP - self.commands[module_name] = {'binds': module.settings.BINDS, 'chans': module.settings.CHANS, 'help': HELP, 'process': getattr(module, 'process')} + if 'defaults.py' in files: + module_name = root.split('/')[-1].lower() + module = importlib.import_module(module_name + '.' + 'command') + defaults = importlib.import_module(module_name + '.' + 'defaults') + if hasattr(defaults, 'HELP'): + HELP = defaults.HELP + else: + HELP = 'No help available' + if 'settings.py' in files: + overridesettings = importlib.import_module(module_name + '.' + 'settings') + if hasattr(overridesettings, 'HELP'): + HELP = overridesettings.HELP + self.commands[module_name] = {'binds': module.settings.BINDS, 'chans': module.settings.CHANS, 'help': HELP, 'process': getattr(module, 'process')} # Start the websocket self.mmDriver.init_websocket(self.handle_raw_message) @@ -138,7 +142,7 @@ async def handle_post(self, data: dict): else: # User is asking for specific module help for module in self.commands: - if channelname in self.commands[module]['chans'] or (((my_id and userid) in channelname) and channelname in userchannels): + if channelname in self.commands[module]['chans'] or (((my_id and userid) in channelname) and channelname in userchannels or self.commands[module]['chans'] == 'any'): if command == '!help' and params and params[0] in self.commands[module]['binds']: try: text = '' @@ -200,6 +204,7 @@ async def handle_post(self, data: dict): await self.send_message(channelid, text, postid) for _ in concurrent.futures.as_completed(results): result = _.result() + if result and 'messages' in result: for message in result['messages']: if 'text' in message: @@ -228,6 +233,9 @@ async def handle_post(self, data: dict): except Exception as e: text = 'A Python error occurred: '+str(type(e))+': '+str(e) await self.send_message(channelid, text) + raise + + if __name__ == '__main__' : ''' From b816db02b1e99d45fc2bc5c8e4c6f7d0ea2785b5 Mon Sep 17 00:00:00 2001 From: Tycho Date: Fri, 12 Jan 2024 16:13:52 +0100 Subject: [PATCH 04/19] Move formatdata to matterbot --- commands/cyberthreat/command.py | 31 +------------------------------ matterbot.py | 20 +++++++++++++++++++- 2 files changed, 20 insertions(+), 31 deletions(-) diff --git a/commands/cyberthreat/command.py b/commands/cyberthreat/command.py index cfb4ba7b..da6f3fd8 100755 --- a/commands/cyberthreat/command.py +++ b/commands/cyberthreat/command.py @@ -53,24 +53,6 @@ """ -def formatdata(data): - logging.debug(f"working with {data}") - text = f"### {data['source']}\n" - if 'intro' in data: - text += f"{data['intro']}\n" - for response in data['responses']: - text += f"#### {response['paragraph']}\n" - if 'preamble' in response: - text += response['preamble'] - text +="\n\n" - if 'data' in response: - text += "|Category | Datapoint | Value|\n| :- | -: | :- |\n" - logging.warning(f"{response['data']}") - for line in response['data']: - text += f"|{line['category']}|{line['datapoint']}|{line['value']}\n" - logging.debug(f"Text to return: {text}") - return text - @@ -156,18 +138,7 @@ def process(command, channel, username, params, files, conn): data['responses'][0]['data'].append({"category":"Hosting", "datapoint":"last seen", "stixtype":"", "value":fqdnlist[domain]['last_seen'].strftime('%Y-%m-%d')}) for item in fqdnlist[domain]['subdomains']: data['responses'][0]['data'].append({"category":"Domain", "datapoint":"fqdn", "stixtype":"", "value":item}) - - - else: - """ In case the params doesnt even look like a valid domain name. """ - return - - if len(data['responses']): - text = formatdata(data) - logging.warning(f"Returned text: {text}") - return {'messages': [ - {'text': text} - ] } + return data else: return {'messages': [ diff --git a/matterbot.py b/matterbot.py index 032bbd96..71af4545 100755 --- a/matterbot.py +++ b/matterbot.py @@ -204,7 +204,25 @@ async def handle_post(self, data: dict): await self.send_message(channelid, text, postid) for _ in concurrent.futures.as_completed(results): result = _.result() - + if result and 'responses' in result: + data=result + logging.debug(f"working with {data}") + text = f"### {data['source']}\n" + if 'intro' in data: + text += f"{data['intro']}\n" + for response in data['responses']: + text += f"#### {response['paragraph']}\n" + if 'preamble' in response: + text += response['preamble'] + text +="\n\n" + if 'data' in response: + text += "|Category | Datapoint | Value|\n| :- | -: | :- |\n" + logging.warning(f"{response['data']}") + for line in response['data']: + text += f"|{line['category']}|{line['datapoint']}|{line['value']}\n" + logging.debug(f"Text to return: {text}") + await self.send_message(channelid, text, postid) + if result and 'messages' in result: for message in result['messages']: if 'text' in message: From d2060832479c31b87220ef0f5888eeb74b394a45 Mon Sep 17 00:00:00 2001 From: Tycho Date: Wed, 31 Jan 2024 20:41:11 +0100 Subject: [PATCH 05/19] Default help message --- matterbot.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/matterbot.py b/matterbot.py index 71af4545..f4fbb0e5 100755 --- a/matterbot.py +++ b/matterbot.py @@ -49,11 +49,12 @@ def __init__(self): if 'defaults.py' in files: module_name = root.split('/')[-1].lower() module = importlib.import_module(module_name + '.' + 'command') - defaults = importlib.import_module(module_name + '.' + 'defaults') - if hasattr(defaults, 'HELP'): - HELP = defaults.HELP - else: - HELP = 'No help available' + HELP = 'No help available' + if 'defaults.py' in files: + defaults = importlib.import_module(module_name + '.' + 'defaults') + if hasattr(defaults, 'HELP'): + HELP = defaults.HELP + if 'settings.py' in files: overridesettings = importlib.import_module(module_name + '.' + 'settings') if hasattr(overridesettings, 'HELP'): From 72b7c4ad3800c6571c1b216e94cf9ac976d8257a Mon Sep 17 00:00:00 2001 From: Tycho Date: Mon, 5 Feb 2024 14:31:34 +0100 Subject: [PATCH 06/19] Small fix to get the channel name in the logging. --- matterbot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/matterbot.py b/matterbot.py index a2db8675..10cd1d43 100755 --- a/matterbot.py +++ b/matterbot.py @@ -171,7 +171,7 @@ async def handle_message(self, message: dict): async def send_message(self, chanid, text, postid=None): try: - channame = channel.lower() + channame = self.channelid_to_chaninfo(chanid)['name'] log.info('Channel:' + channame + ' <- Message: (' + str(len(text)) + ' chars)') if len(text) > options.Matterbot['msglength']: # Mattermost message limit From f1b8fd23e83ed4f7dffd587db303c8872f828e85 Mon Sep 17 00:00:00 2001 From: Tycho Date: Mon, 5 Feb 2024 16:34:50 +0100 Subject: [PATCH 07/19] . --- matterbot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/matterbot.py b/matterbot.py index 10cd1d43..ae4c37ce 100755 --- a/matterbot.py +++ b/matterbot.py @@ -333,7 +333,7 @@ async def help_message(self, userid, params, chaninfo, rootid): commands.add('`' + bind + '`') text = "I know about: `"+'`, `'.join(sorted(options.Matterbot['helpcmds']))+"`, " + ', '.join(sorted(commands)) + " here.\n" text += "*Remember that not every command works everywhere: this depends on the configuration. Modules may offer additional help if you add the subcommand.*" - await self.send_message(chanid, text, rootid) + await self.send_message(channelid, text, rootid) else: # User is asking for specific module help for module in self.commands: From 02bbf5e3a66c59b8eefa8304da034274d7910729 Mon Sep 17 00:00:00 2001 From: Tycho Date: Mon, 5 Feb 2024 16:36:02 +0100 Subject: [PATCH 08/19] Fix for subcommands. --- matterbot.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/matterbot.py b/matterbot.py index ae4c37ce..3b79476e 100755 --- a/matterbot.py +++ b/matterbot.py @@ -395,9 +395,11 @@ async def handle_post(self, data: dict): addparams = False message = mline.split() for idx,word in enumerate(message): - if (word in self.binds and not message in options.Matterbot['helpcmds'] and not message in options.Matterbot['mapcmds']) or \ word in options.Matterbot['helpcmds'] or \ - word in options.Matterbot['mapcmds']: + word in options.Matterbot['mapcmds']) ) + if (word in self.binds or word in options.Matterbot['helpcmds'] or word in options.Matterbot['mapcmds']) and \ + (message[idx-1] not in options.Matterbot['helpcmds'] and + message[idx-1] not in options.Matterbot['mapcmds']): messages.append({'command':word,'parameters':[]}) addparams = True elif addparams: From 9faeb2be2202ac828209c65f2e2fe07ebef029f8 Mon Sep 17 00:00:00 2001 From: Tycho Date: Mon, 5 Feb 2024 21:04:26 +0100 Subject: [PATCH 09/19] Test for words in a message line. Work in progress --- matterbot.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/matterbot.py b/matterbot.py index 3b79476e..8ac29e27 100755 --- a/matterbot.py +++ b/matterbot.py @@ -395,11 +395,8 @@ async def handle_post(self, data: dict): addparams = False message = mline.split() for idx,word in enumerate(message): - word in options.Matterbot['helpcmds'] or \ - word in options.Matterbot['mapcmds']) ) if (word in self.binds or word in options.Matterbot['helpcmds'] or word in options.Matterbot['mapcmds']) and \ - (message[idx-1] not in options.Matterbot['helpcmds'] and - message[idx-1] not in options.Matterbot['mapcmds']): + (message[idx-1] not in options.Matterbot['helpcmds'] and message[idx-1] not in options.Matterbot['mapcmds']): messages.append({'command':word,'parameters':[]}) addparams = True elif addparams: From f7f38caae725b0b185b8fc7d8c65660f1a92b00d Mon Sep 17 00:00:00 2001 From: Tycho Date: Mon, 5 Feb 2024 21:05:45 +0100 Subject: [PATCH 10/19] Still WIP --- matterbot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/matterbot.py b/matterbot.py index 8ac29e27..27063d91 100755 --- a/matterbot.py +++ b/matterbot.py @@ -395,8 +395,8 @@ async def handle_post(self, data: dict): addparams = False message = mline.split() for idx,word in enumerate(message): - if (word in self.binds or word in options.Matterbot['helpcmds'] or word in options.Matterbot['mapcmds']) and \ - (message[idx-1] not in options.Matterbot['helpcmds'] and message[idx-1] not in options.Matterbot['mapcmds']): + if (word in self.binds or word in options.Matterbot['helpcmds'] or word in options.Matterbot['mapcmds']) and not \ + (message[idx-1] in options.Matterbot['helpcmds'] or message[idx-1] in options.Matterbot['mapcmds']): messages.append({'command':word,'parameters':[]}) addparams = True elif addparams: From 74f00064a4a41d166684cabb0933441469ca9715 Mon Sep 17 00:00:00 2001 From: Arnim Date: Mon, 5 Feb 2024 15:54:52 +0100 Subject: [PATCH 11/19] Fixes --- matterbot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/matterbot.py b/matterbot.py index 27063d91..b8ba5cfb 100755 --- a/matterbot.py +++ b/matterbot.py @@ -324,7 +324,7 @@ async def bind_message(self, userid, post, params, chaninfo, rootid): await self.send_message(chanid, message, rootid) async def help_message(self, userid, params, chaninfo, rootid): - channelid=chaninfo['id'] + chanid=chaninfo['id'] commands = set() if not params: for module in self.commands: From dcf85daff8392092c56f6037b2fcfa7dffeb6929 Mon Sep 17 00:00:00 2001 From: Arnim Date: Mon, 5 Feb 2024 15:57:20 +0100 Subject: [PATCH 12/19] Consistency in variables --- matterbot.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/matterbot.py b/matterbot.py index b8ba5cfb..14384929 100755 --- a/matterbot.py +++ b/matterbot.py @@ -127,14 +127,14 @@ def channame_to_chaninfo(self, channame): self.channelmapping['idtoname'][chaninfo['id']] = chaninfo return chaninfo - def channelid_to_chaninfo(self, channelid): - if channelid in self.channelmapping['idtoname']: - return self.channelmapping['idtoname'][channelid] + def chanid_to_chaninfo(self, chanid): + if chanid in self.channelmapping['idtoname']: + return self.channelmapping['idtoname'][chanid] else: try: - chaninfo = self.mmDriver.channels.get_channel(channelid) + chaninfo = self.mmDriver.channels.get_channel(chanid) except Exception as e: - log.error(f"Could not map {channelid}: {e}") + log.error(f"Could not map {chanid}: {e}") return None else: self.channelmapping['nametoid'][chaninfo['name']] = chaninfo @@ -371,9 +371,9 @@ async def help_message(self, userid, params, chaninfo, rootid): if args: text += '\n**Arguments**: `' + args + '`' if len(text)>0: - await self.send_message(channelid, text, rootid) + await self.send_message(chanid, text, rootid) except NameError: - await self.send_message(channelid, text, rootid) + await self.send_message(chanid, text, rootid) async def handle_post(self, data: dict): if 'sender_name' in data: @@ -383,8 +383,8 @@ async def handle_post(self, data: dict): return post = json.loads(data['post']) userid = post['user_id'] - channelid = post['channel_id'] - chaninfo = self.channelid_to_chaninfo(channelid) + chanid = post['channel_id'] + chaninfo = self.chanid_to_chaninfo(chanid) channame = chaninfo['name'] rootid = post['root_id'] if len(post['root_id']) else post['id'] messagelines = post['message'].splitlines() @@ -429,7 +429,7 @@ async def handle_post(self, data: dict): results.append(executor.submit(self.commands[task]['process'], command, channame, username, params, files, self.mmDriver)) except Exception as e: text = 'An error occurred within module: '+task+': '+str(type(e))+': '+e - await self.send_message(channelid, text, rootid) + await self.send_message(chanid, text, rootid) for _ in concurrent.futures.as_completed(results): result = _.result() if result and 'messages' in result: @@ -445,21 +445,21 @@ async def handle_post(self, data: dict): if not isinstance(payload, (bytes, bytearray)): payload = payload.encode() file_id = self.mmDriver.files.upload_file( - channel_id=channelid, + channel_id=chanid, files={'files': (filename, payload)} )['file_infos'][0]['id'] file_ids.append(file_id) - self.mmDriver.posts.create_post(options={'channel_id': channelid, + self.mmDriver.posts.create_post(options={'channel_id': chanid, 'message': text, 'file_ids': file_ids, }) else: - await self.send_message(channelid, text, rootid) + await self.send_message(chanid, text, rootid) else: - await self.send_message(channelid, text, rootid) + await self.send_message(chanid, text, rootid) except Exception as e: text = 'A Python error occurred: '+str(type(e))+': '+str(e) - await self.send_message(channelid, text, rootid) + await self.send_message(chanid, text, rootid) if __name__ == '__main__' : ''' From 42ab279a6a2f5fb06ef3b32a0f32842a756e8efb Mon Sep 17 00:00:00 2001 From: Arnim Date: Mon, 5 Feb 2024 20:48:32 +0100 Subject: [PATCH 13/19] Add Spiceworks --- modules/spiceworks/__init__.py | 0 modules/spiceworks/defaults.py | 8 +++++ modules/spiceworks/feed.py | 56 ++++++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+) create mode 100755 modules/spiceworks/__init__.py create mode 100755 modules/spiceworks/defaults.py create mode 100755 modules/spiceworks/feed.py diff --git a/modules/spiceworks/__init__.py b/modules/spiceworks/__init__.py new file mode 100755 index 00000000..e69de29b diff --git a/modules/spiceworks/defaults.py b/modules/spiceworks/defaults.py new file mode 100755 index 00000000..0e803e7c --- /dev/null +++ b/modules/spiceworks/defaults.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python3 + +NAME = "Spiceworks Tech News" +CHANNELS = ( + "newsfeed", +) +URL = "https://www.spiceworks.com/?feed=rss2&taxonomy=topic&term=tech" +ENTRIES = 10 diff --git a/modules/spiceworks/feed.py b/modules/spiceworks/feed.py new file mode 100755 index 00000000..42f80b93 --- /dev/null +++ b/modules/spiceworks/feed.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 + +# Every module must set the CHANNELS variable to indicate where information should be sent to in Mattermost +# +# Every module must implement the query() function. +# This query() function is called by the main worker and has only one parameter: the number of historic +# items that should be returned in the list. +# +# Every module must return a list [...] with 0, 1 ... n entries +# of 2-tuples: ('', '') +# +# : basically the destination channel in Mattermost, e.g. 'Newsfeed', 'Incident', etc. +# : the content of the message, MD format possible + +import bs4 +import feedparser +import re +from pathlib import Path +try: + from modules.spiceworks import defaults as settings +except ModuleNotFoundError: # local test run + import defaults as settings + if Path('settings.py').is_file(): + import settings +else: + if Path('modules/spiceworks/settings.py').is_file(): + try: + from modules.spiceworks import settings + except ModuleNotFoundError: # local test run + import settings + +def query(MAX=settings.ENTRIES): + items = [] + feed = feedparser.parse(settings.URL) + count = 0 + stripchars = '`\[\]\'\"' + regex = re.compile('[%s]' % stripchars) + while count < MAX: + try: + title = feed.entries[count].title + link = feed.entries[count].link + content = settings.NAME + ': [' + title + '](' + link + ')' + if len(feed.entries[count].description): + description = regex.sub('',bs4.BeautifulSoup(feed.entries[count].description,'lxml').get_text("\n")).strip().replace('\n','. ') + if len(description)>400: + description = description[:396]+' ...' + content += '\n>'+description+'\n' + for channel in settings.CHANNELS: + items.append([channel, content]) + count+=1 + except IndexError: + return items # No more items + return items + +if __name__ == "__main__": + print(query()) From 263a6f30574a1bba6d9e05c13869242bd14b78cf Mon Sep 17 00:00:00 2001 From: Arnim Date: Mon, 5 Feb 2024 20:51:13 +0100 Subject: [PATCH 14/19] Update README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 9414f75f..c4b890d6 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ Matterfeed reports news updates on a set schedule. The currently supported sourc | SebDraven | RSS | No | No | | SecureList News | RSS | No | No | | SecurityAffairs News | RSS | No | No | +| Spiceworks Tech News | RSS | No | No | | TheHackerNews News | RSS | No | No | | Threatpost News | RSS | No | No | | TrendMicro Research | RSS | No | No | From f805994339a760f52c43d5539a4ea79412723930 Mon Sep 17 00:00:00 2001 From: Tycho Date: Mon, 5 Feb 2024 21:56:13 +0100 Subject: [PATCH 15/19] Use chanid_to_chaninfo --- matterbot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/matterbot.py b/matterbot.py index 14384929..80befb09 100755 --- a/matterbot.py +++ b/matterbot.py @@ -171,7 +171,7 @@ async def handle_message(self, message: dict): async def send_message(self, chanid, text, postid=None): try: - channame = self.channelid_to_chaninfo(chanid)['name'] + channame = self.chanid_to_chaninfo(chanid)['name'] log.info('Channel:' + channame + ' <- Message: (' + str(len(text)) + ' chars)') if len(text) > options.Matterbot['msglength']: # Mattermost message limit @@ -264,7 +264,7 @@ def isallowed_module(self, user, module, chaninfo): async def bind_message(self, userid, post, params, chaninfo, rootid): command = post['message'].split()[0] chanid = post['channel_id'] - channame = self.chanid_to_channame(chanid) + channame = chaninfo['name'] username = self.userid_to_username(userid) messages = [] if not params: From 73b297e733cf737ad0a29c45b25e4d74d95e2e4e Mon Sep 17 00:00:00 2001 From: Tycho Date: Mon, 5 Feb 2024 23:18:32 +0100 Subject: [PATCH 16/19] Change logging to named logger. --- matterbot.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/matterbot.py b/matterbot.py index 80befb09..bade2f94 100755 --- a/matterbot.py +++ b/matterbot.py @@ -40,7 +40,7 @@ def __init__(self): try: self.mmDriver.login() except: - logging.error("Mattermost server is unreachable. Perhaps it is down, or you might have misconfigured one or more setting(s). Shutting down!") + log.error("Mattermost server is unreachable. Perhaps it is down, or you might have misconfigured one or more setting(s). Shutting down!") return False self.me = self.mmDriver.users.get_user(user_id='me') self.my_id = self.me['id'] @@ -151,7 +151,7 @@ async def update_bindmap(self): json.dump(self.bindmap,f) except: raise - logging.error("An error occurred updating the `%s` bindmap file; config changes were not successfully saved!" % (options.Matterbot['bindmap'],)) + log.error("An error occurred updating the `%s` bindmap file; config changes were not successfully saved!" % (options.Matterbot['bindmap'],)) async def handle_raw_message(self, raw_json: str): try: @@ -242,7 +242,7 @@ def isallowed_module(self, user, module, chaninfo): """ channame = chaninfo['name'] if chaninfo['type'] in ('O', 'P'): - logging.debug(f"Channel name: {chaninfo['name']}") + log.debug(f"Channel name: {chaninfo['name']}") if (channame or 'any') in self.commands[module]['chans']: return True elif chaninfo['type'] in ('D', 'G'): @@ -257,8 +257,8 @@ def isallowed_module(self, user, module, chaninfo): return True except: # Apparently the channel does not exist; perhaps it is spelled incorrectly or otherwise a misconfiguration? - logging.error("There is a non-existent channel set up in the bot bindings or configuration: %s" % (channame,)) - logging.info(f"User {user} is not allowed to use {module} in {channame}.") + log.error("There is a non-existent channel set up in the bot bindings or configuration: %s" % (channame,)) + log.info(f"User {user} is not allowed to use {module} in {channame}.") return False async def bind_message(self, userid, post, params, chaninfo, rootid): @@ -292,7 +292,7 @@ async def bind_message(self, userid, post, params, chaninfo, rootid): messages.append(text) else: if not userid in options.Matterbot['botadmins']: - logging.warn("User %s attempted to use a bind command without proper authorization.") % (userid,) + log.warning(f"User {userid} attempted to use a bind command without proper authorization.") text = "@" + username + ", you do not have permission to bind commands." else: all_channel_types = [self.chanid_to_channame(_['id']) for _ in self.mmDriver.channels.get_channels_for_user(self.my_id,self.my_team_id)] From 08e60a6b688d8961f39540e3e9e5f12bf06ec054 Mon Sep 17 00:00:00 2001 From: Tycho Date: Mon, 5 Feb 2024 23:18:58 +0100 Subject: [PATCH 17/19] Small bug that keeps reappearing. --- matterbot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/matterbot.py b/matterbot.py index bade2f94..f9a1245c 100755 --- a/matterbot.py +++ b/matterbot.py @@ -333,7 +333,7 @@ async def help_message(self, userid, params, chaninfo, rootid): commands.add('`' + bind + '`') text = "I know about: `"+'`, `'.join(sorted(options.Matterbot['helpcmds']))+"`, " + ', '.join(sorted(commands)) + " here.\n" text += "*Remember that not every command works everywhere: this depends on the configuration. Modules may offer additional help if you add the subcommand.*" - await self.send_message(channelid, text, rootid) + await self.send_message(chanid, text, rootid) else: # User is asking for specific module help for module in self.commands: From 664f538611edcf197cabf4092881e3caec63f889 Mon Sep 17 00:00:00 2001 From: Tycho Date: Mon, 5 Feb 2024 23:21:41 +0100 Subject: [PATCH 18/19] The word test seemed not to work. This version should recognize the binds as commands and use them as parameter if they are preceded by a helpmsg or bindmsg --- matterbot.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/matterbot.py b/matterbot.py index f9a1245c..8f069852 100755 --- a/matterbot.py +++ b/matterbot.py @@ -395,12 +395,16 @@ async def handle_post(self, data: dict): addparams = False message = mline.split() for idx,word in enumerate(message): - if (word in self.binds or word in options.Matterbot['helpcmds'] or word in options.Matterbot['mapcmds']) and not \ - (message[idx-1] in options.Matterbot['helpcmds'] or message[idx-1] in options.Matterbot['mapcmds']): + log.debug(f"(({word in self.binds}) and ({message[idx-1] not in options.Matterbot['helpcmds']} and {message[idx-1] not in options.Matterbot['mapcmds']} ) # In this case hand over the word to elif \ + or ({word in options.Matterbot['helpcmds']}) or (({word in options.Matterbot['mapcmds']}) and ({message[idx-1] not in options.Matterbot['helpcmds'] })) )") + if ((word in self.binds) and (message[idx-1] not in options.Matterbot['helpcmds'] and message[idx-1] not in options.Matterbot['mapcmds'] ) # In this case hand over the word to elif \ + or (word in options.Matterbot['helpcmds']) or ((word in options.Matterbot['mapcmds']) and (message[idx-1] not in options.Matterbot['helpcmds'] )) ): # word is a helpcmd or bind command messages.append({'command':word,'parameters':[]}) addparams = True elif addparams: messages[-1]['parameters'].append(word) + log.debug(f"Messages: {messages}") + for messagedict in messages: command = messagedict['command'] params = messagedict['parameters'] From 9ebdf2284e5a8790fa5ba361a394df33167ad3cb Mon Sep 17 00:00:00 2001 From: Tycho Date: Fri, 1 Mar 2024 17:15:43 +0100 Subject: [PATCH 19/19] Adding the validators package --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f8c56292..eacd26bd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,6 +10,7 @@ pyOpenSSL pypandoc PyYAML Requests -tldextract +tldextract # can be dropped soon! urllib3 weasyprint +validators \ No newline at end of file