Standard message format

commands/cyberthreat/command.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 import re
-import requests
+import logging
 import tldextract
 from pathlib import Path
 from datetime import datetime
@@ -31,26 +31,69 @@
 for actor in results['results']:
     actorlist[actor['name']]=actor
 
+"""
+Input data in the format
+
+{
+    "source":"provider",
+    "responses": [
+        {
+            "paragraph":"subtitle",
+            "preamble":"introduction to source",
+            "data": [
+                {"category":"category", "datapoint":"datapoint", "stixtype":"ipv4-addr", "value":"value"},
+                {"category":"category", "datapoint":"datapoint", "value":"value"}
+            ]
+        }
+    ]
+}
+
+Converts to a message text and possibly an attachment.
+The text can have multiple paragraph with a short introduction
+
+"""
+
+
+
 
 def process(command, channel, username, params, files, conn):
     filters = '&'.join(settings.APIURL['cyberthreat']['filters'])
 
 
     if len(params)>0:
-        params = params[0].replace('[', '').replace(']', '').replace('hxxp','http').lower()
-        intro = f"cyberthreat.nl *Hosting Intelligence* API search for `{params}`:"
+        params = params[0].replace('[', '').replace(']', '').replace('hxxp','http').lower().strip()
+        data = { "source":"cyberthreat hosting intelligence",
+                    "responses": []
+}
+
+        data['intro'] = f"cyberthreat.nl *Hosting Intelligence* API search for `{params}`:"
         listitem = '`\n- `'
         try:
-
             if params in actorlist:
                 text = f"**{actorlist[params]['name'].capitalize()}**\n{actorlist[params]['description']}"
+                data['responses'].append({})
+                data['responses'][0]['paragraph'] = "Bulletproof hosting provider"
+                data['responses'][0]['preamble']  = actorlist[params]['description']
+                data['responses'][0]['data'] = list()
+                data['responses'][0]['data'].append({"category":"Actor", "datapoint":"name", "stixtype":"", "value":actorlist[params]['name']})
+                data['responses'][0]['data'].append({"category":"Actor", "datapoint":"type", "stixtype":"", "value":actorlist[params]['type']})
+
 
             elif re.search(r"^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\.){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])(\:[0-65535]*)?$", params):
                 results = cyberthreat.wget('addresses/'+params+'?'+filters)
                 for address in results:
                     last_seen = datetime.strptime(address['last_seen'], '%Y-%m-%dT%H:%M:%S.%f%z')
-                    text=f"IPv4 address `{params}` {settings.confidence_tabel[address['credibility']]['level']} used by the actor **{address['actor'].capitalize()}**.\n"
-                    text+=f"Last seen: {last_seen.strftime('%Y-%m-%d')}"
+                    data['responses'].append(dict())
+                    data['responses'][0]['paragraph'] = "IP Lookup"
+                    data['responses'][0]['preamble']=f"IPv4 address `{params}` {settings.confidence_tabel[address['credibility']]['level']} used by the actor **{address['actor'].capitalize()}**.\n"
+                    data['responses'][0]['data'] = list()
+                    data['responses'][0]['data'].append({"category":"Indicator", "datapoint":"ipv4 address", "stixtype":"ipv4-addr", "value":params})
+                    data['responses'][0]['data'].append({"category":"Indicator", "datapoint":"last seen", "stixtype":"", "value":last_seen.strftime('%Y-%m-%d')})
+                    data['responses'][0]['data'].append({"category":"Indicator", "datapoint":"actor", "stixtype":"", "value":address['actor'].capitalize() })
+                    data['responses'][0]['data'].append({"category":"Indicator", "datapoint":"actor type", "stixtype":"", "value":address['type'].capitalize() })
+                    data['responses'][0]['data'].append({"category":"Indicator", "datapoint":"credibility", "stixtype":"", "value": settings.confidence_tabel[address['credibility']]['short_description']})
+
+
             elif params:
                 extract = tldextract.extract(params)
                 extracted_domain = extract.registered_domain
@@ -75,6 +118,7 @@ def process(command, channel, username, params, files, conn):
                         fqdnlist[domain]['actor'] = result.get('actor')
                         fqdnlist[domain]['type']  = result.get('type')
 
+                    logging.warning(f"fqdn list: {fqdnlist}")
                     if len(fqdnlist):
                         text='The domainname '
 
@@ -83,23 +127,26 @@ def process(command, channel, username, params, files, conn):
                         """
                         for domain in fqdnlist:
                             text+=f"`{domain}` {settings.confidence_tabel[fqdnlist[domain]['credibility']]['level']} hosted on the {fqdnlist[domain]['type']} network of actor **{fqdnlist[domain]['actor'].capitalize()}**.\n"
-                            text+=f"Last seen: {fqdnlist[domain]['last_seen'].strftime('%Y-%m-%d')}.\n"
-                            if len(fqdnlist[domain]['subdomains']):
-                                text+=f"We have found the following subdomains: \n- `{listitem.join(fqdnlist[domain]['subdomains'])}`."
-
-                else:
-                    """ In case the params doesnt even look like a valid domain name. """
-                    return
-
-            if 'text' in locals():
-                return {'messages': [
-                    {'text': intro + '\n' + text},
-                ]}
-            #else:
-            #    return {'messages': [
-            #        {'text': 'cyberthreat API searched for `%s` without result' % (params.strip(),)}
-            #    ]}
+
+                            data['responses'].append({})
+                            data['responses'][0]['paragraph'] = "Domain search"
+                            data['responses'][0]['preamble']  = text
+                            data['responses'][0]['data'] = list()
+                            data['responses'][0]['data'].append({"category":"Hosting", "datapoint":"domain", "stixtype":"", "value":domain})
+                            data['responses'][0]['data'].append({"category":"Hosting", "datapoint":"actor", "stixtype":"", "value":fqdnlist[domain]['actor']})
+                            data['responses'][0]['data'].append({"category":"Hosting", "datapoint":"credibility", "stixtype":"", "value":settings.confidence_tabel[fqdnlist[domain]['credibility']]['short_description']})
+                            data['responses'][0]['data'].append({"category":"Hosting", "datapoint":"last seen", "stixtype":"", "value":fqdnlist[domain]['last_seen'].strftime('%Y-%m-%d')})
+                            for item in fqdnlist[domain]['subdomains']:
+                                data['responses'][0]['data'].append({"category":"Domain", "datapoint":"fqdn", "stixtype":"", "value":item})
+                return data
+
+            else:
+               return {'messages': [
+                   {'text': 'cyberthreat API searched for `%s` without result' % (params.strip(),)}
+               ]}
         except Exception as e:
+            raise e
             return {'messages': [
                 {'text': 'An error occurred searching cyberthreat for `%s`:\nError: `%s`' % (params, e)},
             ]}
+

matterbot.py

@@ -40,7 +40,7 @@ def __init__(self):
         try:
             self.mmDriver.login()
         except:
-            logging.error("Mattermost server is unreachable. Perhaps it is down, or you might have misconfigured one or more setting(s). Shutting down!")
+            log.error("Mattermost server is unreachable. Perhaps it is down, or you might have misconfigured one or more setting(s). Shutting down!")
             return False
         self.me = self.mmDriver.users.get_user(user_id='me')
         log.info("Who am I: %s" % (self.me,))
@@ -121,7 +121,7 @@ async def update_bindmap(self):
                 json.dump(self.bindmap,f)
         except:
             raise
-            logging.error("An error occurred updating the `%s` bindmap file; config changes were not successfully saved!" % (options.Matterbot['bindmap'],))
+            log.error("An error occurred updating the `%s` bindmap file; config changes were not successfully saved!" % (options.Matterbot['bindmap'],))
 
     async def handle_raw_message(self, raw_json: str):
         try:
@@ -141,7 +141,7 @@ async def handle_message(self, message: dict):
 
     async def send_message(self, chanid, text, postid=None):
         try:
-            channame = self.chanid_to_channame(chanid)
+            channame = self.chanid_to_chaninfo(chanid)['name']
             log.info('Channel:' + channame + ' <- Message: (' + str(len(text)) + ' chars)')
 
             if len(text) > options.Matterbot['msglength']: # Mattermost message limit
@@ -247,7 +247,7 @@ def isallowed_module(self, user, module, chaninfo):
         """
         channame = chaninfo['name']
         if chaninfo['type'] in ('O', 'P'):
-            logging.debug(f"Channel name: {chaninfo['name']}")
+            log.debug(f"Channel name: {chaninfo['name']}")
             if (channame or 'any') in self.commands[module]['chans']:
                 return True
         elif chaninfo['type'] in ('D', 'G'):
@@ -262,14 +262,14 @@ def isallowed_module(self, user, module, chaninfo):
                         return True
                 except:
                     # Apparently the channel does not exist; perhaps it is spelled incorrectly or otherwise a misconfiguration?
-                    logging.error("There is a non-existent channel set up in the bot bindings or configuration: %s" % (channame,))
-        logging.info(f"User {user} is not allowed to use {module} in {channame}.")
+                    log.error("There is a non-existent channel set up in the bot bindings or configuration: %s" % (channame,))
+        log.info(f"User {user} is not allowed to use {module} in {channame}.")
         return False
 
     async def bind_message(self, userid, post, params, chaninfo, rootid):
         command = post['message'].split()[0]
         chanid = post['channel_id']
-        channame = self.chanid_to_channame(chanid)
+        channame = chaninfo['name']
         username = self.userid_to_username(userid)
         messages = []
         if not params:
@@ -400,13 +400,16 @@ async def handle_post(self, data: dict):
                 addparams = False
                 message = mline.split()
                 for idx,word in enumerate(message):
-                    if (word in self.binds and not message in options.Matterbot['helpcmds'] and not message in options.Matterbot['mapcmds']) or \
-                        word in options.Matterbot['helpcmds'] or \
-                        word in options.Matterbot['mapcmds']:
+                    log.debug(f"(({word in self.binds}) and ({message[idx-1] not in options.Matterbot['helpcmds']} and {message[idx-1] not in options.Matterbot['mapcmds']} ) # In this case hand over the word to elif \
+                     or ({word in options.Matterbot['helpcmds']}) or (({word in options.Matterbot['mapcmds']}) and ({message[idx-1] not in options.Matterbot['helpcmds'] }))  )")
+                    if ((word in self.binds) and (message[idx-1] not in options.Matterbot['helpcmds'] and message[idx-1] not in options.Matterbot['mapcmds'] ) # In this case hand over the word to elif \
+                         or (word in options.Matterbot['helpcmds']) or ((word in options.Matterbot['mapcmds']) and (message[idx-1] not in options.Matterbot['helpcmds'] ))  ): # word is a helpcmd or bind command
                         messages.append({'command':word,'parameters':[]})
                         addparams = True
                     elif addparams:
                         messages[-1]['parameters'].append(word)
+            log.debug(f"Messages: {messages}")
+
             for messagedict in messages:
                 command = messagedict['command']
                 params = messagedict['parameters']

requirements.txt

@@ -10,6 +10,7 @@ pyOpenSSL
 pypandoc
 PyYAML
 Requests
-tldextract
+tldextract # can be dropped soon!
 urllib3
 weasyprint
+validators