Merge pull request #26 from passivetotal/illuminate

Support RiskIQ Illuminate Reputation API

Author: aeetos

README.md

@@ -10,9 +10,11 @@
 ## Introduction
 
 This Python library provides an interface to the RiskIQ PassiveTotal Internet
-intelligence database. Security researchers and network defenders use 
-PassiveTotal to map threat actor infrastructure, profile hostnames & IP
-addresses, discover web technologies on Internet hosts.
+intelligence database and the RiskIQ Illuminate Reputation Score. 
+
+Security researchers and network defenders use RiskIQ PassiveTotal to map threat 
+actor infrastructure, profile hostnames & IP addresses, discover web technologies 
+on Internet hosts.
 
 Capabilites of this library include:
 * Credential management - protect API keys from accidental disclosure
@@ -25,7 +27,8 @@ To learn more about RiskIQ and start a free trial, visit [https://community.risk
 ## Getting Started
 
 ### Install the PassiveTotal Library
-The PassiveTotal Python library is availabe in pip under the package name `passivetotal`. Consider setting up a [virtual environment](https://docs.python.org/3/library/venv.html), then run:
+The PassiveTotal Python library is available in pip under the package name `passivetotal`. 
+Consider setting up a [virtual environment](https://docs.python.org/3/library/venv.html), then run:
 ```
 pip install passivetotal
 ```
@@ -36,15 +39,17 @@ Queries to the API must be authenticated with a PassiveTotal API key.
 1. Log in (or sign up) at [community.riskiq.com](https://community.riskiq.com)
 2. Access your profile by clicking the person icon in the upper-right corner of the page.
 3. Click on "Account Settings"
-4. Under "API Access", click "Show" to reveal your user or organization credentials.
+4. Under "API Access", click "Show" to reveal your API credentials.
 
 The identifier for your API account is alternatively called a "username", a "user", or
 an "API key". Look for an email address and use that value when prompted for your 
 "API username".
 
 The "API Secret" is a long string of characters that should be kept secure. It is
-the primary authentication method for your API account. Note your PassiveTotal 
-account may have a seperate "API Secret" for your organization.
+the primary authentication method for your API account. 
+
+Your PassiveTotal account may have a separate "API Secret" for your organization - when 
+available, **always use your organization key** unless you have a specific reason not to.
 
 
 ### Build a Config File

passivetotal/__init__.py

@@ -13,3 +13,4 @@
 from .libs.ssl import SslRequest
 from .libs.whois import WhoisRequest
 from .libs.generic import GenericRequest
+from .libs.illuminate import IlluminateRequest

passivetotal/analyzer/__init__.py

@@ -40,7 +40,8 @@ def init(**kwargs):
         (ProjectsRequest, 'Projects'), 
         (ServicesRequest, 'Services'),
         (SslRequest, 'SSL'), 
-        (WhoisRequest, 'Whois')
+        (WhoisRequest, 'Whois'),
+        (IlluminateRequest, 'Illuminate'),
     ]
     for c, name in api_classes:
         if 'username' in kwargs and 'api_key' in kwargs:

passivetotal/analyzer/_common.py

@@ -78,6 +78,32 @@ def sorted_by(self, field, reverse=False):
         sorted_results._records = sorted(self.all, key=lambda record: getattr(record, field), reverse=reverse)
         return sorted_results
 
+    def _ensure_firstlastseen(self):
+        """Ensure this record list has records of type FirstLastSeen."""
+        if not isinstance(self._records[0], FirstLastSeen):
+            raise TypeError('Cannot filter on a record type without firstseen / lastseen fields')
+    
+    def filter_dateseen_after(self, date_string):
+        self._ensure_firstlastseen()
+        dateobj = datetime.fromisoformat(date_string)
+        filtered_results = self._make_shallow_copy()
+        filtered_results._records = filter(lambda r: r.firstseen > dateobj, self._records)
+        return filtered_results
+
+    def filter_dateseen_before(self, date_string):
+        self._ensure_firstlastseen()
+        dateobj = datetime.fromisoformat(date_string)
+        filtered_results = self._make_shallow_copy()
+        filtered_results._records = filter(lambda r: r.lastseen < dateobj, self._records)
+        return filtered_results
+    
+    def filter_dateseen_between(self, start_date_string, end_date_string):
+        self._ensure_firstlastseen()
+        dateobj_start = datetime.fromisoformat(start_date_string)
+        dateobj_end = datetime.fromisoformat(end_date_string)
+        filtered_results = self._make_shallow_copy()
+        filtered_results._records = filter(lambda r: r.firstseen >= dateobj_start and r.lastseen <= dateobj_end, self._records)
+        return filtered_results
 
 
 class Record:

passivetotal/analyzer/hostname.py

@@ -11,10 +11,11 @@
 from passivetotal.analyzer.cookies import HasCookies
 from passivetotal.analyzer.trackers import HasTrackers
 from passivetotal.analyzer.components import HasComponents
+from passivetotal.analyzer.illuminate import HasReputation
 
 
 
-class Hostname(HasComponents, HasCookies, HasTrackers, HasHostpairs):
+class Hostname(HasComponents, HasCookies, HasTrackers, HasHostpairs, HasReputation):
 
     """Represents a hostname such as api.passivetotal.org.
     
@@ -44,6 +45,7 @@ def __new__(cls, hostname):
             self._pairs = {}
             self._pairs['parents'] = None
             self._pairs['children'] = None
+            self._reputation = None
         return self
 
     def __str__(self):
@@ -52,6 +54,27 @@ def __str__(self):
     def __repr__(self):
         return "Hostname('{}')".format(self.hostname)
 
+    def reset(self, prop=None):
+        """Reset this instance to clear all (default) or one cached properties.
+
+        Useful when changing module-level settings such as analyzer.set_date_range().
+
+        :param str prop: Property to reset (optional, if none provided all values will be cleared)
+        """
+        resettable_fields = ['whois','resolutions','summary','components',
+                             'cookies','trackers','pairs','reputation']
+        if not prop:
+            for field in resettable_fields:
+                setattr(self, '_'+field, None)
+            self._reset_hostpairs()
+        else:
+            if prop not in resettable_fields:
+                raise ValueError('Invalid property to reset')
+            if prop == 'pairs':
+                self._reset_hostpairs()
+            else:
+                setattr(self, '_'+prop, None)
+
     def get_host_identifier(self):
         """Alias for the hostname as a string.
         

passivetotal/analyzer/hostpairs.py

@@ -11,9 +11,10 @@ class HostpairHistory(RecordList, PagedRecordList):
 
     """Historical connections between hosts."""
 
-    def __init__(self, api_response, direction=None):
+    def __init__(self, api_response=None, direction=None):
         self._direction = direction
-        self.parse(api_response)
+        if api_response:
+            self.parse(api_response)
 
     def _get_shallow_copy_fields(self):
         return ['_totalrecords','_direction']
@@ -115,6 +116,12 @@ class HasHostpairs:
 
     """An object with hostpair history."""
 
+    def _reset_hostpairs(self):
+        """Reset the instance hostpairs private attributes."""
+        self._pairs = {}
+        self._pairs['parents'] = None
+        self._pairs['children'] = None
+
     def _api_get_hostpairs(self, direction, start_date=None, end_date=None):
         """Query the hostpairs API for the parent or child relationships.
         

passivetotal/analyzer/illuminate.py

@@ -0,0 +1,77 @@
+from datetime import datetime
+import pprint
+from functools import total_ordering
+from passivetotal.analyzer import get_api, get_config
+
+
+@total_ordering
+class ReputationScore:
+
+    """RiskIQ Illuminate Reputation profile for a hostname or an IP."""
+
+    def __init__(self, api_response):
+        self._response = api_response
+    
+    def __str__(self):
+        return '{0.score} ({0.classification})'.format(self)
+    
+    def __repr__(self):
+        return '<ReputationScore {0.score} "{0.classification}">'.format(self)
+    
+    def __int__(self):
+        return self.score
+    
+    def __gt__(self, other):
+        return self.score > other
+    
+    def __eq__(self, other):
+        return self.score == other
+
+    @property
+    def score(self):
+        """Reputation score as an integer ranging from 0-100.
+
+        Higher values indicate a greater likelihood of maliciousness.
+        """
+        return self._response.get('score')
+    
+    @property
+    def classification(self):
+        """Reputation classification as a string. 
+
+        Typical values include GOOD, SUSPICIOUS, MALICIOUS, or UNKNOWN.
+        """
+        return self._response.get('classification')
+    
+    @property
+    def rules(self):
+        """List of rules that informed the reputation score.
+
+        Returns a list of dictionaries.
+        """
+        return self._response.get('rules')
+
+
+
+class HasReputation:
+
+    """An object with a RiskIQ Illuminate Reputation score."""
+
+    def _api_get_reputation(self):
+        """Query the reputation endpoint."""
+
+        response = get_api('Illuminate').get_reputation(
+            query=self.get_host_identifier()
+        )
+        self._reputation = ReputationScore(response)
+        return self._reputation
+
+    @property
+    def reputation(self):
+        """RiskIQ Illuminate Reputation profile for a hostname or IP.
+
+        :rtype: :class:`passivetotal.analyzer.illuminate.ReputationScore`
+        """
+        if getattr(self, '_reputation'):
+            return self._reputation
+        return self._api_get_reputation()

passivetotal/analyzer/ip.py

@@ -10,10 +10,11 @@
 from passivetotal.analyzer.cookies import HasCookies
 from passivetotal.analyzer.trackers import HasTrackers
 from passivetotal.analyzer.components import HasComponents
+from passivetotal.analyzer.illuminate import HasReputation
 
 
 
-class IPAddress(HasComponents, HasCookies, HasHostpairs, HasTrackers):
+class IPAddress(HasComponents, HasCookies, HasHostpairs, HasTrackers, HasReputation):
 
     """Represents an IPv4 address such as 8.8.8.8
     
@@ -43,6 +44,7 @@ def __new__(cls, ip):
             self._pairs = {}
             self._pairs['parents'] = None
             self._pairs['children'] = None
+            self._reputation = None
         return self
 
     def __str__(self):
@@ -51,6 +53,28 @@ def __str__(self):
     def __repr__(self):
         return "IPAddress('{}')".format(self.ip)
 
+    def reset(self, prop=None):
+        """Reset this instance to clear all (default) or one cached properties.
+
+        Useful when changing module-level settings such as analyzer.set_date_range().
+
+        :param str prop: Property to reset (optional, if none provided all values will be cleared)
+        """
+        resettable_fields = ['whois','resolutions','summary','components',
+                             'services','ssl_history',
+                             'cookies','trackers','pairs','reputation']
+        if not prop:
+            for field in resettable_fields:
+                setattr(self, '_'+field, None)
+            self._reset_hostpairs()
+        else:
+            if prop not in resettable_fields:
+                raise ValueError('Invalid property to reset')
+            if prop == 'pairs':
+                self._reset_hostpairs()
+            else:
+                setattr(self, '_'+prop, None)
+    
     def get_host_identifier(self):
         """Alias for the IP address as a string.
         

passivetotal/cli/client.py

@@ -17,6 +17,7 @@
 from passivetotal.libs.cookies import CookiesRequest, CookiesResponse
 from passivetotal.libs.services import ServicesRequest, ServicesResponse
 from passivetotal.libs.projects import ProjectsRequest, ProjectsResponse
+from passivetotal.libs.illuminate import IlluminateRequest, IlluminateReputationResponse
 from passivetotal.response import Response
 
 __author__ = 'Brandon Dixon (PassiveTotal)'
@@ -280,6 +281,21 @@ def call_projects(args):
     data = ProjectsResponse.process(response)
     return data
 
+def call_illuminate(args):
+    client = IlluminateRequest.from_config()
+    if args.illuminate_cmd == 'reputation':
+        results = []
+        for host in args.hosts:
+            try:
+                response = client.get_reputation(query=host)
+            except Exception as e:
+                response = {}
+            response.update({'host': host})
+            if args.brief:
+                del(response['rules'])
+            results.append(response)
+        data = IlluminateReputationResponse.process(results)
+    return data
 
 def write_output(results, arguments):
     """Format data based on the type.
@@ -506,7 +522,15 @@ def main():
     projects.add_argument('--format', choices=['json'], default='json',
                         help="Format of the output from the query")
 
-
+    illuminate = subs.add_parser('illuminate', help="Query RiskIQ Illuminate API")
+    illuminate.add_argument('--reputation', dest='illuminate_cmd', action='store_const', const='reputation',
+                        help="Get hostname or IP reputation from RiskIQ Illuminate.")
+    illuminate.add_argument('--format', choices=['json','csv','text'], default='json',
+                        help="Format of the output from the query")
+    illuminate.add_argument('--brief', action='store_true',
+                        help="Create a brief output; for reputation, prints score and classification only")
+    illuminate.add_argument('hosts', metavar='query', nargs='+',
+                        help="One or more hostnames or IPs")
     args, unknown = parser.parse_known_args()
     data = None
 
@@ -535,6 +559,8 @@ def main():
             data = call_services(args)
         elif args.cmd == 'projects':
             data = call_projects(args)
+        elif args.cmd == 'illuminate':
+            data = call_illuminate(args)
         else:
             parser.print_usage()
             sys.exit(1)