From 672e61c37e45ce8f11718866acaf0f4523dd80c3 Mon Sep 17 00:00:00 2001 From: Michael Bauer Date: Thu, 23 Aug 2012 17:36:35 +0200 Subject: [PATCH 1/2] turn memcache off by default --- cymruwhois.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cymruwhois.py b/cymruwhois.py index aea9c53..622607e 100644 --- a/cymruwhois.py +++ b/cymruwhois.py @@ -110,7 +110,7 @@ def make_key(self, arg): else: return "cymruwhois:ip:" + arg - def __init__(self, host="whois.cymru.com", port=43, memcache_host='localhost:11211'): + def __init__(self, host="whois.cymru.com", port=43, memcache_host=None): self.host=host self.port=port self._connected=False From 7583f5cc440e6f19123847939f68575df3ba895e Mon Sep 17 00:00:00 2001 From: Michael Bauer Date: Fri, 24 Aug 2012 09:12:54 +0200 Subject: [PATCH 2/2] Added verbose lookups Added first version of verbose lookups: Caveat: the caching does not properly work with verbose ips (lookups in the form of "192.168.0.1 2012-08-20 11:30:00 GMT") - since caching is implemented on IP basis only. To do this the key mechanism has to be changed and with this the way queries are parsed. --- cymruwhois.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/cymruwhois.py b/cymruwhois.py index 622607e..2f57218 100644 --- a/cymruwhois.py +++ b/cymruwhois.py @@ -145,12 +145,14 @@ def read_and_discard(self): finally: self.socket.setblocking(1) - def _begin(self): + def _begin(self,verbose=False): """Explicitly connect and send BEGIN to start the lookup process""" self._connect() self._sendline("BEGIN") self._readline() #discard the message "Bulk mode; one IP per line. [2005-08-02 18:54:55 GMT]" self._sendline("PREFIX\nASNUMBER\nCOUNTRYCODE\nNOTRUNC") + if verbose: + self._sendline("VERBOSE") self._connected=True def disconnect(self): @@ -174,16 +176,16 @@ def cache(self, r): return self.c.set(self.make_key(r.key), r, 60*60*6) - def lookup(self, ip): + def lookup(self, ip,verbose=False): """Look up a single address. .. warning:: Do not call this function inside of a loop, the performance will be terrible. Instead, call lookupmany or lookupmany_dict """ - return list(self.lookupmany([ip]))[0] + return list(self.lookupmany([ip],verbose))[0] - def lookupmany(self, ips): + def lookupmany(self, ips,verbose=False): """Look up many ip addresses""" ips = [str(ip).strip() for ip in ips] @@ -192,22 +194,26 @@ def lookupmany(self, ips): not_cached = [ip for ip in batch if not cached.get(ip)] #print "cached:%d not_cached:%d" % (len(cached), len(not_cached)) if not_cached: - for rec in self._lookupmany_raw(not_cached): + for rec in self._lookupmany_raw(not_cached,verbose): cached[rec.key] = rec for ip in batch: + if verbose: + ip=ip.split()[0] # reduce IP + """ TODO: Implement better caching for verbose lookups + """ if ip in cached: yield cached[ip] - def lookupmany_dict(self, ips): + def lookupmany_dict(self, ips,verbose=False): """Look up many ip addresses, returning a dictionary of ip -> record""" ips = set(ips) - return dict((r.key, r) for r in self.lookupmany(ips)) + return dict((r.key, r) for r in self.lookupmany(ips,verbose)) - def _lookupmany_raw(self, ips): + def _lookupmany_raw(self, ips,verbose=False): """Do a look up for some ips""" if not self._connected: - self._begin() + self._begin(verbose) ips = set(ips) for ip in ips: self._sendline(ip) @@ -220,6 +226,8 @@ def _lookupmany_raw(self, ips): need -=1 continue parts=result.split("|") + if verbose: + parts=parts[0:4]+[parts[-1]] if len(parts)==5: r=record(*parts) else: