diff --git a/.travis.requirements3.txt b/.travis.requirements3.txt new file mode 100644 index 000000000..ce105c411 --- /dev/null +++ b/.travis.requirements3.txt @@ -0,0 +1,19 @@ +#MySQL-python +PyYAML +#beanstalkc +riemann-client +boto +configobj +docker-py +kitchen +mock +psutil +pymongo<3.0 +statsd +#pyutmp +redis +simplejson +setproctitle +psycopg2 +#PySensors +pysnmp diff --git a/.travis.yml b/.travis.yml index fddae9303..3704ec481 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,9 @@ python: # - "2.5" - "2.6" - "2.7" + - "3.4" + - "3.5" + - "3.6" cache: directories: @@ -14,7 +17,8 @@ cache: # command to install dependencies, e.g. pip install -r requirements.txt install: - pip install Cython - - pip install -r .travis.requirements.txt + - if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then pip install -r .travis.requirements.txt; fi + - if [[ $TRAVIS_PYTHON_VERSION == 3* ]]; then pip install -r .travis.requirements3.txt; fi - pip install pep8==1.5.7 - pip install coveralls - if [[ $TRAVIS_PYTHON_VERSION == '2.6' ]]; then pip install unittest2; fi diff --git a/Makefile b/Makefile index a40037f20..e94ae958d 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,8 @@ PROJECT=diamond VERSION :=$(shell bash version.sh ) RELEASE :=$(shell ls -1 dist/*.noarch.rpm 2>/dev/null | wc -l ) HASH :=$(shell git rev-parse HEAD ) -DISTRO=precise +DISTRO:=$(shell python -c "import platform;print(platform.linux_distribution()[0])") +PYTHON?=python3 all: @echo "make run - Run Diamond from this directory" @@ -40,19 +41,19 @@ docs: version make test sdist: version - ./setup.py sdist --prune + $(PYTHON) setup.py sdist bdist: version - ./setup.py bdist --prune + $(PYTHON) setup.py bdist bdist_wheel: version - USE_SETUPTOOLS=1 ./setup.py bdist_wheel + $(PYTHON) setup.py bdist_wheel install: version - ./setup.py install --root $(DESTDIR) + $(PYTHON) setup.py install --root $(DESTDIR) develop: version - USE_SETUPTOOLS=1 ./setup.py develop + $(PYTHON) setup.py develop rpm: buildrpm @@ -62,6 +63,12 @@ buildrpm: sdist --build-requires='python, python-configobj, python-setuptools' \ --requires='python, python-configobj, python-setuptools' +buildrpmpy3: sdist + $(PYTHON) setup.py bdist_rpm \ + --release=`ls dist/*.noarch.rpm | wc -l` \ + --build-requires='python3, python3-configobj' \ + --requires='python3, python3-configobj' + deb: builddeb sdeb: buildsourcedeb @@ -69,7 +76,7 @@ sdeb: buildsourcedeb builddeb: version dch --newversion $(VERSION) --distribution unstable --force-distribution -b "Last Commit: $(shell git log -1 --pretty=format:'(%ai) %H %cn <%ce>')" dch --release "new upstream" - ./setup.py sdist --prune + ./setup.py sdist mkdir -p build tar -C build -zxf dist/$(PROJECT)-$(VERSION).tar.gz (cd build/$(PROJECT)-$(VERSION) && debuild -us -uc -v$(VERSION)) @@ -78,7 +85,7 @@ builddeb: version buildsourcedeb: version dch --newversion $(VERSION)~$(DISTRO) --distribution $(DISTRO) --force-distribution -b "Last Commit: $(shell git log -1 --pretty=format:'(%ai) %H %cn <%ce>')" dch --release "new upstream" - ./setup.py sdist --prune + ./setup.py sdist mkdir -p build tar -C build -zxf dist/$(PROJECT)-$(VERSION).tar.gz (cd build/$(PROJECT)-$(VERSION) && debuild -S -sa -v$(VERSION)) @@ -96,6 +103,7 @@ clean: ./setup.py clean rm -rf dist build MANIFEST .tox *.log find . -name '*.pyc' -delete + find -name __pycache__ -delete version: ./version.sh > version.txt diff --git a/Vagrantfile b/Vagrantfile index aec84d004..643ed8b52 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -81,12 +81,17 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| c.vm.provision "shell", inline: "sudo rpm -ivh http://dl.fedoraproject.org/pub/epel/7/x86_64/e/epel-release-7-9.noarch.rpm" c.vm.provision "shell", inline: "sudo yum install -y git rpm-build python-configobj python-test python-mock tree vim-enhanced MySQL-python htop gcc" + # Install python 3 + c.vm.provision "shell", inline: "sudo yum install -y python34 python34-devel" + c.vm.provision "shell", inline: "curl -O https://bootstrap.pypa.io/get-pip.py && sudo /usr/bin/python3.4 get-pip.py" + # Install python libraries needed by specific collectors c.vm.provision "shell", inline: "sudo yum install -y postgresql-devel" # req for psycopg2 c.vm.provision "shell", inline: "sudo yum install -y Cython" # req for pyutmp c.vm.provision "shell", inline: "sudo yum install -y lm_sensors-devel lm_sensors python-devel" # req for pyutmp c.vm.provision "shell", inline: "sudo yum install -y python-pip" c.vm.provision "shell", inline: "sudo pip install -r /vagrant/.travis.requirements.txt" + c.vm.provision "shell", inline: "sudo pip3 install -r /vagrant/.travis.requirements3.txt" # Setup Diamond to run as a service c.vm.provision "shell", inline: "sudo yum install -y python-setuptools" @@ -104,6 +109,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| # Build Diamond docs and run tests c.vm.provision "shell", inline: "sudo pip install pep8==1.5.7" + c.vm.provision "shell", inline: "sudo pip3 install pep8==1.5.7" c.vm.provision "shell", inline: "echo 'Build docs...' && python /vagrant/build_doc.py" c.vm.provision "shell", inline: "echo 'Running tests...' && python /vagrant/test.py" c.vm.provision "shell", inline: "echo 'Running pep8...' && pep8 --config=/vagrant/.pep8 /vagrant/src /vagrant/bin/diamond /vagrant/bin/diamond-setup /vagrant/build_doc.py /vagrant/setup.py /vagrant/test.py" diff --git a/bin/diamond b/bin/diamond index 9e29d7aba..1a2a81494 100755 --- a/bin/diamond +++ b/bin/diamond @@ -1,5 +1,6 @@ #!/usr/bin/env python # coding=utf-8 +from __future__ import print_function import os import sys @@ -110,7 +111,7 @@ def main(): gid = -1 if options.version: - print "Diamond version %s" % (get_diamond_version()) + print("Diamond version %s" % (get_diamond_version())) sys.exit(0) # Initialize Config @@ -118,8 +119,8 @@ def main(): if os.path.exists(options.configfile): config = configobj.ConfigObj(options.configfile) else: - print >> sys.stderr, "ERROR: Config file: %s does not exist." % ( - options.configfile) + print("ERROR: Config file: %s does not exist." % ( + options.configfile), file=sys.stderr) parser.print_help(sys.stderr) sys.exit(1) @@ -127,10 +128,10 @@ def main(): log = setup_logging(options.configfile, options.log_stdout) # Pass the exit up stream rather then handle it as an general exception - except SystemExit, e: + except SystemExit as e: raise SystemExit - except Exception, e: + except Exception as e: import traceback sys.stderr.write("Unhandled exception: %s" % str(e)) sys.stderr.write("traceback: %s" % traceback.format_exc()) @@ -146,7 +147,7 @@ def main(): # Read existing pid file try: - pf = file(options.pidfile, 'r') + pf = open(options.pidfile, 'r') pid = int(pf.read().strip()) pf.close() except (IOError, ValueError): @@ -159,11 +160,11 @@ def main(): # Pid is not real os.unlink(options.pidfile) pid = None - print >> sys.stderr, ( - "WARN: Bogus pid file was found. I deleted it.") + print("WARN: Bogus pid file was found. I deleted it.", + file=sys.stderr) else: - print >> sys.stderr, ( - "ERROR: Pidfile exists. Server already running?") + print("ERROR: Pidfile exists. Server already running?", + file=sys.stderr) sys.exit(1) # Get final GIDs @@ -185,9 +186,10 @@ def main(): # Write pid file pid = str(os.getpid()) try: - pf = file(options.pidfile, 'w+') - except IOError, e: - print >> sys.stderr, "Failed to write PID file: %s" % (e) + pf = open(options.pidfile, 'w+') + except IOError as e: + print("Failed to write PID file: %s" % (e), + file=sys.stderr) sys.exit(1) pf.write("%s\n" % pid) pf.close() @@ -211,8 +213,9 @@ def main(): # Set UID os.setuid(uid) - except Exception, e: - print >> sys.stderr, "ERROR: Failed to set UID/GID. %s" % (e) + except Exception as e: + print("ERROR: Failed to set UID/GID. %s" % (e), + file=sys.stderr) sys.exit(1) # Log @@ -236,8 +239,8 @@ def main(): if pid > 0: # Exit first paren sys.exit(0) - except OSError, e: - print >> sys.stderr, "Failed to fork process." % (e) + except OSError as e: + print("Failed to fork process." % (e), file=sys.stderr) sys.exit(1) # Decouple from parent environmen os.setsid() @@ -248,8 +251,8 @@ def main(): if pid > 0: # Exit second paren sys.exit(0) - except OSError, e: - print >> sys.stderr, "Failed to fork process." % (e) + except OSError as e: + print("Failed to fork process." % (e), file=sys.stderr) sys.exit(1) # Close file descriptors so that we can detach sys.stdout.close() @@ -268,8 +271,8 @@ def main(): # Write pid file pid = str(os.getpid()) try: - pf = file(options.pidfile, 'w+') - except IOError, e: + pf = open(options.pidfile, 'w+') + except IOError as e: log.error("Failed to write child PID file: %s" % (e)) sys.exit(1) pf.write("%s\n" % pid) @@ -317,10 +320,10 @@ def main(): server.run() # Pass the exit up stream rather then handle it as an general exception - except SystemExit, e: + except SystemExit as e: raise SystemExit - except Exception, e: + except Exception as e: import traceback log.error("Unhandled exception: %s" % str(e)) log.error("traceback: %s" % traceback.format_exc()) diff --git a/bin/diamond-setup b/bin/diamond-setup index 9bf9c2cb4..eb9f1a5e5 100755 --- a/bin/diamond-setup +++ b/bin/diamond-setup @@ -1,5 +1,6 @@ #!/usr/bin/env python ########################################################################## +from __future__ import print_function import os import sys @@ -63,10 +64,10 @@ def getCollectors(path): break except TypeError: continue - # print "Imported module: %s %s" % (modname, cls.__name__) + # print("Imported module: %s %s" % (modname, cls.__name__)) except Exception: - print "Failed to import module: %s. %s" % ( - modname, traceback.format_exc()) + print("Failed to import module: %s. %s" % ( + modname, traceback.format_exc())) collectors[modname] = False continue @@ -77,7 +78,7 @@ def getCollectors(path): def typeToString(key): - if isinstance(obj.config[key], basestring): + if isinstance(obj.config[key], str): user_val = obj.config[key] elif isinstance(obj.config[key], bool): user_val = str(obj.config[key]) @@ -94,7 +95,7 @@ def typeToString(key): def stringToType(key, val): if type(obj.config[key]) is type(val): config_file[key] = val - elif isinstance(obj.config[key], basestring): + elif isinstance(obj.config[key], str): if val.lower() == 'false': config_file[key] = False elif val.lower() == 'true': @@ -102,7 +103,7 @@ def stringToType(key, val): else: config_file[key] = val elif isinstance(obj.config[key], bool): - if isinstance(val, basestring): + if isinstance(val, str): config_file[key] = str_to_bool(val) else: config_file[key] = bool(val) @@ -116,7 +117,7 @@ def stringToType(key, val): def boolCheck(val): - if isinstance(val, basestring): + if isinstance(val, str): return str_to_bool(val) elif isinstance(val, bool): return val @@ -133,9 +134,9 @@ def configureKey(key): except NotImplementedError: return - print "\n" + print("\n") if key in default_conf_help: - print default_conf_help[key] + print(default_conf_help[key]) val = raw_input(key + ' [' + user_val + ']: ') # Empty user input? Default to current value @@ -177,18 +178,18 @@ if __name__ == "__main__": if os.path.exists(options.configfile): config = ConfigObj(os.path.abspath(options.configfile)) else: - print >> sys.stderr, "ERROR: Config file: %s does not exist." % ( - options.configfile) - print >> sys.stderr, ("Please run python config.py -c" - + " /path/to/diamond.conf") + print("ERROR: Config file: %s does not exist." % ( + options.configfile), file=sys.stderr) + print("Please run python config.py -c /path/to/diamond.conf", + file=sys.stderr) parser.print_help(sys.stderr) sys.exit(1) if not options.dump: - print '' - print 'I will be over writing files in' - print config['server']['collectors_config_path'] - print 'Please type yes to continue' + print('') + print('I will be over writing files in') + print(config['server']['collectors_config_path']) + print('Please type yes to continue') val = raw_input('Are you sure? ') if val != 'yes': @@ -225,7 +226,7 @@ if __name__ == "__main__": obj = cls(config=config, handlers={}) if options.dump: - print collector + " " + str(obj.config) + print(collector + " " + str(obj.config)) continue default_conf = obj.get_default_config() @@ -241,9 +242,9 @@ if __name__ == "__main__": config_keys['instance_prefix'] = False config_keys['interval'] = False - print "*" * 60 - print "\n\t\tNow configuring " + collector - print collectors[collector].__doc__ + print("*" * 60) + print("\n\t\tNow configuring " + collector) + print(collectors[collector].__doc__) print "(%s)" % collector configureKey('enabled') @@ -256,12 +257,12 @@ if __name__ == "__main__": config_file.write() - except IOError, (errno, strerror): - print "I/O error({0}): {1}".format(errno, strerror) + except IOError as err: + print("I/O error({0}): {1}".format(err.errno, err.strerror)) except KeyboardInterrupt: - print + print("") sys.exit() except: continue if not foundcollector: - print "Collector not found." + print("Collector not found.") diff --git a/build_doc.py b/build_doc.py index 10cc41443..7afb370e3 100755 --- a/build_doc.py +++ b/build_doc.py @@ -2,6 +2,7 @@ # coding=utf-8 ########################################################################## +from __future__ import print_function import configobj import optparse import os @@ -56,8 +57,8 @@ def getCollectors(path): if cls.__name__ not in collectors: collectors[cls.__name__] = module except Exception: - print "Failed to import module: %s. %s" % ( - modname, traceback.format_exc()) + print("Failed to import module: %s. %s" % ( + modname, traceback.format_exc())) collectors[modname] = False elif os.path.isdir(cPath): @@ -91,8 +92,8 @@ def getHandlers(path, name=None): if cls.__name__ not in handlers: handlers[cls.__name__] = module except Exception: - print "Failed to import module: %s. %s" % ( - modname, traceback.format_exc()) + print("Failed to import module: %s. %s" % ( + modname, traceback.format_exc())) handlers[modname] = False elif os.path.isdir(cPath): @@ -110,7 +111,7 @@ def writeDocString(docFile, name, doc): docFile.write("%s\n" % (name)) docFile.write("=====\n") if doc is None: - print "No __doc__ string for %s!" % name + print("No __doc__ string for %s!" % name) docFile.write("%s\n" % doc) @@ -150,7 +151,7 @@ def writeDoc(items, type_name, doc_path): if item.startswith('Test'): continue - print "Processing %s..." % (item) + print("Processing %s..." % (item)) if not hasattr(items[item], item): continue @@ -172,8 +173,8 @@ def writeDoc(items, type_name, doc_path): default_options = obj.get_default_config() if type_name is "Handler": os.remove(tmpfile[1]) - except Exception, e: - print "Caught Exception %s" % e + except Exception as e: + print("Caught Exception %s" % e) docFile = open(os.path.join(doc_path, item + ".md"), 'w') @@ -228,10 +229,10 @@ def writeDoc(items, type_name, doc_path): if os.path.exists(options.configfile): config = configobj.ConfigObj(os.path.abspath(options.configfile)) else: - print >> sys.stderr, "ERROR: Config file: %s does not exist." % ( - options.configfile) - print >> sys.stderr, ("Please run python config.py -c " + - "/path/to/diamond.conf") + print("ERROR: Config file: %s does not exist." % ( + options.configfile), file=sys.stderr) + print("Please run python config.py -c /path/to/diamond.conf", + file=sys.stderr) parser.print_help(sys.stderr) sys.exit(1) diff --git a/setup.py b/setup.py index cace4c5d1..5093d74e9 100755 --- a/setup.py +++ b/setup.py @@ -17,12 +17,8 @@ def running_under_virtualenv(): return False -if os.environ.get('USE_SETUPTOOLS'): - from setuptools import setup - setup_kwargs = dict(zip_safe=0) -else: - from distutils.core import setup - setup_kwargs = dict() +from setuptools import setup +setup_kwargs = dict(zip_safe=0) if os.name == 'nt': pgm_files = os.environ["ProgramFiles"] diff --git a/src/collectors/aerospike/test/testaerospike27.py b/src/collectors/aerospike/test/testaerospike27.py index 1ff6b2c60..342e9a649 100644 --- a/src/collectors/aerospike/test/testaerospike27.py +++ b/src/collectors/aerospike/test/testaerospike27.py @@ -5,8 +5,9 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock, call -from mock import patch +from mock import call +from test import Mock +from test import patch from diamond.collector import Collector diff --git a/src/collectors/aerospike/test/testaerospike39.py b/src/collectors/aerospike/test/testaerospike39.py index 1319b80eb..d75d40f7b 100644 --- a/src/collectors/aerospike/test/testaerospike39.py +++ b/src/collectors/aerospike/test/testaerospike39.py @@ -5,8 +5,9 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock, call -from mock import patch +from mock import call +from test import Mock +from test import patch from diamond.collector import Collector diff --git a/src/collectors/amavis/amavis.py b/src/collectors/amavis/amavis.py index 8a245a328..aa69b9284 100644 --- a/src/collectors/amavis/amavis.py +++ b/src/collectors/amavis/amavis.py @@ -74,6 +74,8 @@ def collect(self): cmdline = [self.config['amavisd_exe'], '-c', '1'] agent = subprocess.Popen(cmdline, stdout=subprocess.PIPE) agent_out = agent.communicate()[0] + if isinstance(agent_out, bytes): + agent_out = agent_out.decode("utf8") lines = agent_out.strip().split(os.linesep) for line in lines: for rex in self.matchers: diff --git a/src/collectors/amavis/test/testamavis.py b/src/collectors/amavis/test/testamavis.py index cf40ddc02..38829b163 100644 --- a/src/collectors/amavis/test/testamavis.py +++ b/src/collectors/amavis/test/testamavis.py @@ -7,7 +7,7 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import patch +from test import patch import amavis from diamond.collector import Collector diff --git a/src/collectors/apcupsd/test/testapcupsd.py b/src/collectors/apcupsd/test/testapcupsd.py index ff3d28367..d74e5406c 100644 --- a/src/collectors/apcupsd/test/testapcupsd.py +++ b/src/collectors/apcupsd/test/testapcupsd.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from apcupsd import ApcupsdCollector diff --git a/src/collectors/aurora/aurora.py b/src/collectors/aurora/aurora.py index 809c6f335..ed01e196a 100644 --- a/src/collectors/aurora/aurora.py +++ b/src/collectors/aurora/aurora.py @@ -1,6 +1,5 @@ import diamond.collector - -import urllib2 +import diamond.pycompat class AuroraCollector(diamond.collector.Collector): @@ -32,7 +31,7 @@ def collect(self): self.config['host'], self.config['port']) - response = urllib2.urlopen(url) + response = diamond.pycompat.urlopen(url) for line in response.readlines(): properties = line.split() diff --git a/src/collectors/aurora/test/testaurora.py b/src/collectors/aurora/test/testaurora.py index 5b4fe017a..c867ef4ef 100644 --- a/src/collectors/aurora/test/testaurora.py +++ b/src/collectors/aurora/test/testaurora.py @@ -5,11 +5,11 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector - +from diamond.pycompat import URLOPEN from aurora import AuroraCollector ############################################################################### @@ -31,7 +31,7 @@ def se(url): if url == 'http://localhost:8081/vars': return self.getFixture('metrics') - patch_urlopen = patch('urllib2.urlopen', Mock(side_effect=se)) + patch_urlopen = patch(URLOPEN, Mock(side_effect=se)) patch_urlopen.start() self.collector.collect() @@ -46,7 +46,7 @@ def se(url): @patch.object(Collector, 'publish') def test_should_fail_gracefully(self, publish_mock): - patch_urlopen = patch('urllib2.urlopen', Mock( + patch_urlopen = patch(URLOPEN, Mock( return_value=self.getFixture('metrics_blank'))) patch_urlopen.start() diff --git a/src/collectors/beanstalkd/beanstalkd.py b/src/collectors/beanstalkd/beanstalkd.py index 27ba7c199..b1156e71e 100644 --- a/src/collectors/beanstalkd/beanstalkd.py +++ b/src/collectors/beanstalkd/beanstalkd.py @@ -51,7 +51,7 @@ def _get_stats(self): try: connection = beanstalkc.Connection(self.config['host'], int(self.config['port'])) - except beanstalkc.BeanstalkcException, e: + except beanstalkc.BeanstalkcException as e: self.log.error("Couldn't connect to beanstalkd: %s", e) return {} diff --git a/src/collectors/beanstalkd/test/testbeanstalkd.py b/src/collectors/beanstalkd/test/testbeanstalkd.py index a6122ff3e..c8d80e212 100644 --- a/src/collectors/beanstalkd/test/testbeanstalkd.py +++ b/src/collectors/beanstalkd/test/testbeanstalkd.py @@ -6,8 +6,8 @@ from test import get_collector_config from test import unittest from test import run_only -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from beanstalkd import BeanstalkdCollector diff --git a/src/collectors/bind/bind.py b/src/collectors/bind/bind.py index fa033ec35..c390e5312 100644 --- a/src/collectors/bind/bind.py +++ b/src/collectors/bind/bind.py @@ -11,8 +11,8 @@ """ import diamond.collector +import diamond.pycompat import sys -import urllib2 if sys.version_info >= (2, 5): import xml.etree.cElementTree as ElementTree @@ -74,9 +74,9 @@ def clean_counter(self, name, value): def collect(self): try: - req = urllib2.urlopen('http://%s:%d/' % ( + req = diamond.pycompat.urlopen('http://%s:%d/' % ( self.config['host'], int(self.config['port']))) - except Exception, e: + except Exception as e: self.log.error('Couldnt connect to bind: %s', e) return {} diff --git a/src/collectors/bind/test/testbind.py b/src/collectors/bind/test/testbind.py index 10f31714d..64a922ec9 100644 --- a/src/collectors/bind/test/testbind.py +++ b/src/collectors/bind/test/testbind.py @@ -5,10 +5,11 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector +from diamond.pycompat import URLOPEN from bind import BindCollector ########################################################################## @@ -28,7 +29,7 @@ def test_import(self): @patch.object(Collector, 'publish') def test_should_work_with_real_data(self, publish_mock): - patch_urlopen = patch('urllib2.urlopen', Mock( + patch_urlopen = patch(URLOPEN, Mock( return_value=self.getFixture('bind.xml'))) patch_urlopen.start() diff --git a/src/collectors/celerymon/celerymon.py b/src/collectors/celerymon/celerymon.py index 05ba0030d..d8f325832 100644 --- a/src/collectors/celerymon/celerymon.py +++ b/src/collectors/celerymon/celerymon.py @@ -18,7 +18,7 @@ """ import diamond.collector -import urllib2 +import diamond.pycompat import time try: @@ -68,7 +68,7 @@ def collect(self): celerymon_url = "http://%s:%s/api/task/?since=%i" % ( host, port, self.LastCollectTime) - response = urllib2.urlopen(celerymon_url) + response = diamond.pycompat.urlopen(celerymon_url) body = response.read() celery_data = json.loads(body) diff --git a/src/collectors/ceph/ceph.py b/src/collectors/ceph/ceph.py index 4aea17e32..127808583 100644 --- a/src/collectors/ceph/ceph.py +++ b/src/collectors/ceph/ceph.py @@ -108,7 +108,9 @@ def _get_stats_from_socket(self, name): 'perf', 'dump', ]) - except subprocess.CalledProcessError, err: + if isinstance(json_blob, bytes): + json_blob = json_blob.decode("utf8") + except subprocess.CalledProcessError as err: self.log.info('Could not get stats from %s: %s', name, err) self.log.exception('Could not get stats from %s' % name) @@ -116,7 +118,7 @@ def _get_stats_from_socket(self, name): try: json_data = json.loads(json_blob) - except Exception, err: + except Exception as err: self.log.info('Could not parse stats from %s: %s', name, err) self.log.exception('Could not parse stats from %s' % name) diff --git a/src/collectors/ceph/test/testceph.py b/src/collectors/ceph/test/testceph.py index 8095f7e17..3c611ecef 100644 --- a/src/collectors/ceph/test/testceph.py +++ b/src/collectors/ceph/test/testceph.py @@ -12,7 +12,8 @@ from test import get_collector_config from test import unittest from test import run_only -from mock import patch, call +from test import patch +from mock import call from diamond.collector import Collector import ceph diff --git a/src/collectors/cephstats/cephstats.py b/src/collectors/cephstats/cephstats.py index 2f54c9b81..bb1c82684 100644 --- a/src/collectors/cephstats/cephstats.py +++ b/src/collectors/cephstats/cephstats.py @@ -68,7 +68,9 @@ def _get_stats(self): """ try: output = subprocess.check_output(['ceph', '-s']) - except subprocess.CalledProcessError, err: + if isinstance(output, bytes): + output = output.decode("utf8") + except subprocess.CalledProcessError as err: self.log.info( 'Could not get stats: %s' % err) self.log.exception('Could not get stats') diff --git a/src/collectors/chronyd/chronyd.py b/src/collectors/chronyd/chronyd.py index 3b0b7939e..fb7c7154c 100644 --- a/src/collectors/chronyd/chronyd.py +++ b/src/collectors/chronyd/chronyd.py @@ -66,6 +66,8 @@ def get_output(self): def collect(self): output = self.get_output() + if isinstance(output, bytes): + output = output.decode("utf8") for line in output.strip().split("\n"): m = LINE_PATTERN.search(line) @@ -78,7 +80,7 @@ def collect(self): try: value = diamond.convertor.time.convert(offset, unit, 'ms') - except NotImplementedError, e: + except NotImplementedError as e: self.log.error('Unable to convert %s%s: %s', offset, unit, e) continue diff --git a/src/collectors/chronyd/test/testchronyd.py b/src/collectors/chronyd/test/testchronyd.py index e819d3162..1fc47c086 100644 --- a/src/collectors/chronyd/test/testchronyd.py +++ b/src/collectors/chronyd/test/testchronyd.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from chronyd import ChronydCollector diff --git a/src/collectors/conntrack/conntrack.py b/src/collectors/conntrack/conntrack.py index c335545d1..5802e6a5a 100644 --- a/src/collectors/conntrack/conntrack.py +++ b/src/collectors/conntrack/conntrack.py @@ -50,12 +50,12 @@ def collect(self): collected = {} files = [] - if isinstance(self.config['dir'], basestring): + if isinstance(self.config['dir'], str): dirs = [d.strip() for d in self.config['dir'].split(',')] elif isinstance(self.config['dir'], list): dirs = self.config['dir'] - if isinstance(self.config['files'], basestring): + if isinstance(self.config['files'], str): files = [f.strip() for f in self.config['files'].split(',')] elif isinstance(self.config['files'], list): files = self.config['files'] diff --git a/src/collectors/conntrack/test/testconntrack.py b/src/collectors/conntrack/test/testconntrack.py index c98b362f4..e15eb5bf6 100644 --- a/src/collectors/conntrack/test/testconntrack.py +++ b/src/collectors/conntrack/test/testconntrack.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from conntrack import ConnTrackCollector diff --git a/src/collectors/cpu/cpu.py b/src/collectors/cpu/cpu.py index fdb93f9ce..6ada7855b 100644 --- a/src/collectors/cpu/cpu.py +++ b/src/collectors/cpu/cpu.py @@ -157,12 +157,12 @@ def cpu_delta_time(interval): ncpus > 0)): metrics[metric_name] = self.derivative( metric_name, - long(stats[s]), + int(stats[s]), self.MAX_VALUES[s]) / ncpus else: metrics[metric_name] = self.derivative( metric_name, - long(stats[s]), + int(stats[s]), self.MAX_VALUES[s]) # Check for a bug in xen where the idle time is doubled for guest diff --git a/src/collectors/cpu/test/testcpu.py b/src/collectors/cpu/test/testcpu.py index 44336e20b..7ca83ce3d 100644 --- a/src/collectors/cpu/test/testcpu.py +++ b/src/collectors/cpu/test/testcpu.py @@ -5,13 +5,10 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from test import Mock +from test import patch +from test import StringIO +from test import BUILTIN_OPEN from diamond.collector import Collector from cpu import CPUCollector @@ -32,7 +29,7 @@ def setUp(self): def test_import(self): self.assertTrue(CPUCollector) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch('os.access', Mock(return_value=True)) @patch.object(Collector, 'publish') def test_should_open_proc_stat(self, publish_mock, open_mock): @@ -43,7 +40,7 @@ def test_should_open_proc_stat(self, publish_mock, open_mock): @patch.object(Collector, 'publish') def test_should_work_with_synthetic_data(self, publish_mock): - patch_open = patch('__builtin__.open', Mock(return_value=StringIO( + patch_open = patch(BUILTIN_OPEN, Mock(return_value=StringIO( 'cpu 100 200 300 400 500 0 0 0 0 0'))) patch_open.start() @@ -52,7 +49,7 @@ def test_should_work_with_synthetic_data(self, publish_mock): self.assertPublishedMany(publish_mock, {}) - patch_open = patch('__builtin__.open', Mock(return_value=StringIO( + patch_open = patch(BUILTIN_OPEN, Mock(return_value=StringIO( 'cpu 110 220 330 440 550 0 0 0 0 0'))) patch_open.start() @@ -204,7 +201,7 @@ def input_dict_to_proc_string(self, cpu_id, dict_): @patch.object(Collector, 'publish') def test_should_work_proc_stat(self, publish_mock): - patch_open = patch('__builtin__.open', Mock(return_value=StringIO( + patch_open = patch(BUILTIN_OPEN, Mock(return_value=StringIO( "\n".join([self.input_dict_to_proc_string('', self.input_base), self.input_dict_to_proc_string('0', self.input_base), self.input_dict_to_proc_string('1', self.input_base), @@ -217,7 +214,7 @@ def test_should_work_proc_stat(self, publish_mock): self.assertPublishedMany(publish_mock, {}) - patch_open = patch('__builtin__.open', Mock(return_value=StringIO( + patch_open = patch(BUILTIN_OPEN, Mock(return_value=StringIO( "\n".join([self.input_dict_to_proc_string('', self.input_next), self.input_dict_to_proc_string('0', self.input_next), self.input_dict_to_proc_string('1', self.input_next), diff --git a/src/collectors/cpuacct_cgroup/cpuacct_cgroup.py b/src/collectors/cpuacct_cgroup/cpuacct_cgroup.py index ed3ec2675..f15d02e73 100644 --- a/src/collectors/cpuacct_cgroup/cpuacct_cgroup.py +++ b/src/collectors/cpuacct_cgroup/cpuacct_cgroup.py @@ -62,8 +62,8 @@ def collect(self): stat_file.close() # create metrics from collected utimes and stimes for cgroups - for parent, cpuacct in results.iteritems(): - for key, value in cpuacct.iteritems(): + for parent, cpuacct in results.items(): + for key, value in cpuacct.items(): metric_name = '.'.join([parent, key]) self.publish(metric_name, value, metric_type='GAUGE') return True diff --git a/src/collectors/cpuacct_cgroup/test/testcpuacct_cgroup.py b/src/collectors/cpuacct_cgroup/test/testcpuacct_cgroup.py index 4e2430262..56a3fb047 100644 --- a/src/collectors/cpuacct_cgroup/test/testcpuacct_cgroup.py +++ b/src/collectors/cpuacct_cgroup/test/testcpuacct_cgroup.py @@ -5,12 +5,9 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import patch - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from test import patch +from test import StringIO +from test import BUILTIN_OPEN from diamond.collector import Collector from cpuacct_cgroup import CpuAcctCgroupCollector @@ -28,7 +25,7 @@ def setUp(self): def test_import(self): self.assertTrue(CpuAcctCgroupCollector) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch.object(Collector, 'publish') def test_should_open_all_cpuacct_stat(self, publish_mock, open_mock): return diff --git a/src/collectors/darner/darner.py b/src/collectors/darner/darner.py index 33f69b10f..d40d9aa0b 100644 --- a/src/collectors/darner/darner.py +++ b/src/collectors/darner/darner.py @@ -117,7 +117,7 @@ def collect(self): hosts = self.config.get('hosts') # Convert a string config value to be an array - if isinstance(hosts, basestring): + if isinstance(hosts, str): hosts = [hosts] for host in hosts: diff --git a/src/collectors/darner/test/testdarner.py b/src/collectors/darner/test/testdarner.py index e301f3681..bac2ba097 100644 --- a/src/collectors/darner/test/testdarner.py +++ b/src/collectors/darner/test/testdarner.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from darner import DarnerCollector diff --git a/src/collectors/diskspace/diskspace.py b/src/collectors/diskspace/diskspace.py index a79526b46..adff2f99d 100644 --- a/src/collectors/diskspace/diskspace.py +++ b/src/collectors/diskspace/diskspace.py @@ -78,7 +78,7 @@ def process_config(self): super(DiskSpaceCollector, self).process_config() # Precompile things self.exclude_filters = self.config['exclude_filters'] - if isinstance(self.exclude_filters, basestring): + if isinstance(self.exclude_filters, str): self.exclude_filters = [self.exclude_filters] if not self.exclude_filters: @@ -87,7 +87,7 @@ def process_config(self): self.exclude_reg = re.compile('|'.join(self.exclude_filters)) self.filesystems = [] - if isinstance(self.config['filesystems'], basestring): + if isinstance(self.config['filesystems'], str): for filesystem in self.config['filesystems'].split(','): self.filesystems.append(filesystem.strip()) elif isinstance(self.config['filesystems'], list): @@ -187,7 +187,7 @@ def collect(self): self.log.error('No diskspace metrics retrieved') return None - for info in results.itervalues(): + for key, info in results.items(): if info['device'] in labels: name = labels[info['device']] else: @@ -201,7 +201,7 @@ def collect(self): if hasattr(os, 'statvfs'): # POSIX try: data = os.statvfs(info['mount_point']) - except OSError, e: + except OSError as e: self.log.exception(e) continue diff --git a/src/collectors/diskspace/test/testdiskspace.py b/src/collectors/diskspace/test/testdiskspace.py index 522f47ad3..5fbf76537 100644 --- a/src/collectors/diskspace/test/testdiskspace.py +++ b/src/collectors/diskspace/test/testdiskspace.py @@ -6,8 +6,9 @@ from test import get_collector_config from test import unittest from test import run_only -from mock import Mock -from mock import patch +from test import Mock +from test import patch +from test import BUILTIN_OPEN from diamond.collector import Collector from diskspace import DiskSpaceCollector @@ -34,7 +35,7 @@ def test_import(self): def run_collection(self, statvfs_mock, os_major, os_minor): os_stat_mock = patch('os.stat') os_path_isdir_mock = patch('os.path.isdir', Mock(return_value=False)) - open_mock = patch('__builtin__.open', + open_mock = patch(BUILTIN_OPEN, Mock(return_value=self.getFixture('proc_mounts'))) os_statvfs_mock = patch('os.statvfs', Mock(return_value=statvfs_mock)) @@ -54,7 +55,7 @@ def test_get_file_systems(self): os_stat_mock = patch('os.stat') os_realpath_mock = patch('os.path.realpath') - open_mock = patch('__builtin__.open', + open_mock = patch(BUILTIN_OPEN, Mock(return_value=self.getFixture('proc_mounts'))) stat_mock = os_stat_mock.start() diff --git a/src/collectors/disktemp/disktemp.py b/src/collectors/disktemp/disktemp.py index ff900bda0..f29d68918 100644 --- a/src/collectors/disktemp/disktemp.py +++ b/src/collectors/disktemp/disktemp.py @@ -90,6 +90,8 @@ def collect(self): metrics = {} for device, p in instances.items(): output = p.communicate()[0].strip() + if isinstance(output, bytes): + output = output.decode("utf8") try: metrics[device + ".Temperature"] = float(output) diff --git a/src/collectors/disktemp/test/testdisktemp.py b/src/collectors/disktemp/test/testdisktemp.py index 4616218eb..2b7c85a0c 100644 --- a/src/collectors/disktemp/test/testdisktemp.py +++ b/src/collectors/disktemp/test/testdisktemp.py @@ -4,8 +4,8 @@ from test import CollectorTestCase from test import get_collector_config -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from disktemp import DiskTemperatureCollector diff --git a/src/collectors/diskusage/diskusage.py b/src/collectors/diskusage/diskusage.py index 296209c46..015e02094 100644 --- a/src/collectors/diskusage/diskusage.py +++ b/src/collectors/diskusage/diskusage.py @@ -163,13 +163,13 @@ def collect(self): self.log.error('No diskspace metrics retrieved') return None - for key, info in results.iteritems(): + for key, info in results.items(): metrics = {} name = info['device'] if not reg.match(name): continue - for key, value in info.iteritems(): + for key, value in info.items(): if key == 'device': continue oldkey = key diff --git a/src/collectors/diskusage/test/testdiskusage.py b/src/collectors/diskusage/test/testdiskusage.py index ff5030048..65a268dc7 100644 --- a/src/collectors/diskusage/test/testdiskusage.py +++ b/src/collectors/diskusage/test/testdiskusage.py @@ -5,8 +5,9 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch +from test import BUILTIN_OPEN from diamond.collector import Collector from diskusage import DiskUsageCollector @@ -35,7 +36,7 @@ def test_import(self): def test_get_disk_statistics(self): patch_open = patch( - '__builtin__.open', + BUILTIN_OPEN, Mock(return_value=self.getFixture('diskstats'))) open_mock = patch_open.start() @@ -56,7 +57,7 @@ def test_get_disk_statistics(self): def test_should_work_with_real_data(self, publish_mock): patch_open = patch( - '__builtin__.open', + BUILTIN_OPEN, Mock( return_value=self.getFixture('proc_diskstats_1'))) patch_time = patch('time.time', Mock(return_value=10)) @@ -70,7 +71,7 @@ def test_should_work_with_real_data(self, publish_mock): self.assertPublishedMany(publish_mock, {}) patch_open = patch( - '__builtin__.open', + BUILTIN_OPEN, Mock( return_value=self.getFixture('proc_diskstats_2'))) patch_time = patch('time.time', Mock(return_value=20)) @@ -92,7 +93,7 @@ def test_should_work_with_real_data(self, publish_mock): @patch.object(Collector, 'publish') def test_verify_supporting_vda_and_xvdb(self, publish_mock): patch_open = patch( - '__builtin__.open', + BUILTIN_OPEN, Mock( return_value=self.getFixture( 'proc_diskstats_1_vda_xvdb'))) @@ -107,7 +108,7 @@ def test_verify_supporting_vda_and_xvdb(self, publish_mock): self.assertPublishedMany(publish_mock, {}) patch_open = patch( - '__builtin__.open', + BUILTIN_OPEN, Mock( return_value=self.getFixture( 'proc_diskstats_2_vda_xvdb'))) @@ -128,7 +129,7 @@ def test_verify_supporting_vda_and_xvdb(self, publish_mock): @patch.object(Collector, 'publish') def test_verify_supporting_md_dm(self, publish_mock): patch_open = patch( - '__builtin__.open', + BUILTIN_OPEN, Mock( return_value=self.getFixture( 'proc_diskstats_1_md_dm'))) @@ -143,7 +144,7 @@ def test_verify_supporting_md_dm(self, publish_mock): self.assertPublishedMany(publish_mock, {}) patch_open = patch( - '__builtin__.open', + BUILTIN_OPEN, Mock( return_value=self.getFixture( 'proc_diskstats_2_md_dm'))) @@ -163,7 +164,7 @@ def test_verify_supporting_md_dm(self, publish_mock): @patch.object(Collector, 'publish') def test_verify_supporting_disk(self, publish_mock): patch_open = patch( - '__builtin__.open', + BUILTIN_OPEN, Mock( return_value=self.getFixture( 'proc_diskstats_1_disk'))) @@ -178,7 +179,7 @@ def test_verify_supporting_disk(self, publish_mock): self.assertPublishedMany(publish_mock, {}) patch_open = patch( - '__builtin__.open', + BUILTIN_OPEN, Mock( return_value=self.getFixture( 'proc_diskstats_2_disk'))) @@ -197,7 +198,7 @@ def test_verify_supporting_disk(self, publish_mock): @patch.object(Collector, 'publish') def test_service_Time(self, publish_mock): patch_open = patch( - '__builtin__.open', + BUILTIN_OPEN, Mock( return_value=self.getFixture( 'proc_diskstats_1_service_time'))) @@ -212,7 +213,7 @@ def test_service_Time(self, publish_mock): self.assertPublishedMany(publish_mock, {}) patch_open = patch( - '__builtin__.open', + BUILTIN_OPEN, Mock( return_value=self.getFixture( 'proc_diskstats_2_service_time'))) diff --git a/src/collectors/drbd/drbd.py b/src/collectors/drbd/drbd.py index 103f20f10..c7ade8dfa 100644 --- a/src/collectors/drbd/drbd.py +++ b/src/collectors/drbd/drbd.py @@ -75,7 +75,7 @@ def collect(self): else: continue statusfile.close() - except IOError, errormsg: + except IOError as errormsg: self.log.error("Can't read DRBD status file: {0}".format(errormsg)) return diff --git a/src/collectors/dropwizard/dropwizard.py b/src/collectors/dropwizard/dropwizard.py index b116bdc2d..1020393e5 100644 --- a/src/collectors/dropwizard/dropwizard.py +++ b/src/collectors/dropwizard/dropwizard.py @@ -4,15 +4,13 @@ Collect [dropwizard](http://dropwizard.codahale.com/) stats for the local node """ - -import urllib2 - try: import json except ImportError: import simplejson as json - import diamond.collector +import diamond.pycompat +from diamond.pycompat import HTTPError class DropwizardCollector(diamond.collector.Collector): @@ -45,8 +43,8 @@ def collect(self): url = 'http://%s:%i/metrics' % ( self.config['host'], int(self.config['port'])) try: - response = urllib2.urlopen(url) - except urllib2.HTTPError, err: + response = diamond.pycompat.urlopen(url) + except HTTPError as err: self.log.error("%s: %s", url, err) return diff --git a/src/collectors/dropwizard/test/testdropwizard.py b/src/collectors/dropwizard/test/testdropwizard.py index d143b2821..433429ba4 100644 --- a/src/collectors/dropwizard/test/testdropwizard.py +++ b/src/collectors/dropwizard/test/testdropwizard.py @@ -5,10 +5,11 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector +from diamond.pycompat import URLOPEN from dropwizard import DropwizardCollector @@ -27,7 +28,7 @@ def test_import(self): @patch.object(Collector, 'publish') def test_should_work_with_real_data(self, publish_mock): - patch_urlopen = patch('urllib2.urlopen', + patch_urlopen = patch(URLOPEN, Mock(return_value=self.getFixture('stats'))) patch_urlopen.start() @@ -69,7 +70,7 @@ def test_should_work_with_real_data(self, publish_mock): @patch.object(Collector, 'publish') def test_should_fail_gracefully(self, publish_mock): patch_urlopen = patch( - 'urllib2.urlopen', + URLOPEN, Mock( return_value=self.getFixture('stats_blank'))) diff --git a/src/collectors/dseopscenter/dseopscenter.py b/src/collectors/dseopscenter/dseopscenter.py index dce08f1ab..64190c874 100644 --- a/src/collectors/dseopscenter/dseopscenter.py +++ b/src/collectors/dseopscenter/dseopscenter.py @@ -3,8 +3,6 @@ """ Collect the DataStax OpsCenter metrics """ - -import urllib2 import datetime try: @@ -13,6 +11,7 @@ import simplejson as json import diamond.collector +import diamond.pycompat class DseOpsCenterCollector(diamond.collector.Collector): @@ -119,8 +118,8 @@ def _get_schema(self): int(self.config['port']), self.config['cluster_id']) try: - response = urllib2.urlopen(url) - except Exception, err: + response = diamond.pycompat.urlopen(url) + except Exception as err: self.log.error('%s: %s', url, err) return False @@ -157,8 +156,8 @@ def _get(self, start, end, step=60): self.config['default_tail_opts']) try: - response = urllib2.urlopen(url) - except Exception, err: + response = diamond.pycompat.urlopen(url) + except Exception as err: self.log.error('%s: %s', url, err) return False diff --git a/src/collectors/dseopscenter/test/testdseopscenter.py b/src/collectors/dseopscenter/test/testdseopscenter.py index c9f95d351..77a777dec 100644 --- a/src/collectors/dseopscenter/test/testdseopscenter.py +++ b/src/collectors/dseopscenter/test/testdseopscenter.py @@ -4,10 +4,11 @@ from test import CollectorTestCase from test import get_collector_config -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector +from diamond.pycompat import URLOPEN from dseopscenter import DseOpsCenterCollector @@ -27,12 +28,12 @@ def test_import(self): @patch.object(Collector, 'publish') def test_should_work_with_real_data(self, publish_mock): - urlopen_mock1 = patch('urllib2.urlopen', Mock( + urlopen_mock1 = patch(URLOPEN, Mock( side_effect=lambda *args: self.getFixture('keyspaces.json'))) urlopen_mock1.start() self.collector._get_schema() urlopen_mock1.stop() - urlopen_mock2 = patch('urllib2.urlopen', Mock( + urlopen_mock2 = patch(URLOPEN, Mock( side_effect=lambda *args: self.getFixture('new-metrics.json'))) urlopen_mock2.start() self.collector.collect() diff --git a/src/collectors/elasticsearch/elasticsearch.py b/src/collectors/elasticsearch/elasticsearch.py index 534241b36..fa2ec59e6 100755 --- a/src/collectors/elasticsearch/elasticsearch.py +++ b/src/collectors/elasticsearch/elasticsearch.py @@ -6,17 +6,12 @@ Supports multiple instances. When using the 'instances' parameter the instance alias will be appended to the 'path' parameter. - -#### Dependencies - - * urlib2 - """ -import urllib2 import base64 import re from diamond.collector import str_to_bool +import diamond.pycompat try: import json @@ -33,7 +28,7 @@ class ElasticSearchCollector(diamond.collector.Collector): def process_config(self): super(ElasticSearchCollector, self).process_config() instance_list = self.config['instances'] - if isinstance(instance_list, basestring): + if isinstance(instance_list, str): instance_list = [instance_list] if len(instance_list) == 0: @@ -111,13 +106,13 @@ def _get(self, scheme, host, port, path, assert_key=None): """ url = '%s://%s:%i/%s' % (scheme, host, port, path) try: - request = urllib2.Request(url) + request = diamond.pycompat.Request(url) if self.config['user'] and self.config['password']: base64string = base64.standard_b64encode( '%s:%s' % (self.config['user'], self.config['password'])) request.add_header("Authorization", "Basic %s" % base64string) - response = urllib2.urlopen(request) - except Exception, err: + response = diamond.pycompat.urlopen(request) + except Exception as err: self.log.error("%s: %s" % (url, err)) return False @@ -135,13 +130,13 @@ def _get(self, scheme, host, port, path, assert_key=None): return doc def _copy_one_level(self, metrics, prefix, data, filter=lambda key: True): - for key, value in data.iteritems(): + for key, value in data.items(): if filter(key): metric_path = '%s.%s' % (prefix, key) self._set_or_sum_metric(metrics, metric_path, value) def _copy_two_level(self, metrics, prefix, data, filter=lambda key: True): - for key1, d1 in data.iteritems(): + for key1, d1 in data.items(): self._copy_one_level(metrics, '%s.%s' % (prefix, key1), d1, filter) def _index_metrics(self, metrics, prefix, index): @@ -236,7 +231,7 @@ def collect_instance_index_stats(self, scheme, host, port, metrics): else: return - for name, index in indices.iteritems(): + for name, index in indices.items(): self._index_metrics(metrics, 'indices.%s' % name, index['primaries']) @@ -246,7 +241,7 @@ def collect_instance(self, alias, scheme, host, port): return metrics = {} - node = result['nodes'].keys()[0] + node = list(result['nodes'].keys())[0] data = result['nodes'][node] # @@ -367,7 +362,7 @@ def collect_instance(self, alias, scheme, host, port): if 'heap_used_percent' in mem: metrics['jvm.mem.heap_used_percent'] = mem['heap_used_percent'] - for pool, d in mem['pools'].iteritems(): + for pool, d in mem['pools'].items(): pool = pool.replace(' ', '_') metrics['jvm.mem.pools.%s.used' % pool] = d['used_in_bytes'] metrics['jvm.mem.pools.%s.max' % pool] = d['max_in_bytes'] @@ -377,7 +372,7 @@ def collect_instance(self, alias, scheme, host, port): gc = jvm['gc'] collection_count = 0 collection_time_in_millis = 0 - for collector, d in gc['collectors'].iteritems(): + for collector, d in gc['collectors'].items(): metrics['jvm.gc.collection.%s.count' % collector] = d[ 'collection_count'] collection_count += d['collection_count'] diff --git a/src/collectors/elasticsearch/test/testelasticsearch.py b/src/collectors/elasticsearch/test/testelasticsearch.py index 814114d19..3bb111fd0 100644 --- a/src/collectors/elasticsearch/test/testelasticsearch.py +++ b/src/collectors/elasticsearch/test/testelasticsearch.py @@ -5,10 +5,11 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector +from diamond.pycompat import URLOPEN from elasticsearch import ElasticSearchCollector @@ -63,7 +64,7 @@ def test_should_work_with_real_data(self, publish_mock): self.getFixture('cluster_stats'), self.getFixture('indices_stats'), ] - urlopen_mock = patch('urllib2.urlopen', Mock( + urlopen_mock = patch(URLOPEN, Mock( side_effect=lambda *args: returns.pop(0))) self.collector.config['cluster'] = True @@ -131,7 +132,7 @@ def test_should_work_with_real_data_v2(self, publish_mock): self.getFixture('cluster_stats_v2'), self.getFixture('indices_stats'), ] - urlopen_mock = patch('urllib2.urlopen', Mock( + urlopen_mock = patch(URLOPEN, Mock( side_effect=lambda *args: returns.pop(0))) self.collector.config['cluster'] = True @@ -199,7 +200,7 @@ def test_should_work_with_real_data_logstash_mode(self, publish_mock): self.getFixture('stats'), self.getFixture('logstash_indices_stats'), ] - urlopen_mock = patch('urllib2.urlopen', Mock( + urlopen_mock = patch(URLOPEN, Mock( side_effect=lambda *args: returns.pop(0))) self.collector.config['logstash_mode'] = True @@ -271,7 +272,7 @@ def test_should_work_with_real_data_logstash_hourlymode(self, publish_mock): self.getFixture('stats'), self.getFixture('logstash_hourly_indices_stats'), ] - urlopen_mock = patch('urllib2.urlopen', Mock( + urlopen_mock = patch(URLOPEN, Mock( side_effect=lambda *args: returns.pop(0))) self.collector.config['logstash_mode'] = True @@ -343,7 +344,7 @@ def test_should_work_with_real_0_90_data(self, publish_mock): self.getFixture('stats0.90'), self.getFixture('indices_stats'), ] - urlopen_mock = patch('urllib2.urlopen', Mock( + urlopen_mock = patch(URLOPEN, Mock( side_effect=lambda *args: returns.pop(0))) urlopen_mock.start() @@ -369,7 +370,7 @@ def test_should_work_with_real_0_90_data(self, publish_mock): @patch.object(Collector, 'publish') def test_should_fail_gracefully(self, publish_mock): - urlopen_mock = patch('urllib2.urlopen', Mock( + urlopen_mock = patch(URLOPEN, Mock( return_value=self.getFixture('stats_blank'))) urlopen_mock.start() @@ -394,7 +395,7 @@ def test_multi_instances_with_real_data(self, publish_mock): self.getFixture('stats2'), self.getFixture('indices_stats2'), ] - urlopen_mock = patch('urllib2.urlopen', Mock( + urlopen_mock = patch(URLOPEN, Mock( side_effect=lambda *args: returns.pop(0))) urlopen_mock.start() @@ -429,7 +430,7 @@ def test_should_work_with_real_1_7_data(self, publish_mock): self.getFixture('stats1.7'), self.getFixture('indices_stats'), ] - urlopen_mock = patch('urllib2.urlopen', Mock( + urlopen_mock = patch(URLOPEN, Mock( side_effect=lambda *args: returns.pop(0))) urlopen_mock.start() diff --git a/src/collectors/elb/elb.py b/src/collectors/elb/elb.py index 61bb0a7cf..e4bac6820 100644 --- a/src/collectors/elb/elb.py +++ b/src/collectors/elb/elb.py @@ -39,7 +39,11 @@ """ import calendar -import cPickle +try: + import cPickle as pickle +except ImportError: + import pickle + import datetime import functools import re @@ -76,7 +80,7 @@ def __init__(self, func): def __call__(self, *args, **kwargs): # If the function args cannot be used as a cache hash key, fail fast - key = cPickle.dumps((args, kwargs)) + key = pickle.dumps((args, kwargs)) try: return self.cache[key] except KeyError: diff --git a/src/collectors/elb/test/testelb.py b/src/collectors/elb/test/testelb.py index 19a293768..aba09a220 100644 --- a/src/collectors/elb/test/testelb.py +++ b/src/collectors/elb/test/testelb.py @@ -2,14 +2,13 @@ # coding=utf-8 import datetime -import mock from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import patch +from test import patch from test import run_only -from mock import Mock +from test import Mock from diamond.collector import Collector from elb import ElbCollector @@ -99,8 +98,8 @@ def test_ignore(self, publish_metric, elb_connect_to_region, collector = ElbCollector(config, handlers=[]) target = ts + datetime.timedelta(minutes=1) - with mock.patch.object(datetime, 'datetime', - mock.Mock(wraps=datetime.datetime)) as patched: + with patch.object(datetime, 'datetime', + Mock(wraps=datetime.datetime)) as patched: patched.utcnow.return_value = target collector.collect() @@ -175,8 +174,8 @@ def test_collect(self, publish_metric, connect_to_region, cloudwatch): collector = ElbCollector(config, handlers=[]) target = ts + datetime.timedelta(minutes=1) - with mock.patch.object(datetime, 'datetime', - mock.Mock(wraps=datetime.datetime)) as patched: + with patch.object(datetime, 'datetime', + Mock(wraps=datetime.datetime)) as patched: patched.utcnow.return_value = target collector.collect() @@ -203,7 +202,7 @@ def assertRaisesAndContains(excClass, contains_str, callableObj, *args, **kwargs): try: callableObj(*args, **kwargs) - except excClass, e: + except excClass as e: msg = str(e) if contains_str in msg: return diff --git a/src/collectors/endecadgraph/endecadgraph.py b/src/collectors/endecadgraph/endecadgraph.py index 3ee022cdc..4da6a12a3 100644 --- a/src/collectors/endecadgraph/endecadgraph.py +++ b/src/collectors/endecadgraph/endecadgraph.py @@ -12,8 +12,11 @@ """ import diamond.collector -import urllib2 -from StringIO import StringIO +import diamond.pycompat +try: + from StringIO import StringIO +except ImportError: + from io import StringIO import re import sys if sys.version_info >= (2, 5): @@ -109,14 +112,17 @@ def walkXML(context, elemList): processElem(elem, elemList) elif event == 'end': elemList.pop() - except Exception, e: + except Exception as e: self.log.error('Something went wrong: %s', e) url = 'http://%s:%d/admin?op=stats' % (self.config['host'], self.config['port']) try: - xml = urllib2.urlopen(url, timeout=self.config['timeout']).read() - except Exception, e: + xml = diamond.pycompat.urlopen( + url, + timeout=self.config['timeout']).read( + ) + except Exception as e: self.log.error('Could not connect to endeca on %s: %s' % (url, e)) return {} diff --git a/src/collectors/endecadgraph/test/testendecadgraph.py b/src/collectors/endecadgraph/test/testendecadgraph.py index 3ff538192..0419c333f 100644 --- a/src/collectors/endecadgraph/test/testendecadgraph.py +++ b/src/collectors/endecadgraph/test/testendecadgraph.py @@ -4,9 +4,10 @@ from test import CollectorTestCase from test import get_collector_config -from mock import patch +from test import patch from diamond.collector import Collector +from diamond.pycompat import URLOPEN from endecadgraph import EndecaDgraphCollector @@ -20,7 +21,7 @@ def setUp(self): def test_import(self): self.assertTrue(EndecaDgraphCollector) - @patch('urllib2.urlopen') + @patch(URLOPEN) @patch.object(Collector, 'publish') def test_real_data(self, publish_mock, urlopen_mock): urlopen_mock.return_value = self.getFixture('data1.xml') diff --git a/src/collectors/etcdstat/etcdstat.py b/src/collectors/etcdstat/etcdstat.py index c376937cd..02bf88be2 100644 --- a/src/collectors/etcdstat/etcdstat.py +++ b/src/collectors/etcdstat/etcdstat.py @@ -13,8 +13,8 @@ """ import diamond.collector +import diamond.pycompat import json -import urllib2 METRICS_KEYS = ['sendPkgRate', 'recvPkgRate', @@ -100,8 +100,8 @@ def get_metrics(self, category): url = "%s://%s:%s/v2/stats/%s" % (protocol, self.config['host'], self.config['port'], category) - return json.load(urllib2.urlopen(url, **opts)) - except (urllib2.HTTPError, ValueError), err: + return json.load(diamond.pycompat.urlopen(url, **opts)) + except (HTTPError, ValueError) as err: self.log.error('Unable to read JSON response: %s' % err) return {} diff --git a/src/collectors/etcdstat/test/test_etcdstat.py b/src/collectors/etcdstat/test/test_etcdstat.py index abb9d9711..8a2a6358a 100644 --- a/src/collectors/etcdstat/test/test_etcdstat.py +++ b/src/collectors/etcdstat/test/test_etcdstat.py @@ -6,8 +6,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import patch -from mock import Mock +from test import patch +from test import Mock from etcdstat import EtcdCollector diff --git a/src/collectors/eventstoreprojections/eventstoreprojections.py b/src/collectors/eventstoreprojections/eventstoreprojections.py index 7c77a01e7..38cc5debb 100644 --- a/src/collectors/eventstoreprojections/eventstoreprojections.py +++ b/src/collectors/eventstoreprojections/eventstoreprojections.py @@ -17,9 +17,9 @@ """ -import urllib2 import json import diamond.collector +import diamond.pycompat class EventstoreProjectionsCollector(diamond.collector.Collector): @@ -64,7 +64,7 @@ def _json_to_flat_metrics(self, prefix, data): for k, v in self._json_to_flat_metrics( "%s.%s" % (prefix, key), value): yield k, v - elif isinstance(value, basestring): + elif isinstance(value, str): if value == "Running": value = 1 yield ("%s.%s" % (prefix, key), value) @@ -91,12 +91,13 @@ def collect(self): self.config['route'] ) - req = urllib2.Request(eventstore_host, headers=self.config['headers']) + req = diamond.pycompat.Request(eventstore_host, + headers=self.config['headers']) req.add_header('Content-type', 'application/json') try: - resp = urllib2.urlopen(req) - except urllib2.URLError as e: + resp = diamond.pycompat.urlopen(req) + except URLError as e: self.log.error("Can't open url %s. %s", eventstore_host, e) else: content = resp.read() diff --git a/src/collectors/eventstoreprojections/tests/testeventstoreprojections.py b/src/collectors/eventstoreprojections/tests/testeventstoreprojections.py index 211560be6..93b5ea95c 100644 --- a/src/collectors/eventstoreprojections/tests/testeventstoreprojections.py +++ b/src/collectors/eventstoreprojections/tests/testeventstoreprojections.py @@ -5,7 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import call, patch +from test import patch +from mock import call from diamond.collector import Collector @@ -23,7 +24,7 @@ def setUp(self): def test_import(self): self.assertTrue(EventstoreProjectionsCollector) - @patch('urllib2.urlopen') + @patch(URLOPEN) @patch.object(Collector, 'publish') def test_should_work_with_real_data(self, publish_mock, urlopen_mock): returns = [self.getFixture('projections')] diff --git a/src/collectors/example/test/testexample.py b/src/collectors/example/test/testexample.py index 4af108657..51c6346fd 100644 --- a/src/collectors/example/test/testexample.py +++ b/src/collectors/example/test/testexample.py @@ -5,7 +5,7 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import patch +from test import patch from diamond.collector import Collector from example import ExampleCollector diff --git a/src/collectors/exim/test/testexim.py b/src/collectors/exim/test/testexim.py index 650147d99..48d91445c 100644 --- a/src/collectors/exim/test/testexim.py +++ b/src/collectors/exim/test/testexim.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from exim import EximCollector diff --git a/src/collectors/filestat/filestat.py b/src/collectors/filestat/filestat.py index fe3a17557..becf5d021 100644 --- a/src/collectors/filestat/filestat.py +++ b/src/collectors/filestat/filestat.py @@ -116,13 +116,13 @@ def get_userlist(self): based on the variables user_include and user_exclude """ # convert user/group lists to arrays if strings - if isinstance(self.config['user_include'], basestring): + if isinstance(self.config['user_include'], str): self.config['user_include'] = self.config['user_include'].split() - if isinstance(self.config['user_exclude'], basestring): + if isinstance(self.config['user_exclude'], str): self.config['user_exclude'] = self.config['user_exclude'].split() - if isinstance(self.config['group_include'], basestring): + if isinstance(self.config['group_include'], str): self.config['group_include'] = self.config['group_include'].split() - if isinstance(self.config['group_exclude'], basestring): + if isinstance(self.config['group_exclude'], str): self.config['group_exclude'] = self.config['group_exclude'].split() rawusers = os.popen("lsof | awk '{ print $3 }' | sort | uniq -d" @@ -206,9 +206,9 @@ def get_typelist(self): typelist = [] # convert type list into arrays if strings - if isinstance(self.config['type_include'], basestring): + if isinstance(self.config['type_include'], str): self.config['type_include'] = self.config['type_include'].split() - if isinstance(self.config['type_exclude'], basestring): + if isinstance(self.config['type_exclude'], str): self.config['type_exclude'] = self.config['type_exclude'].split() # remove any not in include list diff --git a/src/collectors/filestat/test/testfilestat.py b/src/collectors/filestat/test/testfilestat.py index 42f18eadf..a749b00bc 100644 --- a/src/collectors/filestat/test/testfilestat.py +++ b/src/collectors/filestat/test/testfilestat.py @@ -5,13 +5,10 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from test import Mock +from test import patch +from test import StringIO +from test import BUILTIN_OPEN from diamond.collector import Collector from filestat import FilestatCollector @@ -31,7 +28,7 @@ def setUp(self): def test_import(self): self.assertTrue(FilestatCollector) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch('os.access', Mock(return_value=True)) @patch.object(Collector, 'publish') def test_should_open_proc_sys_fs_file_nr(self, publish_mock, open_mock): diff --git a/src/collectors/fluentd/fluentd.py b/src/collectors/fluentd/fluentd.py index 567a86888..ea566754c 100644 --- a/src/collectors/fluentd/fluentd.py +++ b/src/collectors/fluentd/fluentd.py @@ -20,7 +20,7 @@ """ import diamond.collector -import urllib2 +import diamond.pycompat import json @@ -51,7 +51,7 @@ def collect(self): params = (self.config['host'], self.config['port'], self.API_PATH) url = "http://%s:%s/%s" % params - res = urllib2.urlopen(url) + res = diamond.pycompat.urlopen(url) data = json.load(res) result = self.parse_api_output(data) diff --git a/src/collectors/flume/flume.py b/src/collectors/flume/flume.py index 06584eda0..d10c076a9 100644 --- a/src/collectors/flume/flume.py +++ b/src/collectors/flume/flume.py @@ -5,14 +5,12 @@ #### Dependencies - * urllib2 * json or simplejson """ -import urllib2 import diamond.collector - +import diamond.pycompat try: import simplejson as json except ImportError: @@ -79,22 +77,22 @@ def collect(self): ) try: - resp = urllib2.urlopen(url) + resp = diamond.pycompat.urlopen(url) try: j = json.loads(resp.read()) resp.close() - except Exception, e: + except Exception as e: resp.close() self.log.error('Cannot load json data: %s', e) return None - except urllib2.URLError, e: + except diamond.pycompat.URLError as e: self.log.error('Failed to open url: %s', e) return None - except Exception, e: + except Exception as e: self.log.error('Unknown error opening url: %s', e) return None - for comp in j.iteritems(): + for comp in j.items(): comp_name = comp[0] comp_items = comp[1] comp_type = comp_items['Type'] diff --git a/src/collectors/flume/test/testflume.py b/src/collectors/flume/test/testflume.py index b19cc51f5..4a4f14efe 100644 --- a/src/collectors/flume/test/testflume.py +++ b/src/collectors/flume/test/testflume.py @@ -4,10 +4,11 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import patch -from mock import Mock +from test import patch +from test import Mock from diamond.collector import Collector +from diamond.pycompat import URLOPEN from flume import FlumeCollector @@ -30,7 +31,7 @@ def test_collect_should_work(self, publish_mock, publish_gauge_mock, publish_counter_mock): - patch_urlopen = patch('urllib2.urlopen', + patch_urlopen = patch(URLOPEN, Mock(return_value=self.getFixture('metrics'))) patch_urlopen.start() @@ -70,7 +71,7 @@ def test_collect_should_work(self, @patch.object(Collector, 'publish') def test_blank_should_fail_gracefully(self, publish_mock): - patch_urlopen = patch('urllib2.urlopen', Mock( + patch_urlopen = patch(URLOPEN, Mock( return_value=self.getFixture('metrics_blank'))) patch_urlopen.start() @@ -82,7 +83,7 @@ def test_blank_should_fail_gracefully(self, publish_mock): @patch.object(Collector, 'publish') def test_invalid_should_fail_gracefully(self, publish_mock): patch_urlopen = patch( - 'urllib2.urlopen', + URLOPEN, Mock(return_value=self.getFixture('metrics_invalid'))) patch_urlopen.start() diff --git a/src/collectors/gridengine/test/testgridengine.py b/src/collectors/gridengine/test/testgridengine.py index c84c434e5..870854868 100644 --- a/src/collectors/gridengine/test/testgridengine.py +++ b/src/collectors/gridengine/test/testgridengine.py @@ -3,7 +3,7 @@ from test import CollectorTestCase from test import get_collector_config -from mock import patch +from test import patch import os from diamond.collector import Collector diff --git a/src/collectors/hadoop/test/testhadoop.py b/src/collectors/hadoop/test/testhadoop.py index 66efdb17f..f9c9058dc 100644 --- a/src/collectors/hadoop/test/testhadoop.py +++ b/src/collectors/hadoop/test/testhadoop.py @@ -5,7 +5,7 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import patch +from test import patch from diamond.collector import Collector from hadoop import HadoopCollector diff --git a/src/collectors/haproxy/haproxy.py b/src/collectors/haproxy/haproxy.py index e681e5dc3..0d5772aa2 100644 --- a/src/collectors/haproxy/haproxy.py +++ b/src/collectors/haproxy/haproxy.py @@ -6,16 +6,15 @@ #### Dependencies * urlparse - * urllib2 - """ import re -import urllib2 import base64 import csv import socket import diamond.collector +import diamond.pycompat +from diamond.pycompat import Request class HAProxyCollector(diamond.collector.Collector): @@ -64,11 +63,11 @@ def http_get_csv_data(self, section=None): Request stats from HAProxy Server """ metrics = [] - req = urllib2.Request(self._get_config_value(section, 'url')) + req = Request(self._get_config_value(section, 'url')) try: - handle = urllib2.urlopen(req) + handle = diamond.pycompat.urlopen(req) return handle.readlines() - except Exception, e: + except Exception as e: if not hasattr(e, 'code') or e.code != 401: self.log.error("Error retrieving HAProxy stats. %s", e) return metrics @@ -101,10 +100,10 @@ def http_get_csv_data(self, section=None): authheader = 'Basic %s' % base64string req.add_header("Authorization", authheader) try: - handle = urllib2.urlopen(req) + handle = diamond.pycompat.urlopen(req) metrics = handle.readlines() return metrics - except IOError, e: + except IOError as e: # here we shouldn't fail if the USER/PASS is right self.log.error("Error retrieving HAProxy stats. " + "(Invalid username or password?) %s", e) @@ -122,7 +121,7 @@ def unix_get_csv_data(self): if not buf: break data += buf - except socket.error, e: + except socket.error as e: self.log.error("Error retrieving HAProxy stats. %s", e) return [] diff --git a/src/collectors/haproxy/test/testhaproxy.py b/src/collectors/haproxy/test/testhaproxy.py index d3b98c065..145924e00 100644 --- a/src/collectors/haproxy/test/testhaproxy.py +++ b/src/collectors/haproxy/test/testhaproxy.py @@ -5,10 +5,11 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector +from diamond.pycompat import URLOPEN from haproxy import HAProxyCollector ########################################################################## @@ -30,7 +31,7 @@ def test_import(self): def test_should_work_with_real_data(self, publish_mock): self.collector.config['ignore_servers'] = False - patch_urlopen = patch('urllib2.urlopen', + patch_urlopen = patch(URLOPEN, Mock(return_value=self.getFixture('stats.csv'))) patch_urlopen.start() @@ -84,7 +85,7 @@ def recv(obj, bufsize, *args, **kwargs): def test_should_work_with_real_data_and_ignore_servers(self, publish_mock): self.collector.config['ignore_servers'] = True - patch_urlopen = patch('urllib2.urlopen', + patch_urlopen = patch(URLOPEN, Mock(return_value=self.getFixture('stats.csv'))) patch_urlopen.start() diff --git a/src/collectors/http/http.py b/src/collectors/http/http_collector.py similarity index 90% rename from src/collectors/http/http.py rename to src/collectors/http/http_collector.py index 282154a59..ae6e37af6 100644 --- a/src/collectors/http/http.py +++ b/src/collectors/http/http_collector.py @@ -3,10 +3,6 @@ """ Collect statistics from a HTTP or HTTPS connexion -#### Dependencies - - * urllib2 - #### Usage Add the collector config as : @@ -27,8 +23,9 @@ http:__www_site_com_admin_page_html """ -import urllib2 import diamond.collector +import diamond.pycompat +from diamond.pycompat import Request import datetime @@ -55,7 +52,7 @@ def get_default_config(self): return default_config def collect(self): - # create urllib2 vars + # create Request vars if self.config['req_vhost'] != "": self.config['headers']['Host'] = self.config['req_vhost'] @@ -63,9 +60,9 @@ def collect(self): for url in self.config['req_url']: self.log.debug("collecting %s", str(url)) req_start = datetime.datetime.now() - req = urllib2.Request(url, headers=self.config['headers']) + req = diamond.pycompat.Request(url, headers=self.config['headers']) try: - handle = urllib2.urlopen(req) + handle = diamond.pycompat.urlopen(req) the_page = handle.read() req_end = datetime.datetime.now() req_time = req_end - req_start @@ -86,8 +83,8 @@ def collect(self): metric_name + '.size', len(the_page)) - except IOError, e: + except IOError as e: self.log.error("Unable to open %s", self.config['req_url']) - except Exception, e: + except Exception as e: self.log.error("Unknown error opening url: %s", e) diff --git a/src/collectors/http/test/testhttp.py b/src/collectors/http/test/testhttp.py index 3f4158ab5..4af4b97ab 100644 --- a/src/collectors/http/test/testhttp.py +++ b/src/collectors/http/test/testhttp.py @@ -5,12 +5,13 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector +from diamond.pycompat import URLOPEN -from http import HttpCollector +from http_collector import HttpCollector ########################################################################## @@ -30,7 +31,7 @@ def test_import(self): @patch.object(Collector, 'publish') def test_should_work_with_real_data(self, publish_mock): - patch_urlopen = patch('urllib2.urlopen', Mock( + patch_urlopen = patch(URLOPEN, Mock( return_value=self.getFixture('index'))) patch_urlopen.start() diff --git a/src/collectors/httpd/httpd.py b/src/collectors/httpd/httpd.py index 4ceeab48c..9dd41da1c 100644 --- a/src/collectors/httpd/httpd.py +++ b/src/collectors/httpd/httpd.py @@ -6,15 +6,16 @@ #### Dependencies * mod_status - * httplib - * urlparse - """ import re -import httplib -import urlparse +try: + from httplib import HTTPConnection +except ImportError: + from http.client import HTTPConnection + import diamond.collector +from diamond.pycompat import urlparse class HttpdCollector(diamond.collector.Collector): @@ -25,7 +26,7 @@ def process_config(self): self.config['urls'].append(self.config['url']) self.urls = {} - if isinstance(self.config['urls'], basestring): + if isinstance(self.config['urls'], str): self.config['urls'] = self.config['urls'].split(',') for url in self.config['urls']: @@ -66,7 +67,7 @@ def collect(self): while True: # Parse Url - parts = urlparse.urlparse(url) + parts = urlparse(url) # Parse host and port endpoint = parts[1].split(':') @@ -78,8 +79,8 @@ def collect(self): service_port = 80 # Setup Connection - connection = httplib.HTTPConnection(service_host, - service_port) + connection = HTTPConnection(service_host, + service_port) url = "%s?%s" % (parts[2], parts[4]) @@ -93,7 +94,7 @@ def collect(self): break url = headers['location'] connection.close() - except Exception, e: + except Exception as e: self.log.error( "Error retrieving HTTPD stats for host %s:%s, url '%s': %s", service_host, str(service_port), url, e) diff --git a/src/collectors/httpd/test/testhttpd.py b/src/collectors/httpd/test/testhttpd.py index 643795d4e..233db6bac 100644 --- a/src/collectors/httpd/test/testhttpd.py +++ b/src/collectors/httpd/test/testhttpd.py @@ -5,18 +5,20 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from httpd import HttpdCollector -import httplib +try: + from httplib import HTTPConnection, HTTPResponse +except ImportError: + from http.client import HTTPConnection, HTTPResponse ########################################################################## -class TestHTTPResponse(httplib.HTTPResponse): - +class TestHTTPResponse(HTTPResponse): def __init__(self): pass @@ -39,8 +41,8 @@ def setUp(self, config=None): self.HTTPResponse = TestHTTPResponse() - httplib.HTTPConnection.request = Mock(return_value=True) - httplib.HTTPConnection.getresponse = Mock( + HTTPConnection.request = Mock(return_value=True) + HTTPConnection.getresponse = Mock( return_value=self.HTTPResponse) def test_import(self): diff --git a/src/collectors/httpjson/httpjson.py b/src/collectors/httpjson/httpjson.py index 304f0beee..406f81c3f 100644 --- a/src/collectors/httpjson/httpjson.py +++ b/src/collectors/httpjson/httpjson.py @@ -2,16 +2,12 @@ """ Simple collector which get JSON and parse it into flat metrics - -#### Dependencies - - * urllib2 - """ -import urllib2 import json import diamond.collector +import diamond.pycompat +from diamond.pycompat import Request, URLError class HTTPJSONCollector(diamond.collector.Collector): @@ -51,12 +47,12 @@ def _json_to_flat_metrics(self, prefix, data): def collect(self): url = self.config['url'] - req = urllib2.Request(url, headers=self.config['headers']) + req = Request(url, headers=self.config['headers']) req.add_header('Content-type', 'application/json') try: - resp = urllib2.urlopen(req) - except urllib2.URLError as e: + resp = diamond.pycompat.urlopen(req) + except URLError as e: self.log.error("Can't open url %s. %s", url, e) else: diff --git a/src/collectors/httpjson/test/testhttpjson.py b/src/collectors/httpjson/test/testhttpjson.py index e2dea204c..0a90435c3 100644 --- a/src/collectors/httpjson/test/testhttpjson.py +++ b/src/collectors/httpjson/test/testhttpjson.py @@ -3,9 +3,10 @@ ########################################################################## from test import CollectorTestCase from test import get_collector_config -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector +from diamond.pycompat import URLOPEN from httpjson import HTTPJSONCollector ########################################################################## @@ -22,7 +23,7 @@ def test_import(self): @patch.object(Collector, 'publish') def test_should_work_with_real_data(self, publish_mock): - urlopen_mock = patch('urllib2.urlopen', + urlopen_mock = patch(URLOPEN, Mock(return_value=self.getFixture('stats.json'))) urlopen_mock.start() diff --git a/src/collectors/interrupt/interrupt.py b/src/collectors/interrupt/interrupt.py index ccf52f8b0..cf1ae609d 100644 --- a/src/collectors/interrupt/interrupt.py +++ b/src/collectors/interrupt/interrupt.py @@ -13,6 +13,7 @@ import platform import os import diamond.collector +from diamond.pycompat import long # Detect the architecture of the system # and set the counters for MAX_VALUES diff --git a/src/collectors/interrupt/soft.py b/src/collectors/interrupt/soft.py index 31626fc88..1cfe8e1b6 100644 --- a/src/collectors/interrupt/soft.py +++ b/src/collectors/interrupt/soft.py @@ -13,6 +13,7 @@ import platform import os import diamond.collector +from diamond.pycompat import long # Detect the architecture of the system # and set the counters for MAX_VALUES diff --git a/src/collectors/interrupt/test/testinterrupt.py b/src/collectors/interrupt/test/testinterrupt.py index 3f978c373..178be7a6c 100644 --- a/src/collectors/interrupt/test/testinterrupt.py +++ b/src/collectors/interrupt/test/testinterrupt.py @@ -5,13 +5,10 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from test import Mock +from test import patch +from test import StringIO +from test import BUILTIN_OPEN from diamond.collector import Collector from interrupt import InterruptCollector @@ -31,7 +28,7 @@ def setUp(self): def test_import(self): self.assertTrue(InterruptCollector) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch('os.access', Mock(return_value=True)) @patch.object(Collector, 'publish') def test_should_open_proc_stat(self, publish_mock, open_mock): diff --git a/src/collectors/interrupt/test/testsoft.py b/src/collectors/interrupt/test/testsoft.py index 7c9eb2afd..e04108094 100644 --- a/src/collectors/interrupt/test/testsoft.py +++ b/src/collectors/interrupt/test/testsoft.py @@ -5,13 +5,10 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from test import Mock +from test import patch +from test import StringIO +from test import BUILTIN_OPEN from diamond.collector import Collector from soft import SoftInterruptCollector @@ -31,7 +28,7 @@ def setUp(self): def test_import(self): self.assertTrue(SoftInterruptCollector) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch('os.access', Mock(return_value=True)) @patch.object(Collector, 'publish') def test_should_open_proc_stat(self, publish_mock, open_mock): @@ -41,7 +38,7 @@ def test_should_open_proc_stat(self, publish_mock, open_mock): @patch.object(Collector, 'publish') def test_should_work_with_synthetic_data(self, publish_mock): - patch_open = patch('__builtin__.open', Mock(return_value=StringIO( + patch_open = patch(BUILTIN_OPEN, Mock(return_value=StringIO( 'softirq 0 0 0 0 0 0 0 0 0 0 0' ))) @@ -51,7 +48,7 @@ def test_should_work_with_synthetic_data(self, publish_mock): self.assertPublishedMany(publish_mock, {}) - patch_open = patch('__builtin__.open', Mock(return_value=StringIO( + patch_open = patch(BUILTIN_OPEN, Mock(return_value=StringIO( 'softirq 55 1 2 3 4 5 6 7 8 9 10' ))) diff --git a/src/collectors/ip/ip.py b/src/collectors/ip/ip.py index 5d916b4a8..70ccf77ce 100644 --- a/src/collectors/ip/ip.py +++ b/src/collectors/ip/ip.py @@ -24,6 +24,7 @@ """ import diamond.collector +from diamond.pycompat import long import os @@ -103,7 +104,7 @@ def collect(self): data = data.split() # Zip up the keys and values - for i in xrange(1, len(header)): + for i in range(1, len(header)): metrics[header[i]] = data[i] for metric_name in metrics.keys(): diff --git a/src/collectors/ip/test/testip.py b/src/collectors/ip/test/testip.py index 4ac373d92..748da6548 100644 --- a/src/collectors/ip/test/testip.py +++ b/src/collectors/ip/test/testip.py @@ -5,13 +5,10 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from test import Mock +from test import patch +from test import StringIO +from test import BUILTIN_OPEN from ip import IPCollector @@ -33,7 +30,7 @@ def test_import(self): self.assertTrue(IPCollector) @patch('os.access', Mock(return_value=True)) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch('diamond.collector.Collector.publish') def test_should_open_proc_net_snmp(self, publish_mock, open_mock): IPCollector.PROC = ['/proc/net/snmp'] @@ -42,7 +39,7 @@ def test_should_open_proc_net_snmp(self, publish_mock, open_mock): open_mock.assert_called_once_with('/proc/net/snmp') @patch('os.access', Mock(return_value=True)) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch('diamond.collector.Collector.publish') def test_should_work_with_synthetic_data(self, publish_mock, open_mock): IPCollector.PROC = ['/proc/net/snmp'] diff --git a/src/collectors/ipmisensor/test/testipmisensor.py b/src/collectors/ipmisensor/test/testipmisensor.py index 649fb2cf3..c1c970bc8 100644 --- a/src/collectors/ipmisensor/test/testipmisensor.py +++ b/src/collectors/ipmisensor/test/testipmisensor.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from ipmisensor import IPMISensorCollector diff --git a/src/collectors/ipvs/ipvs.py b/src/collectors/ipvs/ipvs.py index e5a4dd868..924118ad0 100644 --- a/src/collectors/ipvs/ipvs.py +++ b/src/collectors/ipvs/ipvs.py @@ -12,7 +12,6 @@ import diamond.collector import subprocess import os -import string from diamond.collector import str_to_bool @@ -76,7 +75,10 @@ def collect(self): lambda a: a != '--exact', self.statcommand) p = subprocess.Popen(self.statcommand, - stdout=subprocess.PIPE).communicate()[0][:-1] + stdout=subprocess.PIPE).communicate()[0] + if isinstance(p, bytes): + p = p.decode("utf8") + p = p[:-1] columns = { 'conns': 2, @@ -94,14 +96,14 @@ def collect(self): row = line.split() if row[0] == "TCP" or row[0] == "UDP": - external = row[0] + "_" + string.replace(row[1], ".", "_") + external = row[0] + "_" + row[1].replace(".", "_") backend = "total" elif row[0] == "->": - backend = string.replace(row[1], ".", "_") + backend = row[1].replace(".", "_") else: continue - for metric, column in columns.iteritems(): + for metric, column in columns.items(): metric_name = ".".join([external, backend, metric]) # metric_value = int(row[column]) value = row[column] @@ -118,7 +120,10 @@ def collect(self): self.publish(metric_name, metric_value) p = subprocess.Popen(self.concommand, - stdout=subprocess.PIPE).communicate()[0][:-1] + stdout=subprocess.PIPE).communicate()[0] + if isinstance(p, bytes): + p = p.decode("utf8") + p = p[:-1] columns = { 'active': 4, @@ -135,21 +140,21 @@ def collect(self): if row[0] == "TCP" or row[0] == "UDP": if total: - for metric, value in total.iteritems(): + for metric, value in total.items(): self.publish( ".".join([external, "total", metric]), value) for k in columns.keys(): total[k] = 0.0 - external = row[0] + "_" + string.replace(row[1], ".", "_") + external = row[0] + "_" + row[1].replace(".", "_") continue elif row[0] == "->": - backend = string.replace(row[1], ".", "_") + backend = row[1].replace(".", "_") else: continue - for metric, column in columns.iteritems(): + for metric, column in columns.items(): metric_name = ".".join([external, backend, metric]) # metric_value = int(row[column]) value = row[column] @@ -167,5 +172,5 @@ def collect(self): self.publish(metric_name, metric_value) if total: - for metric, value in total.iteritems(): + for metric, value in total.items(): self.publish(".".join([external, "total", metric]), value) diff --git a/src/collectors/ipvs/test/testipvs.py b/src/collectors/ipvs/test/testipvs.py index b0e1c188b..95471734a 100644 --- a/src/collectors/ipvs/test/testipvs.py +++ b/src/collectors/ipvs/test/testipvs.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from ipvs import IPVSCollector diff --git a/src/collectors/jbossapi/jbossapi.py b/src/collectors/jbossapi/jbossapi.py index 70d52c4dc..89c22e596 100644 --- a/src/collectors/jbossapi/jbossapi.py +++ b/src/collectors/jbossapi/jbossapi.py @@ -356,8 +356,10 @@ def get_data(self, op_type, current_host, current_port, current_proto, attributes = subprocess.Popen(the_cmd, shell=True, stdout=subprocess.PIPE ).communicate()[0] + if isinstance(attributes, bytes): + attributes = attributes.decode("utf8") output = json.loads(attributes) - except Exception, e: + except Exception as e: self.log.error("JbossApiCollector: There was an exception %s", e) output = '' return output diff --git a/src/collectors/jcollectd/collectd_network.py b/src/collectors/jcollectd/collectd_network.py index 4cda897d8..224066576 100644 --- a/src/collectors/jcollectd/collectd_network.py +++ b/src/collectors/jcollectd/collectd_network.py @@ -367,7 +367,7 @@ def interpret(self, iterable=None, poll_interval=0.2): if iterable is None: return None - if isinstance(iterable, basestring): + if isinstance(iterable, str): iterable = self.decode(poll_interval, iterable) return interpret_opcodes(iterable) diff --git a/src/collectors/jcollectd/jcollectd.py b/src/collectors/jcollectd/jcollectd.py index edeed256c..958cd5aa9 100644 --- a/src/collectors/jcollectd/jcollectd.py +++ b/src/collectors/jcollectd/jcollectd.py @@ -25,10 +25,10 @@ import threading import re -import Queue import diamond.collector import diamond.metric +from diamond.pycompat import Empty, Full, Queue import collectd_network @@ -63,7 +63,7 @@ def collect(self): try: dp = q.get(False) metric = self.make_metric(dp) - except Queue.Empty: + except Empty: break self.publish_metric(metric) @@ -118,7 +118,7 @@ def __init__(self, host, port, log, poll_interval=0.4): self.log = log self.poll_interval = poll_interval - self.queue = Queue.Queue() + self.queue = Queue() def run(self): self.log.info('ListenerThread started on {0}:{1}(udp)'.format( @@ -131,9 +131,9 @@ def run(self): try: items = rdr.interpret(poll_interval=self.poll_interval) self.send_to_collector(items) - except ValueError, e: + except ValueError as e: self.log.warn('Dropping bad packet: {0}'.format(e)) - except Exception, e: + except Exception as e: self.log.error('caught exception: type={0}, exc={1}'.format(type(e), e)) @@ -147,9 +147,9 @@ def send_to_collector(self, items): try: metric = self.transform(item) self.queue.put(metric) - except Queue.Full: + except Full: self.log.error('Queue to collector is FULL') - except Exception, e: + except Exception as e: self.log.error('B00M! type={0}, exception={1}'.format(type(e), e)) diff --git a/src/collectors/jolokia/jolokia.py b/src/collectors/jolokia/jolokia.py index 37c0adb89..899d0073e 100644 --- a/src/collectors/jolokia/jolokia.py +++ b/src/collectors/jolokia/jolokia.py @@ -51,10 +51,10 @@ import diamond.collector import base64 from contextlib import closing +import diamond.pycompat import json import re -import urllib -import urllib2 +from diamond.pycompat import HTTPError, long, quote class JolokiaCollector(diamond.collector.Collector): @@ -120,7 +120,7 @@ def get_default_config(self): def __init__(self, *args, **kwargs): super(JolokiaCollector, self).__init__(*args, **kwargs) self.mbeans = [] - if isinstance(self.config['mbeans'], basestring): + if isinstance(self.config['mbeans'], str): for mbean in self.config['mbeans'].split('|'): self.mbeans.append(mbean.strip()) elif isinstance(self.config['mbeans'], list): @@ -139,7 +139,7 @@ def __init__(self, *args, **kwargs): self.domains = [] if 'domains' in self.config: - if isinstance(self.config['domains'], basestring): + if isinstance(self.config['domains'], str): for domain in self.config['domains'].split('|'): self.domains.append(domain.strip()) elif isinstance(self.config['domains'], list): @@ -202,7 +202,7 @@ def collect(self): # The reponse was totally empty, or not an expected format self.log.error('Unable to retrieve domain %s.', domain) continue - for k, v in mbeans.iteritems(): + for k, v in mbeans.items(): if self._check_mbean(k): self.collect_bean(k, v) @@ -230,16 +230,16 @@ def _list_request(self): # need some time to process the downloaded metrics, so that's why # timeout is lower than the interval. timeout = max(2, float(self.config['interval']) * 2 / 3) - with closing(urllib2.urlopen(self._create_request(url), - timeout=timeout)) as response: + with closing(diamond.pycompat.urlopen(self._create_request(url), + timeout=timeout)) as response: return self._read_json(response) - except (urllib2.HTTPError, ValueError) as e: + except (HTTPError, ValueError) as e: self.log.error('Unable to read JSON response: %s', str(e)) return {} def _read_request(self, domain): try: - url_path = '/?%s' % urllib.urlencode({ + url_path = '/?%s' % diamond.pycompat.urlencode({ 'maxCollectionSize': '0', 'ignoreErrors': 'true', 'canonicalNaming': @@ -253,10 +253,10 @@ def _read_request(self, domain): # need some time to process the downloaded metrics, so that's why # timeout is lower than the interval. timeout = max(2, float(self.config['interval']) * 2 / 3) - with closing(urllib2.urlopen(self._create_request(url), - timeout=timeout)) as response: + with closing(diamond.pycompat.urlopen(self._create_request(url), + timeout=timeout)) as response: return self._read_json(response) - except (urllib2.HTTPError, ValueError): + except (HTTPError, ValueError): self.log.error('Unable to read JSON response.') return {} @@ -268,11 +268,11 @@ def _escape_domain(self, domain): domain = re.sub('!', '!!', domain) domain = re.sub('/', '!/', domain) domain = re.sub('"', '!"', domain) - domain = urllib.quote(domain) + domain = quote(domain) return domain def _create_request(self, url): - req = urllib2.Request(url) + req = diamond.pycompat.Request(url) username = self.config["username"] password = self.config["password"] if username is not None and password is not None: @@ -287,7 +287,7 @@ def clean_up(self, text): return text def collect_bean(self, prefix, obj): - for k, v in obj.iteritems(): + for k, v in obj.items(): if type(v) in [int, float, long]: key = "%s.%s" % (prefix, k) key = self.clean_up(key) diff --git a/src/collectors/jolokia/test/testcassandra_jolokia.py b/src/collectors/jolokia/test/testcassandra_jolokia.py index a2cd6831c..3ad9ffd73 100644 --- a/src/collectors/jolokia/test/testcassandra_jolokia.py +++ b/src/collectors/jolokia/test/testcassandra_jolokia.py @@ -5,7 +5,7 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import patch +from test import patch from diamond.collector import Collector diff --git a/src/collectors/jolokia/test/testjolokia.py b/src/collectors/jolokia/test/testjolokia.py index 4af23123c..4ffa2f862 100644 --- a/src/collectors/jolokia/test/testjolokia.py +++ b/src/collectors/jolokia/test/testjolokia.py @@ -5,11 +5,12 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch import re from diamond.collector import Collector +from diamond.pycompat import URLOPEN from jolokia import JolokiaCollector @@ -33,7 +34,7 @@ def se(url, timeout=0): return self.getFixture('listing') else: return self.getFixture('stats') - patch_urlopen = patch('urllib2.urlopen', Mock(side_effect=se)) + patch_urlopen = patch(URLOPEN, Mock(side_effect=se)) patch_urlopen.start() self.collector.collect() @@ -52,7 +53,7 @@ def se(url, timeout=0): return self.getFixture('listing') else: return self.getFixture('stats') - patch_urlopen = patch('urllib2.urlopen', Mock(side_effect=se)) + patch_urlopen = patch(URLOPEN, Mock(side_effect=se)) patch_urlopen.start() rewrite = [ @@ -74,7 +75,7 @@ def test_should_work_with_real_data_and_basic_auth(self, publish_mock): @patch.object(Collector, 'publish') def test_should_fail_gracefully(self, publish_mock): - patch_urlopen = patch('urllib2.urlopen', Mock( + patch_urlopen = patch(URLOPEN, Mock( return_value=self.getFixture('stats_blank'))) patch_urlopen.start() @@ -93,7 +94,7 @@ def se(url, timeout=0): return self.getFixture('stats_error') else: return self.getFixture('stats') - patch_urlopen = patch('urllib2.urlopen', Mock(side_effect=se)) + patch_urlopen = patch(URLOPEN, Mock(side_effect=se)) patch_urlopen.start() self.collector.collect() @@ -126,7 +127,7 @@ def test_canonical_names_setting_not_set(self): logger_mock.error.assert_not_called() @patch('jolokia.JolokiaCollector._create_request') - @patch('urllib2.urlopen') + @patch(URLOPEN) def test_should_handle_canonical_names_setting_True(self, urlopen_mock, create_request_mock): config = get_collector_config('JolokiaCollector', {}) @@ -144,7 +145,7 @@ def test_should_handle_canonical_names_setting_True(self, urlopen_mock, self.assertIs(collector.config['use_canonical_names'], True) @patch('jolokia.JolokiaCollector._create_request') - @patch('urllib2.urlopen') + @patch(URLOPEN) def test_should_handle_canonical_names_setting_False(self, urlopen_mock, create_request_mock): config = get_collector_config('JolokiaCollector', {}) diff --git a/src/collectors/kafka_consumer_lag/test/testkafka_consumer_lag.py b/src/collectors/kafka_consumer_lag/test/testkafka_consumer_lag.py index 54be46a1b..dff0311a5 100644 --- a/src/collectors/kafka_consumer_lag/test/testkafka_consumer_lag.py +++ b/src/collectors/kafka_consumer_lag/test/testkafka_consumer_lag.py @@ -4,7 +4,8 @@ from test import CollectorTestCase from test import get_collector_config -from mock import patch, Mock +from test import patch +from test import Mock from diamond.collector import Collector from kafka_consumer_lag import KafkaConsumerLagCollector diff --git a/src/collectors/kafkastat/kafkastat.py b/src/collectors/kafkastat/kafkastat.py index fefd3ab9f..b08a810f6 100644 --- a/src/collectors/kafkastat/kafkastat.py +++ b/src/collectors/kafkastat/kafkastat.py @@ -5,13 +5,8 @@ #### Dependencies - * urllib2 * xml.etree """ -import urllib2 - -from urllib import urlencode - try: from xml.etree import ElementTree except ImportError: @@ -23,6 +18,8 @@ ETParseError = Exception import diamond.collector +import diamond.pycompat +from diamond.pycompat import long, urlencode, URLError class KafkaCollector(diamond.collector.Collector): @@ -67,8 +64,8 @@ def _get(self, path, query_args=None): path, urlencode(qargs)) try: - response = urllib2.urlopen(url) - except urllib2.URLError, err: + response = diamond.pycompat.urlopen(url) + except URLError as err: self.log.error("%s: %s", url, err) return None @@ -179,5 +176,5 @@ def collect(self): metrics.update(stats) # Publish stats - for metric, value in metrics.iteritems(): + for metric, value in metrics.items(): self.publish(metric, value) diff --git a/src/collectors/kafkastat/test/testkafka.py b/src/collectors/kafkastat/test/testkafka.py index b00ddeebc..5c05dbea2 100644 --- a/src/collectors/kafkastat/test/testkafka.py +++ b/src/collectors/kafkastat/test/testkafka.py @@ -1,9 +1,6 @@ #!/usr/bin/python # coding=utf-8 ############################################################################### -import urllib2 -from urlparse import urlparse, parse_qs - try: from xml.etree import ElementTree except ImportError: @@ -13,9 +10,10 @@ from test import get_collector_config from test import run_only from test import unittest -from mock import patch +from test import patch from diamond.collector import Collector +from diamond.pycompat import long, URLError, URLOPEN from kafkastat import KafkaCollector ########## @@ -48,26 +46,26 @@ def test_import(self): self.assertTrue(KafkaCollector) @run_only_if_ElementTree_is_available - @patch('urllib2.urlopen') + @patch(URLOPEN) def test_get(self, urlopen_mock): urlopen_mock.return_value = self.getFixture('empty.xml') result = self.collector._get('/path') result_string = ElementTree.tostring(result) - self.assertEqual(result_string, '') + self.assertEqual(result_string, b'') @run_only_if_ElementTree_is_available - @patch('urllib2.urlopen') + @patch(URLOPEN) def test_get_httperror(self, urlopen_mock): - urlopen_mock.side_effect = urllib2.URLError('BOOM') + urlopen_mock.side_effect = URLError('BOOM') result = self.collector._get('/path') self.assertFalse(result) @run_only_if_ElementTree_is_available - @patch('urllib2.urlopen') + @patch(URLOPEN) def test_get_bad_xml(self, urlopen_mock): urlopen_mock.return_value = self.getFixture('bad.xml') @@ -173,7 +171,7 @@ def getKafkaFixture(self, url): return '' @run_only_if_ElementTree_is_available - @patch('urllib2.urlopen') + @patch(URLOPEN) @patch.object(Collector, 'publish') def test(self, publish_mock, urlopen_mock): urlopen_mock.side_effect = self.getKafkaFixture diff --git a/src/collectors/ksm/test/testksm.py b/src/collectors/ksm/test/testksm.py index 8f4e2fd9e..bd2177fd4 100644 --- a/src/collectors/ksm/test/testksm.py +++ b/src/collectors/ksm/test/testksm.py @@ -7,8 +7,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from ksm import KSMCollector diff --git a/src/collectors/kvm/test/testkvm.py b/src/collectors/kvm/test/testkvm.py index f1f8cd951..1bbcf8115 100644 --- a/src/collectors/kvm/test/testkvm.py +++ b/src/collectors/kvm/test/testkvm.py @@ -7,13 +7,10 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from test import Mock +from test import patch +from test import StringIO +from test import BUILTIN_OPEN from diamond.collector import Collector from kvm import KVMCollector @@ -37,7 +34,7 @@ def test_import(self): @patch('os.access', Mock(return_value=True)) @patch.object(Collector, 'publish') def test_should_work_with_synthetic_data(self, publish_mock): - patch_open = patch('__builtin__.open', Mock(return_value=StringIO( + patch_open = patch(BUILTIN_OPEN, Mock(return_value=StringIO( '0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0' + '\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n' ))) diff --git a/src/collectors/lmsensors/test/testlmsensors.py b/src/collectors/lmsensors/test/testlmsensors.py index 9c7060f71..356480af1 100644 --- a/src/collectors/lmsensors/test/testlmsensors.py +++ b/src/collectors/lmsensors/test/testlmsensors.py @@ -5,7 +5,7 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import patch +from test import patch from diamond.collector import Collector from lmsensors import LMSensorsCollector diff --git a/src/collectors/loadavg/test/testloadavg.py b/src/collectors/loadavg/test/testloadavg.py index 6f237019e..1f5dd4c13 100644 --- a/src/collectors/loadavg/test/testloadavg.py +++ b/src/collectors/loadavg/test/testloadavg.py @@ -6,13 +6,10 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from test import Mock +from test import patch +from test import StringIO +from test import BUILTIN_OPEN from diamond.collector import Collector from loadavg import LoadAverageCollector @@ -32,7 +29,7 @@ def setUp(self): def test_import(self): self.assertTrue(LoadAverageCollector) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch('os.access', Mock(return_value=True)) @patch.object(Collector, 'publish') def test_should_open_proc_loadavg(self, publish_mock, open_mock): diff --git a/src/collectors/mdstat/test/test_mdstat.py b/src/collectors/mdstat/test/test_mdstat.py index 94f724b23..976df091f 100644 --- a/src/collectors/mdstat/test/test_mdstat.py +++ b/src/collectors/mdstat/test/test_mdstat.py @@ -6,8 +6,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from mdstat import MdStatCollector diff --git a/src/collectors/memcached/memcached.py b/src/collectors/memcached/memcached.py index 22a9fbad5..00c021415 100644 --- a/src/collectors/memcached/memcached.py +++ b/src/collectors/memcached/memcached.py @@ -151,7 +151,7 @@ def collect(self): hosts = self.config.get('hosts') # Convert a string config value to be an array - if isinstance(hosts, basestring): + if isinstance(hosts, str): hosts = [hosts] for host in hosts: diff --git a/src/collectors/memcached/test/testmemcached.py b/src/collectors/memcached/test/testmemcached.py index 1fd69cc25..39eefb507 100644 --- a/src/collectors/memcached/test/testmemcached.py +++ b/src/collectors/memcached/test/testmemcached.py @@ -6,8 +6,8 @@ from test import get_collector_config from test import unittest from mock import MagicMock -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from memcached import MemcachedCollector diff --git a/src/collectors/memory/test/testmemory.py b/src/collectors/memory/test/testmemory.py index 4ea45e808..5822519eb 100644 --- a/src/collectors/memory/test/testmemory.py +++ b/src/collectors/memory/test/testmemory.py @@ -5,13 +5,10 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from test import Mock +from test import patch +from test import StringIO +from test import BUILTIN_OPEN from diamond.collector import Collector from memory import MemoryCollector @@ -32,7 +29,7 @@ def setUp(self): def test_import(self): self.assertTrue(MemoryCollector) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch('os.access', Mock(return_value=True)) @patch.object(Collector, 'publish') def test_should_open_proc_meminfo(self, publish_mock, open_mock): diff --git a/src/collectors/memory_cgroup/memory_cgroup.py b/src/collectors/memory_cgroup/memory_cgroup.py index abbc3f653..e07be6bd4 100644 --- a/src/collectors/memory_cgroup/memory_cgroup.py +++ b/src/collectors/memory_cgroup/memory_cgroup.py @@ -112,8 +112,8 @@ def collect(self): break # create metrics from collected utimes and stimes for cgroups - for parent, cpuacct in results.iteritems(): - for key, value in cpuacct.iteritems(): + for parent, cpuacct in results.items(): + for key, value in cpuacct.items(): metric_name = '.'.join([parent, key]) self.publish(metric_name, value, metric_type='GAUGE') return True diff --git a/src/collectors/memory_cgroup/test/testmemory_cgroup.py b/src/collectors/memory_cgroup/test/testmemory_cgroup.py index 3981e1595..8510a0ce0 100644 --- a/src/collectors/memory_cgroup/test/testmemory_cgroup.py +++ b/src/collectors/memory_cgroup/test/testmemory_cgroup.py @@ -5,13 +5,10 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from test import Mock +from test import patch +from test import StringIO +from test import BUILTIN_OPEN from diamond.collector import Collector from memory_cgroup import MemoryCgroupCollector @@ -28,7 +25,7 @@ class TestMemoryCgroupCollector(CollectorTestCase): def test_import(self): self.assertTrue(MemoryCgroupCollector) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch('os.walk', Mock(return_value=iter(fixtures))) @patch.object(Collector, 'publish') def test_should_open_all_memory_stat(self, publish_mock, open_mock): @@ -112,9 +109,9 @@ def test_should_not_include_filtered_metrics(self, publish_mock): 'lxc.testcontainer.total_swap': 1, } [self.assertPublished(publish_mock, k, v) - for k, v in should_be_published.iteritems()] + for k, v in should_be_published.items()] [self.assertUnpublished(publish_mock, k, v) - for k, v in should_not_be_published.iteritems()] + for k, v in should_not_be_published.items()] if __name__ == "__main__": unittest.main() diff --git a/src/collectors/memory_docker/test/testmemory_docker.py b/src/collectors/memory_docker/test/testmemory_docker.py index 00cdfa5b4..949f176ea 100644 --- a/src/collectors/memory_docker/test/testmemory_docker.py +++ b/src/collectors/memory_docker/test/testmemory_docker.py @@ -6,13 +6,10 @@ from test import get_collector_config from test import unittest from test import run_only -from mock import Mock -from mock import patch - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from test import Mock +from test import patch +from test import StringIO +from test import BUILTIN_OPEN try: from docker import Client @@ -59,7 +56,7 @@ def test_import(self): self.assertTrue(MemoryDockerCollector) @run_only_if_docker_client_is_available - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch.object(Client, 'containers', Mock(return_value=[])) @patch.object(Collector, 'publish') def test_should_open_all_cpuacct_stat(self, publish_mock, open_mock): @@ -71,7 +68,7 @@ def test_should_open_all_cpuacct_stat(self, publish_mock, open_mock): open_mock.assert_any_call(fixtures_path + 'memory.stat') @run_only_if_docker_client_is_available - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch.object(Client, 'containers') @patch.object(Collector, 'publish') def test_should_get_containers(self, publish_mock, containers_mock, diff --git a/src/collectors/mesos/mesos.py b/src/collectors/mesos/mesos.py index 596cf8401..c5ca5866a 100644 --- a/src/collectors/mesos/mesos.py +++ b/src/collectors/mesos/mesos.py @@ -19,9 +19,9 @@ import copy import diamond.collector +import diamond.pycompat +from diamond.pycompat import HTTPError import json -import urllib2 -from urlparse import urlparse import diamond.collector @@ -199,8 +199,8 @@ def _get(self, path): """ url = self._get_url(path) try: - response = urllib2.urlopen(url) - except Exception, err: + response = diamond.pycompat.urlopen(url) + except Exception as err: self.log.error("%s: %s", url, err) return False diff --git a/src/collectors/mesos/test/testmesos.py b/src/collectors/mesos/test/testmesos.py index bb140fa34..bac193925 100644 --- a/src/collectors/mesos/test/testmesos.py +++ b/src/collectors/mesos/test/testmesos.py @@ -5,10 +5,11 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector +from diamond.pycompat import URLOPEN from mesos import MesosCollector @@ -31,7 +32,7 @@ def test_import(self): @patch.object(Collector, 'publish') def test_should_work_for_master_with_real_data(self, publish_mock): returns = self.getFixture('master_metrics_snapshot.json') - urlopen_mock = patch('urllib2.urlopen', Mock( + urlopen_mock = patch('diamond.pycompat.urlopen', Mock( side_effect=lambda *args: returns)) urlopen_mock.start() @@ -61,7 +62,7 @@ def test_should_work_for_slave_with_real_data(self, publish_mock): self.getFixture('slave_monitor_statistics.json') ] - urlopen_mock = patch('urllib2.urlopen', Mock( + urlopen_mock = patch(URLOPEN, Mock( side_effect=lambda *args: returns.pop(0))) urlopen_mock.start() @@ -114,7 +115,7 @@ def test_should_compute_cpus_utilisation(self, publish_mock): @patch.object(Collector, 'publish') def test_should_fail_gracefully(self, publish_mock): - patch_urlopen = patch('urllib2.urlopen', Mock( + patch_urlopen = patch(URLOPEN, Mock( return_value=self.getFixture('metrics_blank'))) patch_urlopen.start() @@ -146,7 +147,7 @@ def fixture_cpu_utilisation(self, publish_mock): self.getFixture('slave_metrics_state.json'), self.getFixture('slave_monitor_statistics_cpus_utilisation.json'), ] - urlopen_mock = patch('urllib2.urlopen', Mock( + urlopen_mock = patch(URLOPEN, Mock( side_effect=lambda *args: returns.pop(0))) urlopen_mock.start() self.collector.collect() diff --git a/src/collectors/mesos_cgroup/mesos_cgroup.py b/src/collectors/mesos_cgroup/mesos_cgroup.py index 325671de3..61b3e9a22 100644 --- a/src/collectors/mesos_cgroup/mesos_cgroup.py +++ b/src/collectors/mesos_cgroup/mesos_cgroup.py @@ -23,8 +23,8 @@ """ import diamond.collector +import diamond.pycompat import json -import urllib2 import os @@ -125,8 +125,8 @@ def get_mesos_state(self): self.config['port'], self.config['mesos_state_path']) - return json.load(urllib2.urlopen(url)) - except (urllib2.HTTPError, ValueError), err: + return json.load(diamond.pycompat.urlopen(url)) + except (HTTPError, ValueError) as err: self.log.error('Unable to read JSON response: %s' % err) return {} diff --git a/src/collectors/mesos_cgroup/test/testmesos_cgroup.py b/src/collectors/mesos_cgroup/test/testmesos_cgroup.py index 152cc9d6f..6fc2d527b 100644 --- a/src/collectors/mesos_cgroup/test/testmesos_cgroup.py +++ b/src/collectors/mesos_cgroup/test/testmesos_cgroup.py @@ -5,10 +5,12 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import MagicMock, Mock, mock_open -from mock import patch +from mock import MagicMock, mock_open +from test import patch +from test import Mock from diamond.collector import Collector +from diamond.pycompat import URLOPEN from mesos_cgroup import MesosCGroupCollector @@ -34,7 +36,7 @@ def urlopen_se(url): if url == 'http://localhost:5051/state.json': return self.getFixture('state.json') else: - print url + print(url) raise NotImplementedError() def listdir_se(directory): @@ -47,7 +49,7 @@ def listdir_se(directory): if directory in cgroup_directories: return ["b0d5971e-915c-414b-aa25-0da46e64ff4e"] else: - print directory + print(directory) raise NotImplementedError() def isdir_se(directory): @@ -60,7 +62,7 @@ def isdir_se(directory): if directory in task_directories: return True else: - print directory + print(directory) raise NotImplementedError() def open_se(path, mode='r', create=True): @@ -90,7 +92,7 @@ def open_se(path, mode='r', create=True): patch_open.start() return o - patch_urlopen = patch('urllib2.urlopen', Mock(side_effect=urlopen_se)) + patch_urlopen = patch(URLOPEN, Mock(side_effect=urlopen_se)) patch_listdir = patch('os.listdir', Mock(side_effect=listdir_se)) patch_isdir = patch('os.path.isdir', Mock(side_effect=isdir_se)) patch_open = patch('__builtin__.open', MagicMock(spec=file, diff --git a/src/collectors/mogilefs/test/testmogilefs.py b/src/collectors/mogilefs/test/testmogilefs.py index 8787e165a..e13b07153 100644 --- a/src/collectors/mogilefs/test/testmogilefs.py +++ b/src/collectors/mogilefs/test/testmogilefs.py @@ -5,8 +5,9 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock, call -from mock import patch +from mock import call +from test import Mock +from test import patch from diamond.collector import Collector diff --git a/src/collectors/mongodb/mongodb.py b/src/collectors/mongodb/mongodb.py index 8c3126ad1..7e5f83040 100644 --- a/src/collectors/mongodb/mongodb.py +++ b/src/collectors/mongodb/mongodb.py @@ -115,7 +115,7 @@ def collect(self): hosts = self.config.get('hosts') # Convert a string config value to be an array - if isinstance(hosts, basestring): + if isinstance(hosts, str): hosts = [hosts] # we need this for backwards compatibility @@ -175,7 +175,7 @@ def collect(self): ssl=self.config['ssl'], read_preference=ReadPreference.SECONDARY, ) - except Exception, e: + except Exception as e: self.log.error('Couldnt connect to mongodb: %s', e) continue @@ -183,7 +183,7 @@ def collect(self): if user: try: conn.admin.authenticate(user, passwd) - except Exception, e: + except Exception as e: self.log.error( 'User auth given, but could not autheticate' + ' with host: %s, err: %s' % (host, e)) diff --git a/src/collectors/mongodb/test/testmongodb.py b/src/collectors/mongodb/test/testmongodb.py index 340e192f4..2696ba071 100644 --- a/src/collectors/mongodb/test/testmongodb.py +++ b/src/collectors/mongodb/test/testmongodb.py @@ -6,9 +6,9 @@ from test import get_collector_config from test import unittest from test import run_only -from mock import MagicMock -from mock import patch -from mock import call +from test import MagicMock +from test import patch +from test import call from diamond.collector import Collector from mongodb import MongoDBCollector @@ -19,8 +19,10 @@ except ImportError: import simplejson as json - -########################################################################## +try: + long +except NameError: + long = int def run_only_if_pymongo_is_available(func): diff --git a/src/collectors/monit/monit.py b/src/collectors/monit/monit.py index c3bbd5936..cb7657c44 100644 --- a/src/collectors/monit/monit.py +++ b/src/collectors/monit/monit.py @@ -8,8 +8,6 @@ * monit serving up /_status """ - -import urllib2 import base64 from xml.dom.minidom import parseString @@ -17,6 +15,9 @@ import diamond.collector from diamond.collector import str_to_bool +import diamond.pycompat +from diamond.pycompat import HTTPError, Request + class MonitCollector(diamond.collector.Collector): @@ -47,15 +48,16 @@ def collect(self): url = 'http://%s:%i/_status?format=xml' % (self.config['host'], int(self.config['port'])) try: - request = urllib2.Request(url) + request = Request(url) - # # shouldn't need to check this - base64string = base64.encodestring('%s:%s' % ( - self.config['user'], self.config['passwd'])).replace('\n', '') + original_string = '%s:%s' % (self.config['user'], + self.config['passwd']) + base64string = base64.encodestring(original_string.encode("utf8")) + base64string = base64string.replace(b'\n', b'') request.add_header("Authorization", "Basic %s" % base64string) - response = urllib2.urlopen(request) - except urllib2.HTTPError, err: + response = diamond.pycompat.urlopen(request) + except HTTPError as err: self.log.error("%s: %s", err, url) return diff --git a/src/collectors/monit/test/testmonit.py b/src/collectors/monit/test/testmonit.py index d7ac85a03..d5641ab4f 100644 --- a/src/collectors/monit/test/testmonit.py +++ b/src/collectors/monit/test/testmonit.py @@ -5,10 +5,11 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector +from diamond.pycompat import URLOPEN from monit import MonitCollector @@ -28,7 +29,7 @@ def test_import(self): @patch.object(Collector, 'publish') def test_should_work_with_real_data(self, publish_mock): - patch_urlopen = patch('urllib2.urlopen', Mock( + patch_urlopen = patch(URLOPEN, Mock( return_value=self.getFixture('status.xml'))) patch_urlopen.start() @@ -66,7 +67,7 @@ def test_should_work_with_real_data(self, publish_mock): @patch.object(Collector, 'publish') def test_should_fail_gracefully(self, publish_mock): patch_urlopen = patch( - 'urllib2.urlopen', + URLOPEN, Mock( return_value=self.getFixture( 'status_blank.xml'))) diff --git a/src/collectors/mountstats/mountstats.py b/src/collectors/mountstats/mountstats.py index f8d699657..802e9c6f1 100755 --- a/src/collectors/mountstats/mountstats.py +++ b/src/collectors/mountstats/mountstats.py @@ -60,7 +60,7 @@ class MountStatsCollector(diamond.collector.Collector): def process_config(self): super(MountStatsCollector, self).process_config() self.exclude_filters = self.config['exclude_filters'] - if isinstance(self.exclude_filters, basestring): + if isinstance(self.exclude_filters, str): self.exclude_filters = [self.exclude_filters] if len(self.exclude_filters) > 0: @@ -69,7 +69,7 @@ def process_config(self): self.exclude_reg = None self.include_filters = self.config['include_filters'] - if isinstance(self.include_filters, basestring): + if isinstance(self.include_filters, str): self.include_filters = [self.include_filters] if len(self.include_filters) > 0: @@ -120,7 +120,10 @@ def collect(self): command = [self.config['sudo_cmd'], '/bin/cat', self.MOUNTSTATS] p = subprocess.Popen(command, - stdout=subprocess.PIPE).communicate()[0][:-1] + stdout=subprocess.PIPE).communicate()[0] + if isinstance(p, bytes): + p = p.decode("utf8") + p = p[:-1] lines = p.split("\n") else: @@ -161,12 +164,12 @@ def collect(self): elif tokens[0] == 'events:': for i in range(0, len(self.EVENTS_MAP)): metric_name = "%s.events.%s" % (path, self.EVENTS_MAP[i]) - metric_value = long(tokens[i + 1]) + metric_value = int(tokens[i + 1]) self.publish_counter(metric_name, metric_value) elif tokens[0] == 'bytes:': for i in range(0, len(self.BYTES_MAP)): metric_name = "%s.bytes.%s" % (path, self.BYTES_MAP[i]) - metric_value = long(tokens[i + 1]) + metric_value = int(tokens[i + 1]) self.publish_counter(metric_name, metric_value) elif tokens[0] == 'xprt:': proto = tokens[1] @@ -177,13 +180,13 @@ def collect(self): for i in range(0, len(self.XPRT_MAP[proto])): metric_name = "%s.xprt.%s.%s" % (path, proto, self.XPRT_MAP[proto][i]) - metric_value = long(tokens[i + 2]) + metric_value = int(tokens[i + 2]) self.publish_counter(metric_name, metric_value) elif tokens[0][:-1] in self.RPCS_MAP: rpc = tokens[0][:-1] - ops = long(tokens[1]) - rtt = long(tokens[7]) - exe = long(tokens[8]) + ops = int(tokens[1]) + rtt = int(tokens[7]) + exe = int(tokens[8]) metric_fmt = "%s.rpc.%s.%s" ops_name = metric_fmt % (path, rpc.lower(), 'ops') diff --git a/src/collectors/mountstats/test/testmountstats.py b/src/collectors/mountstats/test/testmountstats.py index e006b74c9..89fc446a6 100644 --- a/src/collectors/mountstats/test/testmountstats.py +++ b/src/collectors/mountstats/test/testmountstats.py @@ -4,8 +4,10 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import MagicMock, Mock -from mock import patch +from test import MagicMock +from test import patch +from test import Mock +from test import BUILTIN_OPEN from diamond.collector import Collector from mountstats import MountStatsCollector @@ -24,7 +26,7 @@ def setUp(self): def test_import(self): self.assertTrue(MountStatsCollector) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch('os.access', Mock(return_value=True)) @patch.object(Collector, 'publish') def test_should_open_mountstats(self, publish_mock, open_mock): diff --git a/src/collectors/mysqlstat/mysql55.py b/src/collectors/mysqlstat/mysql55.py index 3be99a4ea..397edbc5a 100644 --- a/src/collectors/mysqlstat/mysql55.py +++ b/src/collectors/mysqlstat/mysql55.py @@ -133,7 +133,7 @@ def connect(self, params): try: self.db = MySQLdb.connect(**params) - except MySQLError, e: + except MySQLError as e: self.log.error('MySQLPerfCollector couldnt connect to database %s', e) return {} diff --git a/src/collectors/mysqlstat/mysqlstat.py b/src/collectors/mysqlstat/mysqlstat.py index 74969dae1..3fcd1db3a 100644 --- a/src/collectors/mysqlstat/mysqlstat.py +++ b/src/collectors/mysqlstat/mysqlstat.py @@ -296,7 +296,7 @@ def get_db_stats(self, query): try: cursor.execute(query) return cursor.fetchall() - except MySQLError, e: + except MySQLError as e: self.log.error('MySQLCollector could not get db stats', e) return () @@ -304,7 +304,7 @@ def connect(self, params): try: self.db = MySQLdb.connect(**params) self.log.debug('MySQLCollector: Connected to database.') - except MySQLError, e: + except MySQLError as e: self.log.error('MySQLCollector couldnt connect to database %s', e) return False return True @@ -403,7 +403,7 @@ def get_stats(self, params): for key in todo: self.log.debug("MySQLCollector: %s regexp not matched " + "in innodb status", key) - except Exception, innodb_status_error: + except Exception as innodb_status_error: self.log.error('MySQLCollector: Couldnt get engine innodb ' + 'status, check user permissions: %s', innodb_status_error) @@ -472,7 +472,7 @@ def collect(self): try: metrics = self.get_stats(params=params) - except Exception, e: + except Exception as e: try: self.disconnect() except MySQLdb.ProgrammingError: diff --git a/src/collectors/mysqlstat/test/testmysql.py b/src/collectors/mysqlstat/test/testmysql.py index 3ac98d1f4..138079813 100644 --- a/src/collectors/mysqlstat/test/testmysql.py +++ b/src/collectors/mysqlstat/test/testmysql.py @@ -6,8 +6,8 @@ from test import get_collector_config from test import unittest from test import run_only -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from mysqlstat import MySQLCollector diff --git a/src/collectors/nagios/nagios.py b/src/collectors/nagios/nagios.py index 61dcdc9eb..90371bba8 100644 --- a/src/collectors/nagios/nagios.py +++ b/src/collectors/nagios/nagios.py @@ -82,7 +82,10 @@ def collect(self): command.insert(0, self.config['sudo_cmd']) p = subprocess.Popen(command, - stdout=subprocess.PIPE).communicate()[0][:-1] + stdout=subprocess.PIPE).communicate()[0] + if isinstance(p, bytes): + p = p.decode("utf8") + p = p[:-1] for i, v in enumerate(p.split("\n")): metric_name = self.config['vars'][i] diff --git a/src/collectors/nagios/test/testnagios.py b/src/collectors/nagios/test/testnagios.py index b5878d64c..fca470745 100644 --- a/src/collectors/nagios/test/testnagios.py +++ b/src/collectors/nagios/test/testnagios.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from nagios import NagiosStatsCollector diff --git a/src/collectors/nagiosperfdata/nagiosperfdata.py b/src/collectors/nagiosperfdata/nagiosperfdata.py index 238bb7ee8..6079639e3 100644 --- a/src/collectors/nagiosperfdata/nagiosperfdata.py +++ b/src/collectors/nagiosperfdata/nagiosperfdata.py @@ -201,7 +201,7 @@ def _process_file(self, path): self._process_line(line) os.remove(path) - except IOError, ex: + except IOError as ex: self.log.error("Could not open file `{path}': {error}".format( path=path, error=ex.strerror)) diff --git a/src/collectors/nagiosperfdata/test/testnagiosperfdata.py b/src/collectors/nagiosperfdata/test/testnagiosperfdata.py index 101adc441..8d7a96cd8 100644 --- a/src/collectors/nagiosperfdata/test/testnagiosperfdata.py +++ b/src/collectors/nagiosperfdata/test/testnagiosperfdata.py @@ -3,7 +3,7 @@ from test import CollectorTestCase from test import get_collector_config -from mock import patch +from test import patch import os from diamond.collector import Collector diff --git a/src/collectors/netapp/netapp.py b/src/collectors/netapp/netapp.py index 414e92802..8287670dd 100644 --- a/src/collectors/netapp/netapp.py +++ b/src/collectors/netapp/netapp.py @@ -319,8 +319,8 @@ def collect(self, device, ip, user, password): res = server.invoke_elem(query) if(res.results_status() == "failed"): - print "Connection to filer %s failed; %s" % ( - device, res.results_reason()) + print("Connection to filer %s failed; %s" % ( + device, res.results_reason())) return num_records = res.child_get_int("records") diff --git a/src/collectors/netapp/netappDisk.py b/src/collectors/netapp/netappDisk.py index ef48f5d42..dd0dd51b6 100644 --- a/src/collectors/netapp/netappDisk.py +++ b/src/collectors/netapp/netappDisk.py @@ -273,7 +273,7 @@ def get_netapp_elem(self, netapp_api=None, sub_element=None): self.log.error( 'While using netapp API failed to retrieve ' 'disk-list-info for netapp filer %s' % self.device) - print netapp_data.sprintf() + print(netapp_data.sprintf()) return netapp_xml = \ ET.fromstring(netapp_data.sprintf()).find(sub_element) diff --git a/src/collectors/netstat/test/testnetstat.py b/src/collectors/netstat/test/testnetstat.py index 4c316ac94..261503a77 100644 --- a/src/collectors/netstat/test/testnetstat.py +++ b/src/collectors/netstat/test/testnetstat.py @@ -5,7 +5,7 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import patch +from test import patch from diamond.collector import Collector from netstat import NetstatCollector @@ -32,7 +32,7 @@ def test(self, publish_mock): self.setDocExample(collector=self.collector.__class__.__name__, metrics=metrics, defaultpath=self.collector.config['path']) - print publish_mock + print(publish_mock) self.assertPublishedMany(publish_mock, metrics) ################################################################################ diff --git a/src/collectors/network/network.py b/src/collectors/network/network.py index 366f5e070..f2ef034b8 100644 --- a/src/collectors/network/network.py +++ b/src/collectors/network/network.py @@ -13,6 +13,7 @@ import diamond.collector from diamond.collector import str_to_bool import diamond.convertor +from diamond.pycompat import long import os import re diff --git a/src/collectors/network/test/testnetwork.py b/src/collectors/network/test/testnetwork.py index 24d5873f5..ffb125675 100644 --- a/src/collectors/network/test/testnetwork.py +++ b/src/collectors/network/test/testnetwork.py @@ -5,13 +5,10 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from test import Mock +from test import patch +from test import StringIO +from test import BUILTIN_OPEN from diamond.collector import Collector from network import NetworkCollector @@ -33,7 +30,7 @@ def setUp(self): def test_import(self): self.assertTrue(NetworkCollector) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch('os.access', Mock(return_value=True)) @patch.object(Collector, 'publish') def test_should_open_proc_net_dev(self, publish_mock, open_mock): diff --git a/src/collectors/nfacct/test/testnfacct.py b/src/collectors/nfacct/test/testnfacct.py index 97d98731c..7576d681d 100644 --- a/src/collectors/nfacct/test/testnfacct.py +++ b/src/collectors/nfacct/test/testnfacct.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from nfacct import NetfilterAccountingCollector diff --git a/src/collectors/nfs/nfs.py b/src/collectors/nfs/nfs.py index ac8f12ad2..49948aa05 100644 --- a/src/collectors/nfs/nfs.py +++ b/src/collectors/nfs/nfs.py @@ -11,6 +11,7 @@ import diamond.collector import os +from diamond.pycompat import long class NfsCollector(diamond.collector.Collector): diff --git a/src/collectors/nfs/test/testnfs.py b/src/collectors/nfs/test/testnfs.py index 55b9c0078..62e7f2ffa 100644 --- a/src/collectors/nfs/test/testnfs.py +++ b/src/collectors/nfs/test/testnfs.py @@ -5,13 +5,10 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from test import Mock +from test import patch +from test import StringIO +from test import BUILTIN_OPEN from diamond.collector import Collector from nfs import NfsCollector @@ -31,7 +28,7 @@ def setUp(self): def test_import(self): self.assertTrue(NfsCollector) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch('os.access', Mock(return_value=True)) @patch.object(Collector, 'publish') def test_should_open_proc_stat(self, publish_mock, open_mock): diff --git a/src/collectors/nfsd/nfsd.py b/src/collectors/nfsd/nfsd.py index 63d4b26f3..7382fa021 100644 --- a/src/collectors/nfsd/nfsd.py +++ b/src/collectors/nfsd/nfsd.py @@ -10,6 +10,7 @@ """ import diamond.collector +from diamond.pycompat import long import os diff --git a/src/collectors/nfsd/test/testnfsd.py b/src/collectors/nfsd/test/testnfsd.py index f4de04989..36fd8bc64 100644 --- a/src/collectors/nfsd/test/testnfsd.py +++ b/src/collectors/nfsd/test/testnfsd.py @@ -5,13 +5,10 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from test import Mock +from test import patch +from test import StringIO +from test import BUILTIN_OPEN from diamond.collector import Collector from nfsd import NfsdCollector @@ -31,7 +28,7 @@ def setUp(self): def test_import(self): self.assertTrue(NfsdCollector) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch('os.access', Mock(return_value=True)) @patch.object(Collector, 'publish') def test_should_open_proc_stat(self, publish_mock, open_mock): diff --git a/src/collectors/nginx/nginx.py b/src/collectors/nginx/nginx.py index 6e843f64b..9e1bef749 100644 --- a/src/collectors/nginx/nginx.py +++ b/src/collectors/nginx/nginx.py @@ -5,7 +5,6 @@ #### Dependencies - * urllib2 * json #### Usage @@ -55,12 +54,12 @@ """ - -import urllib2 import re import diamond.collector import json +import diamond.pycompat + class NginxCollector(diamond.collector.Collector): @@ -217,9 +216,9 @@ def collect(self): int(self.config['req_port']), self.config['req_path']) - req = urllib2.Request(url=url, headers=headers) + req = diamond.pycompat.Request(url=url, headers=headers) try: - handle = urllib2.urlopen(req) + handle = diamond.pycompat.urlopen(req) # Test for json payload; indicates nginx+ if handle.info().gettype() == 'application/json': @@ -228,8 +227,7 @@ def collect(self): # Plain payload; indicates open source nginx else: self.collect_nginx(handle) - - except IOError, e: + except IOError as e: self.log.error("Unable to open %s" % url) - except Exception, e: + except Exception as e: self.log.error("Unknown error opening url: %s", e) diff --git a/src/collectors/nginx/test/testnginx.py b/src/collectors/nginx/test/testnginx.py index 4c015fb88..f01404a69 100644 --- a/src/collectors/nginx/test/testnginx.py +++ b/src/collectors/nginx/test/testnginx.py @@ -5,10 +5,11 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector +from diamond.pycompat import URLOPEN from nginx import NginxCollector @@ -38,7 +39,7 @@ def test_should_work_with_real_data(self, publish_counter_mock, } ) - patch_urlopen = patch('urllib2.urlopen', Mock( + patch_urlopen = patch(URLOPEN, Mock( return_value=mockResponse)) patch_urlopen.start() @@ -78,7 +79,7 @@ def test_plus_should_work_with_real_data(self, publish_counter_mock, } ) - patch_urlopen = patch('urllib2.urlopen', Mock( + patch_urlopen = patch(URLOPEN, Mock( return_value=mockResponse)) patch_urlopen.start() @@ -179,7 +180,7 @@ def test_should_fail_gracefully(self, publish_mock): } ) - patch_urlopen = patch('urllib2.urlopen', Mock( + patch_urlopen = patch(URLOPEN, Mock( return_value=mockResponse)) patch_urlopen.start() diff --git a/src/collectors/ntp/test/testntp.py b/src/collectors/ntp/test/testntp.py index 60120f65e..4366698a8 100644 --- a/src/collectors/ntp/test/testntp.py +++ b/src/collectors/ntp/test/testntp.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector diff --git a/src/collectors/ntpd/ntpd.py b/src/collectors/ntpd/ntpd.py index bd66b4944..c120e562a 100644 --- a/src/collectors/ntpd/ntpd.py +++ b/src/collectors/ntpd/ntpd.py @@ -46,8 +46,10 @@ def run_command(self, command): if str_to_bool(self.config['use_sudo']): command.insert(0, self.config['sudo_cmd']) - return subprocess.Popen(command, - stdout=subprocess.PIPE).communicate()[0] + stdout = subprocess.Popen(command, + stdout=subprocess.PIPE).communicate()[0] + if isinstance(stdout, bytes): + stdout = stdout.decode("utf8") except OSError: self.log.exception("Unable to run %s", command) return "" diff --git a/src/collectors/ntpd/test/testntpd.py b/src/collectors/ntpd/test/testntpd.py index 7fe3070e2..dbfd75ef4 100644 --- a/src/collectors/ntpd/test/testntpd.py +++ b/src/collectors/ntpd/test/testntpd.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector diff --git a/src/collectors/numa/numa.py b/src/collectors/numa/numa.py index 0ffeeadba..9e58d441b 100644 --- a/src/collectors/numa/numa.py +++ b/src/collectors/numa/numa.py @@ -31,9 +31,9 @@ def get_default_config(self): def collect(self): p = Popen([self.config['bin'], '--hardware'], stdout=PIPE, stderr=PIPE) - output, errors = p.communicate() - + if isinstance(output, bytes): + output = output.decode("utf8") lines = output.split('\n') for line in lines: try: diff --git a/src/collectors/numa/test/testnuma.py b/src/collectors/numa/test/testnuma.py index 9b12afaa3..b56bddef4 100644 --- a/src/collectors/numa/test/testnuma.py +++ b/src/collectors/numa/test/testnuma.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import patch -from mock import Mock +from test import patch +from test import Mock from diamond.collector import Collector from numa import NumaCollector diff --git a/src/collectors/nvidia_gpu/nvidia_gpu.py b/src/collectors/nvidia_gpu/nvidia_gpu.py index 2f1cfb96b..5371ae99d 100644 --- a/src/collectors/nvidia_gpu/nvidia_gpu.py +++ b/src/collectors/nvidia_gpu/nvidia_gpu.py @@ -11,7 +11,6 @@ * nvidia-ml-py (Optional) """ -from itertools import izip try: import pynvml USE_PYTHON_BINDING = True @@ -71,7 +70,7 @@ def collect_via_nvidia_smi(self, stats_config): stats = result.strip().split(',') assert len(stats) == len(stats_config) index = stats[0] - for stat_name, metric in izip(stats_config[1:], stats[1:]): + for stat_name, metric in zip(stats_config[1:], stats[1:]): metric_name = 'gpu_{index}.{stat_name}'.format( index=str(index), stat_name=stat_name diff --git a/src/collectors/nvidia_gpu/test/testnvidia_gpu.py b/src/collectors/nvidia_gpu/test/testnvidia_gpu.py index 3c64123b4..4a10e5733 100644 --- a/src/collectors/nvidia_gpu/test/testnvidia_gpu.py +++ b/src/collectors/nvidia_gpu/test/testnvidia_gpu.py @@ -5,7 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import patch, Mock +from test import patch +from test import Mock from diamond.collector import Collector from nvidia_gpu import NvidiaGPUCollector diff --git a/src/collectors/onewire/onewire.py b/src/collectors/onewire/onewire.py index d3c40d6fd..a0b3553f7 100644 --- a/src/collectors/onewire/onewire.py +++ b/src/collectors/onewire/onewire.py @@ -58,11 +58,11 @@ def collect(self): if '.' in ld: self.read_values(ld, self.config['scan'], metrics) - for oid, files in self.config.iteritems(): + for oid, files in self.config.items(): if oid[:3] == 'id:': self.read_values(oid[3:], files, metrics) - for fn, fv in metrics.iteritems(): + for fn, fv in metrics.items(): self.publish(fn, fv, 2) def read_values(self, oid, files, metrics): @@ -74,7 +74,7 @@ def read_values(self, oid, files, metrics): oid_path = os.path.join(self.config['owfs'], oid) oid = oid.replace('.', '_') - for fn, alias in files.iteritems(): + for fn, alias in files.items(): fv = os.path.join(oid_path, fn) if os.path.isfile(fv): try: diff --git a/src/collectors/onewire/test/testonewire.py b/src/collectors/onewire/test/testonewire.py index 597125a44..7b5a90e32 100644 --- a/src/collectors/onewire/test/testonewire.py +++ b/src/collectors/onewire/test/testonewire.py @@ -6,7 +6,7 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import patch +from test import patch from diamond.collector import Collector from onewire import OneWireCollector diff --git a/src/collectors/openldap/openldap.py b/src/collectors/openldap/openldap.py index 9c972e72c..0a24b18e1 100644 --- a/src/collectors/openldap/openldap.py +++ b/src/collectors/openldap/openldap.py @@ -160,7 +160,7 @@ def collect(self): datapoints = self.get_datapoints(ldap_url, self.config['username'], self.config['password']) - except Exception, e: + except Exception as e: self.log.error('Unable to query %s: %s' % (ldap_url, e)) return {} diff --git a/src/collectors/openstackswiftrecon/openstackswiftrecon.py b/src/collectors/openstackswiftrecon/openstackswiftrecon.py index 51ad82a60..278ecf943 100644 --- a/src/collectors/openstackswiftrecon/openstackswiftrecon.py +++ b/src/collectors/openstackswiftrecon/openstackswiftrecon.py @@ -50,7 +50,7 @@ def get_default_config(self): def _process_cache(self, d, path=()): """Recusively walk a nested recon cache dict to obtain path/values""" - for k, v in d.iteritems(): + for k, v in d.items(): if not isinstance(v, dict): self.metrics.append((path + (k,), v)) else: diff --git a/src/collectors/openstackswiftrecon/test/testopenstackswiftrecon.py b/src/collectors/openstackswiftrecon/test/testopenstackswiftrecon.py index e98e7aa81..3bcd67a8e 100644 --- a/src/collectors/openstackswiftrecon/test/testopenstackswiftrecon.py +++ b/src/collectors/openstackswiftrecon/test/testopenstackswiftrecon.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest - -from mock import patch, Mock +from test import BUILTIN_OPEN +from test import patch, Mock from diamond.collector import Collector from openstackswiftrecon import OpenstackSwiftReconCollector @@ -29,7 +29,7 @@ def setUp(self, allowed_names=None): def test_import(self): self.assertTrue(OpenstackSwiftReconCollector) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch('os.access', Mock(return_value=False)) @patch.object(Collector, 'publish') def test_recon_no_access(self, publish_mock, open_mock): diff --git a/src/collectors/openvpn/openvpn.py b/src/collectors/openvpn/openvpn.py index 68c5584a9..1b0ee92a3 100644 --- a/src/collectors/openvpn/openvpn.py +++ b/src/collectors/openvpn/openvpn.py @@ -27,18 +27,13 @@ You can also specify multiple and mixed instances:: instances = file:///var/log/openvpn/openvpn.log, tcp://10.0.0.1:1195?admins - -#### Dependencies - - * urlparse - """ import socket import diamond.collector import os.path -import urlparse import time +from diamond.pycompat import long, urlparse class OpenVPNCollector(diamond.collector.Collector): @@ -67,7 +62,7 @@ def parse_url(self, uri): """ Convert urlparse from a python 2.4 layout to a python 2.7 layout """ - parsed = urlparse.urlparse(uri) + parsed = urlparse(uri) if 'scheme' not in parsed: class Object(object): pass @@ -86,7 +81,7 @@ class Object(object): return parsed def collect(self): - if isinstance(self.config['instances'], basestring): + if isinstance(self.config['instances'], str): instances = [self.config['instances']] else: instances = self.config['instances'] @@ -165,7 +160,7 @@ def collect_tcp(self, uri): # Bye server.close() - except socket.error, e: + except socket.error as e: self.log.error('OpenVPN management connection error: %s', e) return diff --git a/src/collectors/openvpn/test/testopenvpn.py b/src/collectors/openvpn/test/testopenvpn.py index d496a0a92..1a8e38d36 100644 --- a/src/collectors/openvpn/test/testopenvpn.py +++ b/src/collectors/openvpn/test/testopenvpn.py @@ -5,7 +5,7 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import patch +from test import patch from diamond.collector import Collector from openvpn import OpenVPNCollector diff --git a/src/collectors/openvz/test/testopenvz.py b/src/collectors/openvz/test/testopenvz.py index a008e8153..8f081b5b0 100644 --- a/src/collectors/openvz/test/testopenvz.py +++ b/src/collectors/openvz/test/testopenvz.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector diff --git a/src/collectors/ossec/ossec.py b/src/collectors/ossec/ossec.py index 05c9e314a..09d078260 100644 --- a/src/collectors/ossec/ossec.py +++ b/src/collectors/ossec/ossec.py @@ -54,7 +54,9 @@ def collect(self): try: p = subprocess.Popen(command, stdout=subprocess.PIPE) res = p.communicate()[0] - except Exception, e: + if isinstance(res, bytes): + res = res.decode("utf8") + except Exception as e: self.log.error('Unable to exec cmd: %s, because %s' % (' '.join(command), str(e))) return diff --git a/src/collectors/passenger_stats/passenger_stats.py b/src/collectors/passenger_stats/passenger_stats.py index 82ba557ed..52d0a4114 100644 --- a/src/collectors/passenger_stats/passenger_stats.py +++ b/src/collectors/passenger_stats/passenger_stats.py @@ -77,7 +77,9 @@ def get_passenger_memory_stats(self): try: proc1 = subprocess.Popen(command, stdout=subprocess.PIPE) - (std_out, std_err) = proc1.communicate() + std_out = proc1.communicate()[0] + if isinstance(std_out, bytes): + std_out = std_out.decode("utf8") except OSError: return {} @@ -135,7 +137,9 @@ def get_passenger_cpu_usage(self, dict_stats): proc1 = subprocess.Popen( ["top", "-b", "-n", "2"], stdout=subprocess.PIPE) - (std_out, std_err) = proc1.communicate() + std_out = proc1.communicate()[0] + if isinstance(std_out, bytes): + std_out = std_out.decode("utf8") except OSError: return (-1) diff --git a/src/collectors/pgbouncer/pgbouncer.py b/src/collectors/pgbouncer/pgbouncer.py index 2c174c0a8..05ee80940 100644 --- a/src/collectors/pgbouncer/pgbouncer.py +++ b/src/collectors/pgbouncer/pgbouncer.py @@ -83,15 +83,15 @@ def collect(self): } } - for name, instance in instances.iteritems(): + for name, instance in instances.items(): host = instance['host'] port = instance['port'] user = instance.get('user') or self.config['user'] password = instance.get('password') or self.config['password'] for database, stats in self._get_stats_by_database( - host, port, user, password).iteritems(): - for stat_name, stat_value in stats.iteritems(): + host, port, user, password).items(): + for stat_name, stat_value in stats.items(): self.publish( self._get_metric_name(name, database, stat_name), stat_value) diff --git a/src/collectors/pgbouncer/test/testpgbouncer.py b/src/collectors/pgbouncer/test/testpgbouncer.py index 15b963de1..3ea799e18 100644 --- a/src/collectors/pgbouncer/test/testpgbouncer.py +++ b/src/collectors/pgbouncer/test/testpgbouncer.py @@ -6,7 +6,7 @@ from test import get_collector_config from test import run_only from test import unittest -from mock import patch +from test import patch from pgbouncer import PgbouncerCollector diff --git a/src/collectors/pgq/pgq.py b/src/collectors/pgq/pgq.py index ad3e8358d..4e5a36c72 100644 --- a/src/collectors/pgq/pgq.py +++ b/src/collectors/pgq/pgq.py @@ -52,7 +52,7 @@ def collect(self): self.log.error('Unable to import module psycopg2') return None - for instance, configuration in self.config['instances'].iteritems(): + for instance, configuration in self.config['instances'].items(): connection = psycopg2.connect(configuration['dsn']) connection.set_isolation_level( psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT, diff --git a/src/collectors/pgq/test/testpgq.py b/src/collectors/pgq/test/testpgq.py index 862760251..40014376f 100644 --- a/src/collectors/pgq/test/testpgq.py +++ b/src/collectors/pgq/test/testpgq.py @@ -1,6 +1,7 @@ #!/usr/bin/python from test import CollectorTestCase, get_collector_config -from mock import MagicMock, patch +from test import patch +from test import MagicMock from pgq import PgQCollector diff --git a/src/collectors/phpfpm/phpfpm.py b/src/collectors/phpfpm/phpfpm.py index 636535cb5..9d283645b 100644 --- a/src/collectors/phpfpm/phpfpm.py +++ b/src/collectors/phpfpm/phpfpm.py @@ -26,8 +26,7 @@ #### Dependencies - * urllib2 - * json (or simeplejson) + * json (or simplejson) """ @@ -36,8 +35,8 @@ except ImportError: import simplejson as json -import urllib2 import diamond.collector +import diamond.pycompat class PhpFpmCollector(diamond.collector.Collector): @@ -72,16 +71,16 @@ def collect(self): self.config['uri'] = self.config['uri'][1:] try: - response = urllib2.urlopen("http://%s:%s/%s?json" % ( + response = diamond.pycompat.urlopen("http://%s:%s/%s?json" % ( self.config['host'], int(self.config['port']), self.config['uri'])) - except Exception, e: + except Exception as e: self.log.error('Couldnt connect to php-fpm status page: %s', e) return {} try: j = json.loads(response.read()) - except Exception, e: + except Exception as e: self.log.error('Couldnt parse json: %s', e) return {} diff --git a/src/collectors/phpfpm/test/testphpfpm.py b/src/collectors/phpfpm/test/testphpfpm.py index ab8dea909..b07158b6a 100644 --- a/src/collectors/phpfpm/test/testphpfpm.py +++ b/src/collectors/phpfpm/test/testphpfpm.py @@ -5,10 +5,11 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector +from diamond.pycompat import URLOPEN from phpfpm import PhpFpmCollector ########################################################################## @@ -28,7 +29,7 @@ def test_import(self): @patch.object(Collector, 'publish') def test_should_work_with_real_data(self, publish_mock): - patch_urlopen = patch('urllib2.urlopen', Mock( + patch_urlopen = patch(URLOPEN, Mock( return_value=self.getFixture('stats'))) patch_urlopen.start() @@ -52,7 +53,7 @@ def test_should_work_with_real_data(self, publish_mock): @patch.object(Collector, 'publish') def test_should_fail_gracefully(self, publish_mock): - patch_urlopen = patch('urllib2.urlopen', Mock( + patch_urlopen = patch(URLOPEN, Mock( return_value=self.getFixture('stats_blank'))) patch_urlopen.start() diff --git a/src/collectors/ping/test/testping.py b/src/collectors/ping/test/testping.py index 7a529d3f4..574ec8dc9 100644 --- a/src/collectors/ping/test/testping.py +++ b/src/collectors/ping/test/testping.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from ping import PingCollector diff --git a/src/collectors/portstat/portstat.py b/src/collectors/portstat/portstat.py index 634a0ed60..8a7f23b27 100644 --- a/src/collectors/portstat/portstat.py +++ b/src/collectors/portstat/portstat.py @@ -66,10 +66,10 @@ def collect(self): self.log.error('Unable to import module psutil') return {} - for port_name, port_cfg in self.ports.iteritems(): + for port_name, port_cfg in self.ports.items(): port = int(port_cfg['number']) stats = get_port_stats(port) - for stat_name, stat_value in stats.iteritems(): + for stat_name, stat_value in stats.items(): metric_name = '%s.%s' % (port_name, stat_name) self.publish(metric_name, stat_value) diff --git a/src/collectors/portstat/tests/test_portstat.py b/src/collectors/portstat/tests/test_portstat.py index 187b52789..992d2b975 100644 --- a/src/collectors/portstat/tests/test_portstat.py +++ b/src/collectors/portstat/tests/test_portstat.py @@ -1,7 +1,9 @@ from test import CollectorTestCase from test import get_collector_config +from test import patch +from test import Mock +from test import call -from mock import call, Mock, patch from unittest import TestCase from diamond.collector import Collector diff --git a/src/collectors/postfix/postfix.py b/src/collectors/postfix/postfix.py index 16bbbea6d..1c16eb202 100644 --- a/src/collectors/postfix/postfix.py +++ b/src/collectors/postfix/postfix.py @@ -101,7 +101,7 @@ def collect(self): return if str_to_bool(self.config['include_clients']) and u'clients' in data: - for client, value in data['clients'].iteritems(): + for client, value in data['clients'].items(): # translate dots to underscores in client names metric = u'.'.join(['clients', client.translate(DOTS_TO_UNDERS)]) @@ -114,8 +114,8 @@ def collect(self): if action not in data: continue - for sect, stats in data[action].iteritems(): - for status, value in stats.iteritems(): + for sect, stats in data[action].items(): + for status, value in stats.items(): metric = '.'.join([action, sect, status.translate(DOTS_TO_UNDERS)]) @@ -125,7 +125,7 @@ def collect(self): self.publish(metric, dvalue, precision=4) if u'local' in data: - for key, value in data[u'local'].iteritems(): + for key, value in data[u'local'].items(): metric = '.'.join(['local', key]) dvalue = self.derivative(metric, value) diff --git a/src/collectors/postfix/test/testpostfix.py b/src/collectors/postfix/test/testpostfix.py index 8dd7bd5a5..b0511d386 100644 --- a/src/collectors/postfix/test/testpostfix.py +++ b/src/collectors/postfix/test/testpostfix.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from postfix import PostfixCollector diff --git a/src/collectors/postgres/postgres.py b/src/collectors/postgres/postgres.py index a2c19db5d..b9fd5aca8 100644 --- a/src/collectors/postgres/postgres.py +++ b/src/collectors/postgres/postgres.py @@ -170,7 +170,7 @@ def _connect(self, database=None): try: conn = psycopg2.connect(**conn_args) - except Exception, e: + except Exception as e: self.log.error(e) raise e @@ -220,7 +220,7 @@ def fetch(self, pg_version): # If row > length 2, assume each column name maps to # key => value else: - for key, value in row.iteritems(): + for key, value in row.items(): if key in ('datname', 'schemaname', 'relname', 'indexrelname', 'funcname',): continue diff --git a/src/collectors/postgres/test/testpostgres.py b/src/collectors/postgres/test/testpostgres.py index 866145196..fa124ae25 100644 --- a/src/collectors/postgres/test/testpostgres.py +++ b/src/collectors/postgres/test/testpostgres.py @@ -4,8 +4,8 @@ from test import CollectorTestCase from test import get_collector_config - -from mock import patch, Mock +from test import patch +from test import Mock from postgres import PostgresqlCollector diff --git a/src/collectors/postqueue/postqueue.py b/src/collectors/postqueue/postqueue.py index ab1a8fc78..f133185bb 100644 --- a/src/collectors/postqueue/postqueue.py +++ b/src/collectors/postqueue/postqueue.py @@ -45,8 +45,11 @@ def get_postqueue_output(self): if str_to_bool(self.config['use_sudo']): command.insert(0, self.config['sudo_cmd']) - return subprocess.Popen(command, - stdout=subprocess.PIPE).communicate()[0] + stdout = subprocess.Popen(command, + stdout=subprocess.PIPE).communicate()[0] + if isinstance(stdout, bytes): + stdout = stdout.decode("utf8") + return stdout except OSError: return "" diff --git a/src/collectors/postqueue/test/testpostqueue.py b/src/collectors/postqueue/test/testpostqueue.py index b667b051b..95b5a411c 100644 --- a/src/collectors/postqueue/test/testpostqueue.py +++ b/src/collectors/postqueue/test/testpostqueue.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from postqueue import PostqueueCollector diff --git a/src/collectors/powerdns/powerdns.py b/src/collectors/powerdns/powerdns.py index af7991a1f..a10cb639a 100644 --- a/src/collectors/powerdns/powerdns.py +++ b/src/collectors/powerdns/powerdns.py @@ -60,6 +60,8 @@ def collect(self): data = subprocess.Popen(command, stdout=subprocess.PIPE).communicate()[0] + if isinstance(data, bytes): + data = data.decode("utf8") for metric in data.split(','): if not metric.strip(): diff --git a/src/collectors/powerdns/test/testpowerdns.py b/src/collectors/powerdns/test/testpowerdns.py index 5a9c173fd..c0815deb7 100644 --- a/src/collectors/powerdns/test/testpowerdns.py +++ b/src/collectors/powerdns/test/testpowerdns.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from powerdns import PowerDNSCollector diff --git a/src/collectors/proc/proc.py b/src/collectors/proc/proc.py index 4ee9444ba..d224efda6 100644 --- a/src/collectors/proc/proc.py +++ b/src/collectors/proc/proc.py @@ -13,7 +13,7 @@ import platform import os import diamond.collector - +from diamond.pycompat import long # Detect the architecture of the system # and set the counters for MAX_VALUES # appropriately. Otherwise, rolling over diff --git a/src/collectors/proc/test/testproc.py b/src/collectors/proc/test/testproc.py index 997d9ef6f..dc91e3bef 100644 --- a/src/collectors/proc/test/testproc.py +++ b/src/collectors/proc/test/testproc.py @@ -5,13 +5,10 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from test import Mock +from test import patch +from test import StringIO +from test import BUILTIN_OPEN from diamond.collector import Collector from proc import ProcessStatCollector @@ -31,7 +28,7 @@ def setUp(self): def test_import(self): self.assertTrue(ProcessStatCollector) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch('os.access', Mock(return_value=True)) @patch.object(Collector, 'publish') def test_should_open_proc_stat(self, publish_mock, open_mock): diff --git a/src/collectors/processresources/processresources.py b/src/collectors/processresources/processresources.py index 618a43dc5..035017ede 100644 --- a/src/collectors/processresources/processresources.py +++ b/src/collectors/processresources/processresources.py @@ -81,7 +81,7 @@ def process_info(process, info_keys): if type(value) in [float, int]: results.update({key: value}) elif hasattr(value, '_asdict'): - for subkey, subvalue in value._asdict().iteritems(): + for subkey, subvalue in value._asdict().items(): results.update({"%s.%s" % (key, subkey): subvalue}) return results @@ -158,7 +158,7 @@ def get_default_config(self): return config def save_process_info(self, pg_name, process_info): - for key, value in process_info.iteritems(): + for key, value in process_info.items(): if key in self.processes_info[pg_name]: self.processes_info[pg_name][key] += value else: @@ -181,7 +181,7 @@ def collect_process_info(self, process): uptime = time.time() - get_value(process, 'create_time') pi.update({'uptime': uptime}) self.save_process_info(pg_name, pi) - except psutil.NoSuchProcess, e: + except psutil.NoSuchProcess as e: self.log.info("Process exited while trying to get info: %s", e) def collect(self): @@ -198,11 +198,11 @@ def collect(self): self.collect_process_info(process) # publish results - for pg_name, counters in self.processes_info.iteritems(): + for pg_name, counters in self.processes_info.items(): if counters: metrics = ( ("%s.%s" % (pg_name, key), value) - for key, value in counters.iteritems()) + for key, value in counters.items()) else: if self.processes[pg_name]['count_workers']: metrics = (('%s.workers_count' % pg_name, 0), ) diff --git a/src/collectors/processresources/test/testprocessresources.py b/src/collectors/processresources/test/testprocessresources.py index 7e4e14316..4c6b7e320 100644 --- a/src/collectors/processresources/test/testprocessresources.py +++ b/src/collectors/processresources/test/testprocessresources.py @@ -8,7 +8,7 @@ from test import get_collector_config from test import unittest from test import run_only -from mock import patch +from test import patch from diamond.collector import Collector from processresources import ProcessResourcesCollector diff --git a/src/collectors/puppetagent/puppetagent.py b/src/collectors/puppetagent/puppetagent.py index 579cc0ece..d4c117495 100644 --- a/src/collectors/puppetagent/puppetagent.py +++ b/src/collectors/puppetagent/puppetagent.py @@ -55,9 +55,9 @@ def collect(self): summary = self._get_summary() - for sect, data in summary.iteritems(): - for stat, value in data.iteritems(): - if value is None or isinstance(value, basestring): + for sect, data in summary.items(): + for stat, value in data.items(): + if value is None or isinstance(value, str): continue metric = '.'.join([sect, stat]) diff --git a/src/collectors/puppetagent/test/testpuppetagent.py b/src/collectors/puppetagent/test/testpuppetagent.py index 5aa339caa..d22c77cff 100644 --- a/src/collectors/puppetagent/test/testpuppetagent.py +++ b/src/collectors/puppetagent/test/testpuppetagent.py @@ -6,7 +6,7 @@ from test import get_collector_config from test import unittest from test import run_only -from mock import patch +from test import patch from diamond.collector import Collector from puppetagent import PuppetAgentCollector diff --git a/src/collectors/puppetdashboard/puppetdashboard.py b/src/collectors/puppetdashboard/puppetdashboard.py index 81ca10804..2ed44c5e0 100644 --- a/src/collectors/puppetdashboard/puppetdashboard.py +++ b/src/collectors/puppetdashboard/puppetdashboard.py @@ -2,16 +2,11 @@ """ Collect metrics from Puppet Dashboard - -#### Dependencies - - * urllib2 - """ -import urllib2 import re import diamond.collector +import diamond.pycompat class PuppetDashboardCollector(diamond.collector.Collector): @@ -40,9 +35,9 @@ def get_default_config(self): def collect(self): try: - response = urllib2.urlopen("http://%s:%s/" % ( + response = diamond.pycompat.urlopen("http://%s:%s/" % ( self.config['host'], int(self.config['port']))) - except Exception, e: + except Exception as e: self.log.error('Couldnt connect to puppet-dashboard: %s', e) return {} @@ -59,5 +54,5 @@ def collect(self): results = r.groupdict() self.publish(results['key'], results['count']) - except Exception, e: + except Exception as e: self.log.error('Couldnt parse the output: %s', e) diff --git a/src/collectors/puppetdashboard/test/testpuppetdashboard.py b/src/collectors/puppetdashboard/test/testpuppetdashboard.py index 04fcf7dad..54a7f3f90 100644 --- a/src/collectors/puppetdashboard/test/testpuppetdashboard.py +++ b/src/collectors/puppetdashboard/test/testpuppetdashboard.py @@ -5,10 +5,11 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector +from diamond.pycompat import URLOPEN from puppetdashboard import PuppetDashboardCollector ########################################################################## @@ -28,7 +29,7 @@ def test_import(self): @patch.object(Collector, 'publish') def test_should_work_with_real_data(self, publish_mock): - patch_urlopen = patch('urllib2.urlopen', Mock( + patch_urlopen = patch(URLOPEN, Mock( return_value=self.getFixture('index.html'))) patch_urlopen.start() @@ -50,7 +51,7 @@ def test_should_work_with_real_data(self, publish_mock): @patch.object(Collector, 'publish') def test_should_fail_gracefully(self, publish_mock): - patch_urlopen = patch('urllib2.urlopen', Mock( + patch_urlopen = patch(URLOPEN, Mock( return_value=self.getFixture('index.blank'))) patch_urlopen.start() diff --git a/src/collectors/puppetdb/puppetdb.py b/src/collectors/puppetdb/puppetdb.py index b9ad400ce..eeb7894d5 100644 --- a/src/collectors/puppetdb/puppetdb.py +++ b/src/collectors/puppetdb/puppetdb.py @@ -5,14 +5,13 @@ #### Dependencies - * urllib2 * json """ -import urllib2 import diamond.collector from diamond.convertor import time as time_convertor +import diamond.pycompat try: import json @@ -90,8 +89,8 @@ def fetch_metrics(self, url): try: url = "http://%s:%s/%s" % ( self.config['host'], int(self.config['port']), url) - response = urllib2.urlopen(url) - except Exception, e: + response = diamond.pycompat.urlopen(url) + except Exception as e: self.log.error('Couldn\'t connect to puppetdb: %s -> %s', url, e) return {} return json.load(response) diff --git a/src/collectors/rabbitmq/rabbitmq.py b/src/collectors/rabbitmq/rabbitmq.py index 8084fa7d4..d671ae1f2 100644 --- a/src/collectors/rabbitmq/rabbitmq.py +++ b/src/collectors/rabbitmq/rabbitmq.py @@ -22,10 +22,9 @@ """ import diamond.collector +import diamond.pycompat +from diamond.pycompat import quote, Request, urljoin import re -from urlparse import urljoin -from urllib import quote -import urllib2 from base64 import b64encode try: @@ -47,9 +46,9 @@ def __init__(self, log, host, user, password, timeout=5, scheme="http"): def do_call(self, path): url = urljoin(self.base_url, path) - req = urllib2.Request(url) + req = Request(url) req.add_header('Authorization', self._authorization) - return json.load(urllib2.urlopen(req, timeout=self.timeout)) + return json.load(diamond.pycompat.urlopen(req, timeout=self.timeout)) def get_all_vhosts(self): return self.do_call('vhosts') @@ -66,7 +65,7 @@ def get_queue(self, vhost, queue_name): try: queue = self.do_call(path) return queue or None - except Exception, e: + except Exception as e: self.log.error('Error querying queue %s/%s: %s' % ( vhost, queue_name, e )) @@ -80,7 +79,7 @@ def get_queues(self, vhost): try: queues = self.do_call(path) return queues or [] - except Exception, e: + except Exception as e: self.log.error('Error querying queues %s: %s' % ( vhost, e )) diff --git a/src/collectors/rabbitmq/test/testrabbitmq.py b/src/collectors/rabbitmq/test/testrabbitmq.py index f5006a590..d1329e099 100644 --- a/src/collectors/rabbitmq/test/testrabbitmq.py +++ b/src/collectors/rabbitmq/test/testrabbitmq.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from mock import call from diamond.collector import Collector diff --git a/src/collectors/redisstat/redisstat.py b/src/collectors/redisstat/redisstat.py index 35fb8304d..a8d1f91c4 100644 --- a/src/collectors/redisstat/redisstat.py +++ b/src/collectors/redisstat/redisstat.py @@ -110,7 +110,7 @@ def process_config(self): super(RedisCollector, self).process_config() instance_list = self.config['instances'] # configobj make str of single-element list, let's convert - if isinstance(instance_list, basestring): + if isinstance(instance_list, str): instance_list = [instance_list] # process original single redis instance @@ -221,7 +221,7 @@ def _client(self, host, port, unix_socket, auth): unix_socket_path=unix_socket) cli.ping() return cli - except Exception, ex: + except Exception as ex: self.log.error("RedisCollector: failed to connect to %s:%i. %s.", unix_socket or host, port, ex) diff --git a/src/collectors/redisstat/test/testredisstat.py b/src/collectors/redisstat/test/testredisstat.py index 987473282..61fa3dbbf 100644 --- a/src/collectors/redisstat/test/testredisstat.py +++ b/src/collectors/redisstat/test/testredisstat.py @@ -6,8 +6,9 @@ from test import get_collector_config from test import unittest from test import run_only -from mock import Mock -from mock import patch, call +from test import Mock +from test import patch +from mock import call from diamond.collector import Collector from redisstat import RedisCollector diff --git a/src/collectors/resqueweb/resqueweb.py b/src/collectors/resqueweb/resqueweb.py index 8258d0d6c..683a2f8c0 100644 --- a/src/collectors/resqueweb/resqueweb.py +++ b/src/collectors/resqueweb/resqueweb.py @@ -2,15 +2,10 @@ """ Collects data for Resque Web - -#### Dependencies - - * urllib2 - """ -import urllib2 import diamond.collector +import diamond.pycompat class ResqueWebCollector(diamond.collector.Collector): @@ -35,9 +30,9 @@ def get_default_config(self): def collect(self): try: - response = urllib2.urlopen("http://%s:%s/stats.txt" % ( + response = diamond.pycompat.urlopen("http://%s:%s/stats.txt" % ( self.config['host'], int(self.config['port']))) - except Exception, e: + except Exception as e: self.log.error('Couldnt connect to resque-web: %s', e) return {} @@ -59,5 +54,5 @@ def collect(self): else: self.publish("queue.%s.current" % queue, count) - except Exception, e: + except Exception as e: self.log.error('Couldnt parse the queue: %s', e) diff --git a/src/collectors/resqueweb/test/testresqueweb.py b/src/collectors/resqueweb/test/testresqueweb.py index 5b492756f..7b37eb584 100644 --- a/src/collectors/resqueweb/test/testresqueweb.py +++ b/src/collectors/resqueweb/test/testresqueweb.py @@ -5,10 +5,11 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector +from diamond.pycompat import URLOPEN from resqueweb import ResqueWebCollector ########################################################################## @@ -28,7 +29,7 @@ def test_import(self): @patch.object(Collector, 'publish') def test_should_work_with_real_data(self, publish_mock): - patch_urlopen = patch('urllib2.urlopen', Mock( + patch_urlopen = patch(URLOPEN, Mock( return_value=self.getFixture('stats.txt'))) patch_urlopen.start() @@ -54,7 +55,7 @@ def test_should_work_with_real_data(self, publish_mock): @patch.object(Collector, 'publish') def test_should_fail_gracefully(self, publish_mock): - patch_urlopen = patch('urllib2.urlopen', Mock( + patch_urlopen = patch(URLOPEN, Mock( return_value=self.getFixture('stats_blank.txt'))) patch_urlopen.start() diff --git a/src/collectors/scribe/scribe.py b/src/collectors/scribe/scribe.py index d31cef02e..31999598f 100644 --- a/src/collectors/scribe/scribe.py +++ b/src/collectors/scribe/scribe.py @@ -36,7 +36,7 @@ def get_default_config(self): def key_to_metric(self, key): """Replace all non-letter characters with underscores""" - return ''.join(l if l in string.letters else '_' for l in key) + return ''.join(l if l in string.ascii_letters else '_' for l in key) def get_scribe_ctrl_output(self): cmd = [self.config['scribe_ctrl_bin'], 'counters'] diff --git a/src/collectors/scribe/test/test_scribe.py b/src/collectors/scribe/test/test_scribe.py index a4d218276..1fbbd3fc8 100644 --- a/src/collectors/scribe/test/test_scribe.py +++ b/src/collectors/scribe/test/test_scribe.py @@ -3,7 +3,7 @@ from test import CollectorTestCase from test import get_collector_config -from mock import Mock, patch +from test import Mock, patch from scribe import ScribeCollector diff --git a/src/collectors/sidekiq/sidekiq.py b/src/collectors/sidekiq/sidekiq.py index 8c35fad2e..9e686d7de 100644 --- a/src/collectors/sidekiq/sidekiq.py +++ b/src/collectors/sidekiq/sidekiq.py @@ -8,8 +8,6 @@ * redis """ -from itertools import izip - try: import redis from redis.sentinel import Sentinel @@ -89,7 +87,7 @@ def get_redis_client(self): else: sentinel_ports = [None for _ in xrange(len(ports))] - for port, sentinel_port in izip(ports, sentinel_ports): + for port, sentinel_port in zip(ports, sentinel_ports): for db in xrange(0, int(databases)): master = self.get_master( host, port, sentinel_port, sentinel_name diff --git a/src/collectors/sidekiq/test/testsidekiq.py b/src/collectors/sidekiq/test/testsidekiq.py index fb41aac82..d1b68d0d4 100644 --- a/src/collectors/sidekiq/test/testsidekiq.py +++ b/src/collectors/sidekiq/test/testsidekiq.py @@ -7,8 +7,8 @@ from test import get_collector_config from test import unittest from test import run_only -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from sidekiq import SidekiqCollector diff --git a/src/collectors/sidekiqweb/sidekiqweb.py b/src/collectors/sidekiqweb/sidekiqweb.py index 1a9acd833..86d5fb6e4 100644 --- a/src/collectors/sidekiqweb/sidekiqweb.py +++ b/src/collectors/sidekiqweb/sidekiqweb.py @@ -5,7 +5,6 @@ #### Dependencies - * urllib2 * json (or simeplejson) """ @@ -15,8 +14,8 @@ except ImportError: import simplejson as json -import urllib2 import diamond.collector +import diamond.pycompat class SidekiqWebCollector(diamond.collector.Collector): @@ -42,22 +41,24 @@ def get_default_config(self): def collect(self): try: - response = urllib2.urlopen("http://%s:%s/dashboard/stats" % ( - self.config['host'], int(self.config['port']))) - except Exception, e: + url = "http://%s:%s/dashboard/stats" % ( + self.config['host'], int(self.config['port'])) + response = diamond.pycompat.urlopen(url) + except Exception as e: self.log.error('Couldnt connect to sidekiq-web: %s', e) return {} try: j = json.loads(response.read()) - except Exception, e: + except Exception as e: self.log.error('Couldnt parse json: %s', e) return {} for k in j: for item, value in j[k].items(): - if isinstance(value, (str, unicode)) and 'M' in value: + if isinstance(value, (bytes, diamond.pycompat.unicode)) \ + and 'M' in value: value = float(value.replace('M', '')) for unit in self.config['byte_unit']: unit_value = diamond.convertor.binary.convert( diff --git a/src/collectors/sidekiqweb/test/testsidekiqweb.py b/src/collectors/sidekiqweb/test/testsidekiqweb.py index cb87037de..f540f28df 100644 --- a/src/collectors/sidekiqweb/test/testsidekiqweb.py +++ b/src/collectors/sidekiqweb/test/testsidekiqweb.py @@ -5,10 +5,11 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector +from diamond.pycompat import URLOPEN from sidekiqweb import SidekiqWebCollector ########################################################################## @@ -28,7 +29,7 @@ def test_import(self): @patch.object(Collector, 'publish') def test_should_work_with_real_data(self, publish_mock): - patch_urlopen = patch('urllib2.urlopen', Mock( + patch_urlopen = patch(URLOPEN, Mock( return_value=self.getFixture('stats'))) patch_urlopen.start() @@ -53,7 +54,7 @@ def test_should_work_with_real_data(self, publish_mock): @patch.object(Collector, 'publish') def test_should_fail_gracefully(self, publish_mock): - patch_urlopen = patch('urllib2.urlopen', Mock( + patch_urlopen = patch(URLOPEN, Mock( return_value=self.getFixture('stats_blank'))) patch_urlopen.start() diff --git a/src/collectors/slabinfo/test/testslabinfo.py b/src/collectors/slabinfo/test/testslabinfo.py index 106514db4..558e981cf 100644 --- a/src/collectors/slabinfo/test/testslabinfo.py +++ b/src/collectors/slabinfo/test/testslabinfo.py @@ -5,13 +5,10 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from test import Mock +from test import patch +from test import StringIO +from test import BUILTIN_OPEN from diamond.collector import Collector from slabinfo import SlabInfoCollector @@ -31,7 +28,7 @@ def setUp(self): def test_import(self): self.assertTrue(SlabInfoCollector) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch('os.access', Mock(return_value=True)) @patch.object(Collector, 'publish') def test_should_open_proc_stat(self, publish_mock, open_mock): diff --git a/src/collectors/slony/slony.py b/src/collectors/slony/slony.py index 1a2ef9dc5..ae0c91eee 100644 --- a/src/collectors/slony/slony.py +++ b/src/collectors/slony/slony.py @@ -94,7 +94,7 @@ def collect(self): } } - for name, instance in instances.iteritems(): + for name, instance in instances.items(): host = self.config['host'] port = self.config['port'] user = instance.get('user') or self.config['user'] diff --git a/src/collectors/slony/test/testslony.py b/src/collectors/slony/test/testslony.py index 17e9b15c4..13f957d72 100644 --- a/src/collectors/slony/test/testslony.py +++ b/src/collectors/slony/test/testslony.py @@ -5,7 +5,7 @@ from test import CollectorTestCase from test import get_collector_config from test import run_only -from mock import patch +from test import patch from slony import SlonyCollector diff --git a/src/collectors/smart/smart.py b/src/collectors/smart/smart.py index 6df3a28f7..b3624ca03 100644 --- a/src/collectors/smart/smart.py +++ b/src/collectors/smart/smart.py @@ -60,7 +60,10 @@ def collect(self): attributes = subprocess.Popen( command, stdout=subprocess.PIPE - ).communicate()[0].strip().splitlines() + ).communicate()[0] + if isinstance(attributes, bytes): + attributes = attributes.decode("utf8") + attributes = attributes.strip().splitlines() metrics = {} diff --git a/src/collectors/smart/test/testsmart.py b/src/collectors/smart/test/testsmart.py index 72431bb43..212d67416 100644 --- a/src/collectors/smart/test/testsmart.py +++ b/src/collectors/smart/test/testsmart.py @@ -5,9 +5,9 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import call -from mock import patch +from test import Mock +from test import call +from test import patch from diamond.collector import Collector from smart import SmartCollector diff --git a/src/collectors/snmpraw/test/testsnmpraw.py b/src/collectors/snmpraw/test/testsnmpraw.py index 2dbe447d7..4a6e9b28e 100644 --- a/src/collectors/snmpraw/test/testsnmpraw.py +++ b/src/collectors/snmpraw/test/testsnmpraw.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from snmpraw import SNMPRawCollector from diamond.collector import Collector diff --git a/src/collectors/sockstat/test/testsockstat.py b/src/collectors/sockstat/test/testsockstat.py index bd8db36b5..0753fe724 100644 --- a/src/collectors/sockstat/test/testsockstat.py +++ b/src/collectors/sockstat/test/testsockstat.py @@ -5,13 +5,15 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch -from mock import call -from collections import Iterator +from test import Mock +from test import patch +from test import call +from test import BUILTIN_OPEN +from collections import Iterator from diamond.collector import Collector from sockstat import SockstatCollector +import sys ########################################################################## @@ -28,19 +30,28 @@ def setUp(self): def test_import(self): self.assertTrue(SockstatCollector) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch('os.access', Mock(return_value=True)) @patch.object(Collector, 'publish') def test_should_open_proc_net_sockstat(self, publish_mock, open_mock): - class Klass(Iterator): - + class KlassPy2(Iterator): def close(self): pass def next(self): - raise StopIteration + raise StopIteration() + + class KlassPy3(Iterator): + def close(self): + pass - open_mock.return_value = Klass() + def __next__(self): + raise StopIteration() + # py2.6x does not support the namedtuple .major + if sys.version_info[0] == 2: + open_mock.return_value = KlassPy2() + else: + open_mock.return_value = KlassPy3() self.collector.collect() calls = [call('/proc/net/sockstat'), call('/proc/net/sockstat6')] open_mock.assert_has_calls(calls) diff --git a/src/collectors/solr/solr.py b/src/collectors/solr/solr.py index d08ecbc7d..cf127e35b 100644 --- a/src/collectors/solr/solr.py +++ b/src/collectors/solr/solr.py @@ -6,13 +6,11 @@ #### Dependencies * posixpath - * urllib2 * json """ import posixpath -import urllib2 try: import json @@ -20,6 +18,7 @@ import simplejson as json import diamond.collector +import diamond.pycompat class SolrCollector(diamond.collector.Collector): @@ -75,8 +74,8 @@ def _get(self, path): url = 'http://%s:%i/%s' % ( self.config['host'], int(self.config['port']), path) try: - response = urllib2.urlopen(url) - except Exception, err: + response = diamond.pycompat.urlopen(url) + except Exception as err: self.log.error("%s: %s", url, err) return False @@ -131,7 +130,7 @@ def collect(self): continue s = result['solr-mbeans'] - stats = dict((s[i], s[i + 1]) for i in xrange(0, len(s), 2)) + stats = dict((s[i], s[i + 1]) for i in range(0, len(s), 2)) if 'core' in self.config['stats']: core_searcher = stats["CORE"]["searcher"]["stats"] diff --git a/src/collectors/solr/test/testsolr.py b/src/collectors/solr/test/testsolr.py index c0d01752c..a7af2cf29 100644 --- a/src/collectors/solr/test/testsolr.py +++ b/src/collectors/solr/test/testsolr.py @@ -5,10 +5,11 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import call, patch +from mock import call +from test import patch from diamond.collector import Collector - +from diamond.pycompat import URLOPEN from solr import SolrCollector ########################################################################## @@ -23,14 +24,15 @@ def setUp(self): def test_import(self): self.assertTrue(SolrCollector) - @patch('urllib2.urlopen') + @patch(URLOPEN) @patch.object(Collector, 'publish') def test_should_work_with_real_data(self, publish_mock, urlopen_mock): returns = [self.getFixture('cores'), self.getFixture('ping'), self.getFixture('stats'), self.getFixture('system')] - urlopen_mock.side_effect = lambda *args: returns.pop(0) + urlopen_mock = patch(URLOPEN, Mock( + side_effect=lambda *args: returns.pop(0))) self.collector.collect() @@ -135,10 +137,12 @@ def test_should_work_with_real_data(self, publish_mock, urlopen_mock): call('http://localhost:8983/solr/admin/system?stats=true&wt=json') ]) - @patch('urllib2.urlopen') + @patch(URLOPEN) @patch.object(Collector, 'publish') def test_should_fail_gracefully(self, publish_mock, urlopen_mock): urlopen_mock.return_value = self.getFixture('stats_blank') + urlopen_mock = patch(URLOPEN, Mock( + return_value=self.getFixture('stats_blank'))) self.collector.collect() diff --git a/src/collectors/squid/squid.py b/src/collectors/squid/squid.py index 90697d006..d5afe3119 100644 --- a/src/collectors/squid/squid.py +++ b/src/collectors/squid/squid.py @@ -79,7 +79,7 @@ def _getData(self, host, port): if not data: break fulldata = fulldata + data - except Exception, e: + except Exception as e: self.log.error('Couldnt connect to squid: %s', e) return None squid_sock.close() diff --git a/src/collectors/squid/test/testsquid.py b/src/collectors/squid/test/testsquid.py index a44b55ba8..aeb5222e9 100644 --- a/src/collectors/squid/test/testsquid.py +++ b/src/collectors/squid/test/testsquid.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from squid import SquidCollector diff --git a/src/collectors/supervisord/supervisord.py b/src/collectors/supervisord/supervisord.py index c1e56dfae..e48385ebe 100644 --- a/src/collectors/supervisord/supervisord.py +++ b/src/collectors/supervisord/supervisord.py @@ -27,7 +27,10 @@ """ -import xmlrpclib +try: + from xmlrpclib import Server, ServerProxy +except ImportError: + from xmlrpc.client import Server, ServerProxy try: import supervisor.xmlrpc @@ -66,13 +69,13 @@ def getAllProcessInfo(self): 'Attempting to connect to XML-RPC server "%s"', uri) if protocol == 'unix': - server = xmlrpclib.ServerProxy( + server = ServerProxy( 'http://127.0.0.1', supervisor.xmlrpc.SupervisorTransport(None, None, uri) ).supervisor elif protocol == 'http': - server = xmlrpclib.Server(uri).supervisor + server = Server(uri).supervisor else: self.log.debug( diff --git a/src/collectors/supervisord/test/testsupervisord.py b/src/collectors/supervisord/test/testsupervisord.py index bf5b2b512..c93a34d56 100644 --- a/src/collectors/supervisord/test/testsupervisord.py +++ b/src/collectors/supervisord/test/testsupervisord.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from supervisord import SupervisordCollector diff --git a/src/collectors/tcp/tcp.py b/src/collectors/tcp/tcp.py index a357dd65b..c7a86a9c1 100644 --- a/src/collectors/tcp/tcp.py +++ b/src/collectors/tcp/tcp.py @@ -177,6 +177,7 @@ """ import diamond.collector +from diamond.pycompat import long import os @@ -260,7 +261,7 @@ def collect(self): header = header.split() data = data.split() - for i in xrange(1, len(header)): + for i in range(1, len(header)): metrics[header[i]] = data[i] for metric_name in metrics.keys(): diff --git a/src/collectors/tcp/test/testtcp.py b/src/collectors/tcp/test/testtcp.py index addabf76b..948e8b564 100644 --- a/src/collectors/tcp/test/testtcp.py +++ b/src/collectors/tcp/test/testtcp.py @@ -5,13 +5,10 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from test import Mock +from test import patch +from test import StringIO +from test import BUILTIN_OPEN from tcp import TCPCollector @@ -34,7 +31,7 @@ def test_import(self): self.assertTrue(TCPCollector) @patch('os.access', Mock(return_value=True)) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch('diamond.collector.Collector.publish') def test_should_open_proc_net_netstat(self, publish_mock, open_mock): TCPCollector.PROC = ['/proc/net/netstat'] @@ -43,7 +40,7 @@ def test_should_open_proc_net_netstat(self, publish_mock, open_mock): open_mock.assert_called_once_with('/proc/net/netstat') @patch('os.access', Mock(return_value=True)) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch('diamond.collector.Collector.publish') def test_should_work_with_synthetic_data(self, publish_mock, open_mock): TCPCollector.PROC = ['/proc/net/netstat'] diff --git a/src/collectors/tokumx/test/testtokumx.py b/src/collectors/tokumx/test/testtokumx.py index e43ef159a..69fc79d1c 100644 --- a/src/collectors/tokumx/test/testtokumx.py +++ b/src/collectors/tokumx/test/testtokumx.py @@ -6,9 +6,9 @@ from test import get_collector_config from test import unittest from test import run_only -from mock import MagicMock -from mock import patch -from mock import call +from test import MagicMock +from test import patch +from test import call from diamond.collector import Collector from tokumx import TokuMXCollector diff --git a/src/collectors/tokumx/tokumx.py b/src/collectors/tokumx/tokumx.py index fd7123939..736a2d3e8 100644 --- a/src/collectors/tokumx/tokumx.py +++ b/src/collectors/tokumx/tokumx.py @@ -125,7 +125,7 @@ def collect(self): network_timeout=self.config['network_timeout'], read_preference=ReadPreference.SECONDARY, ) - except Exception, e: + except Exception as e: self.log.error('Couldnt connect to mongodb: %s', e) continue @@ -133,7 +133,7 @@ def collect(self): if user: try: conn.admin.authenticate(user, passwd) - except Exception, e: + except Exception as e: self.log.error( 'User auth given, but could not autheticate' + ' with host: %s, err: %s' % (host, e)) diff --git a/src/collectors/twemproxy/test/testtwemproxy.py b/src/collectors/twemproxy/test/testtwemproxy.py index 6ba15d3c8..15b6066cd 100644 --- a/src/collectors/twemproxy/test/testtwemproxy.py +++ b/src/collectors/twemproxy/test/testtwemproxy.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from twemproxy import TwemproxyCollector diff --git a/src/collectors/twemproxy/twemproxy.py b/src/collectors/twemproxy/twemproxy.py index 0df1b2b28..58549de94 100644 --- a/src/collectors/twemproxy/twemproxy.py +++ b/src/collectors/twemproxy/twemproxy.py @@ -113,17 +113,17 @@ def get_stats(self, host, port): stats = {} pools = {} - for stat, value in data.iteritems(): + for stat, value in data.items(): # Test if this is a pool if isinstance(value, dict): pool_name = stat.replace('.', '_') pools[pool_name] = {} - for pool_stat, pool_value in value.iteritems(): + for pool_stat, pool_value in value.items(): # Test if this is a pool server if isinstance(pool_value, dict): server_name = pool_stat.replace('.', '_') pools[pool_name][server_name] = {} - for server_stat, server_value in pool_value.iteritems(): + for server_stat, server_value in pool_value.items(): pools[pool_name][server_name][server_stat] = \ int(server_value) else: @@ -140,7 +140,7 @@ def collect(self): hosts = self.config.get('hosts') # Convert a string config value to be an array - if isinstance(hosts, basestring): + if isinstance(hosts, str): hosts = [hosts] for host in hosts: @@ -161,11 +161,11 @@ def collect(self): self.publish_counter(alias + "." + stat, stats[stat]) # Pool stats - for pool, pool_stats in pools.iteritems(): - for stat, stat_value in pool_stats.iteritems(): + for pool, pool_stats in pools.items(): + for stat, stat_value in pool_stats.items(): # Test if this is a pool server if isinstance(stat_value, dict): - for server_stat, server_value in stat_value.iteritems(): + for server_stat, server_value in stat_value.items(): if server_stat in self.GAUGES: self.publish_gauge( alias + ".pools." + pool + ".servers." + diff --git a/src/collectors/udp/test/testudp.py b/src/collectors/udp/test/testudp.py index 7edcff02f..f92480c62 100644 --- a/src/collectors/udp/test/testudp.py +++ b/src/collectors/udp/test/testudp.py @@ -5,13 +5,10 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from test import Mock +from test import patch +from test import StringIO +from test import BUILTIN_OPEN from diamond.collector import Collector from udp import UDPCollector @@ -34,7 +31,7 @@ def test_import(self): self.assertTrue(UDPCollector) @patch('os.access', Mock(return_value=True)) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch.object(Collector, 'publish') def test_should_open_proc_net_snmp(self, publish_mock, open_mock): UDPCollector.PROC = ['/proc/net/snmp'] diff --git a/src/collectors/udp/udp.py b/src/collectors/udp/udp.py index 8e4f04acc..6b73b075d 100644 --- a/src/collectors/udp/udp.py +++ b/src/collectors/udp/udp.py @@ -10,6 +10,7 @@ """ import diamond.collector +from diamond.pycompat import long import os @@ -83,7 +84,7 @@ def collect(self): header = header.split() data = data.split() - for i in xrange(1, len(header)): + for i in range(1, len(header)): metrics[header[i]] = data[i] for metric_name in metrics.keys(): diff --git a/src/collectors/unbound/test/testunbound.py b/src/collectors/unbound/test/testunbound.py index cf08767bf..5c6b79982 100644 --- a/src/collectors/unbound/test/testunbound.py +++ b/src/collectors/unbound/test/testunbound.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector diff --git a/src/collectors/unbound/unbound.py b/src/collectors/unbound/unbound.py index ff17d53ad..177eb15d9 100644 --- a/src/collectors/unbound/unbound.py +++ b/src/collectors/unbound/unbound.py @@ -86,5 +86,5 @@ def collect(self): if include_hist: histogram = self.get_massaged_histogram(raw_histogram) - for intv, value in histogram.iteritems(): + for intv, value in histogram.items(): self.publish('histogram.' + intv, value) diff --git a/src/collectors/ups/test/testups.py b/src/collectors/ups/test/testups.py index 00693278f..a3f7db2f5 100644 --- a/src/collectors/ups/test/testups.py +++ b/src/collectors/ups/test/testups.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from ups import UPSCollector diff --git a/src/collectors/ups/ups.py b/src/collectors/ups/ups.py index 9aa8e0024..eda46a370 100644 --- a/src/collectors/ups/ups.py +++ b/src/collectors/ups/ups.py @@ -54,6 +54,8 @@ def collect(self): p = subprocess.Popen(command, stdout=subprocess.PIPE).communicate()[0] + if isinstance(p, bytes): + p = p.decode("utf8") for ln in p.strip().splitlines(): datapoint = ln.split(": ") diff --git a/src/collectors/uptime/test/test_uptime.py b/src/collectors/uptime/test/test_uptime.py index 9dc2bec29..4f03f9ba7 100644 --- a/src/collectors/uptime/test/test_uptime.py +++ b/src/collectors/uptime/test/test_uptime.py @@ -4,13 +4,10 @@ from test import CollectorTestCase from test import get_collector_config -from mock import Mock -from mock import patch - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from test import Mock +from test import patch +from test import StringIO +from test import BUILTIN_OPEN from diamond.collector import Collector from uptime import UptimeCollector @@ -33,7 +30,7 @@ def setUp(self, config=None): def test_import(self): self.assertTrue(UptimeCollector) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch('os.path.exists', Mock(return_value=True)) @patch.object(Collector, 'publish') def test_should_open_proc_uptime(self, publish_mock, open_mock): diff --git a/src/collectors/uptime/uptime.py b/src/collectors/uptime/uptime.py index 02a0c55ec..4525ff3d1 100644 --- a/src/collectors/uptime/uptime.py +++ b/src/collectors/uptime/uptime.py @@ -42,7 +42,7 @@ def read(self): fd.close() v = float(uptime.split()[0].strip()) return convertor.time.convert(v, 's', self.config['metric_name']) - except Exception, e: + except Exception as e: self.log.error('Unable to read uptime from %s: %s' % (self.PROC, e)) return None diff --git a/src/collectors/users/test/testusers.py b/src/collectors/users/test/testusers.py index 3d85693b3..a44fde4b0 100644 --- a/src/collectors/users/test/testusers.py +++ b/src/collectors/users/test/testusers.py @@ -6,7 +6,7 @@ from test import get_collector_config from test import unittest from test import run_only -from mock import patch +from test import patch from diamond.collector import Collector from users import UsersCollector diff --git a/src/collectors/userscripts/test/testuserscripts.py b/src/collectors/userscripts/test/testuserscripts.py index fcb8bb77e..b87b73790 100644 --- a/src/collectors/userscripts/test/testuserscripts.py +++ b/src/collectors/userscripts/test/testuserscripts.py @@ -9,7 +9,7 @@ from test import get_collector_config from test import unittest from test import run_only -from mock import patch +from test import patch from diamond.collector import Collector from userscripts import UserScriptsCollector diff --git a/src/collectors/userscripts/userscripts.py b/src/collectors/userscripts/userscripts.py index 7f2d77ec7..c1ce5c688 100644 --- a/src/collectors/userscripts/userscripts.py +++ b/src/collectors/userscripts/userscripts.py @@ -70,8 +70,8 @@ def collect(self): proc = subprocess.Popen([absolutescriptpath], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - (out, err) = proc.communicate() - except subprocess.CalledProcessError, e: + out, err = proc.communicate() + except subprocess.CalledProcessError as e: self.log.error("%s error launching: %s; skipping" % (absolutescriptpath, e)) continue @@ -82,10 +82,12 @@ def collect(self): self.log.info("%s return no output" % absolutescriptpath) continue if err: - self.log.error("%s returned error output (stderr): %s" % + self.log.error("%s returned error output (stderr): %r" % (absolutescriptpath, err)) + if isinstance(out, bytes): + out = out.decode("utf8") # Use filter to remove empty lines of output - for line in filter(None, out.split('\n')): + for line in [_f for _f in out.split('\n') if _f]: # Ignore invalid lines try: name, value = line.split() diff --git a/src/collectors/varnish/test/testvarnish.py b/src/collectors/varnish/test/testvarnish.py index 40bc8f4ae..354bf4a34 100644 --- a/src/collectors/varnish/test/testvarnish.py +++ b/src/collectors/varnish/test/testvarnish.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector diff --git a/src/collectors/varnish/varnish.py b/src/collectors/varnish/varnish.py index 1baa40031..827992a12 100644 --- a/src/collectors/varnish/varnish.py +++ b/src/collectors/varnish/varnish.py @@ -176,6 +176,8 @@ def poll(self): output = subprocess.Popen(command, stdout=subprocess.PIPE).communicate()[0] + if isinstance(output, bytes): + output = output.decode("utf8") except OSError: output = "" diff --git a/src/collectors/vmstat/test/testvmstat.py b/src/collectors/vmstat/test/testvmstat.py index 81adbcdcb..4d6db4cf3 100644 --- a/src/collectors/vmstat/test/testvmstat.py +++ b/src/collectors/vmstat/test/testvmstat.py @@ -2,16 +2,13 @@ # coding=utf-8 ############################################################################### +from test import BUILTIN_OPEN from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from test import Mock +from test import patch +from test import StringIO from diamond.collector import Collector from vmstat import VMStatCollector @@ -31,7 +28,7 @@ def setUp(self): def test_import(self): self.assertTrue(VMStatCollector) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch('os.access', Mock(return_value=True)) @patch.object(Collector, 'publish') def test_should_open_proc_vmstat(self, publish_mock, open_mock): diff --git a/src/collectors/websitemonitor/test/testwebsitemonitor.py b/src/collectors/websitemonitor/test/testwebsitemonitor.py index 6d5b56588..b3db9efb1 100644 --- a/src/collectors/websitemonitor/test/testwebsitemonitor.py +++ b/src/collectors/websitemonitor/test/testwebsitemonitor.py @@ -4,9 +4,10 @@ from test import CollectorTestCase from test import get_collector_config -from mock import patch +from test import patch from diamond.collector import Collector +from diamond.pycompat import URLOPEN from websitemonitor import WebsiteMonitorCollector @@ -38,7 +39,7 @@ def setUp(self, config=None): self.collector = WebsiteMonitorCollector(config, None) - self.patcher = patch('urllib2.urlopen') + self.patcher = patch(URLOPEN) self.urlopen_mock = self.patcher.start() def test_import(self): diff --git a/src/collectors/websitemonitor/websitemonitor.py b/src/collectors/websitemonitor/websitemonitor.py index a75371548..cd3700baa 100644 --- a/src/collectors/websitemonitor/websitemonitor.py +++ b/src/collectors/websitemonitor/websitemonitor.py @@ -1,16 +1,12 @@ # coding=utf-8 """ Gather HTTP Response code and Duration of HTTP request - -#### Dependencies - * urllib2 - """ - -import urllib2 import time from datetime import datetime import diamond.collector +import diamond.pycompat +from diamond.pycompat import URLError class WebsiteMonitorCollector(diamond.collector.Collector): @@ -38,8 +34,6 @@ def get_default_config(self): return default_config def collect(self): - req = urllib2.Request('%s' % (self.config['URL'])) - try: # time in seconds since epoch as a floating number start_time = time.time() @@ -48,7 +42,7 @@ def collect(self): ).strftime('%B %d, %Y %H:%M:%S') self.log.debug('Start time: %s' % (st)) - resp = urllib2.urlopen(req) + resp = diamond.pycompat.urlopen(self.config['URL']) # time in seconds since epoch as a floating number end_time = time.time() # human-readable end time e.eg. November 25, 2013 18:15:56 @@ -59,8 +53,8 @@ def collect(self): # Publish metrics self.publish('response_time.%s' % (resp.code), rt, metric_type='COUNTER') - # urllib2 will puke on non HTTP 200/OK URLs - except urllib2.URLError, e: + # urllib will puke on non HTTP 200/OK URLs + except URLError as e: if e.code != 200: # time in seconds since epoch as a floating number end_time = time.time() @@ -71,8 +65,8 @@ def collect(self): self.publish('response_time.%s' % (e.code), rt, metric_type='COUNTER') - except IOError, e: + except IOError as e: self.log.error('Unable to open %s' % (self.config['URL'])) - except Exception, e: + except Exception as e: self.log.error("Unknown error opening url: %s", e) diff --git a/src/collectors/xen_collector/test/testxen.py b/src/collectors/xen_collector/test/testxen.py index aca58c007..65ce5bfe1 100644 --- a/src/collectors/xen_collector/test/testxen.py +++ b/src/collectors/xen_collector/test/testxen.py @@ -6,8 +6,8 @@ from test import get_collector_config from test import unittest from test import run_only -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from xen_collector import XENCollector @@ -48,15 +48,15 @@ def __init__(self, id): def info(self): if self.id == 0: - return [1, 49420888L, 49420888L, 8, 911232000000000L] + return [1, 49420888, 49420888, 8, 911232000000000] if self.id == 1: - return [1, 2097152L, 2097152L, 2, 310676150000000L] + return [1, 2097152, 2097152, 2, 310676150000000] if self.id == 2: - return [1, 2097152L, 2097152L, 2, 100375300000000L] + return [1, 2097152, 2097152, 2, 100375300000000] if self.id == 3: - return [1, 10485760L, 10485760L, 2, 335312040000000L] + return [1, 10485760, 10485760, 2, 335312040000000] if self.id == 4: - return [1, 10485760L, 10485760L, 2, 351313480000000L] + return [1, 10485760, 10485760, 2, 351313480000000] libvirt_m = Mock() libvirt_m.getInfo.return_value = ['x86_64', 48262, 8, 1200, 2, 1, 4, 1] diff --git a/src/collectors/xfs/test/testxfs.py b/src/collectors/xfs/test/testxfs.py index 924a4a332..4311cd9a9 100644 --- a/src/collectors/xfs/test/testxfs.py +++ b/src/collectors/xfs/test/testxfs.py @@ -5,13 +5,10 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from test import StringIO from diamond.collector import Collector from xfs import XFSCollector diff --git a/src/collectors/zookeeper/zookeeper.py b/src/collectors/zookeeper/zookeeper.py index b132ba25c..d7e8f5937 100644 --- a/src/collectors/zookeeper/zookeeper.py +++ b/src/collectors/zookeeper/zookeeper.py @@ -64,7 +64,7 @@ def get_default_config(self): return config def get_raw_stats(self, host, port): - data = '' + data = b'' # connect try: if port is None: @@ -74,13 +74,13 @@ def get_raw_stats(self, host, port): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect((host, int(port))) # request stats - sock.send('mntr\n') + sock.send(b'mntr\n') # something big enough to get whatever is sent back data = sock.recv(4096) except socket.error: self.log.exception('Failed to get stats from %s:%s', host, port) - return data + return data.decode("utf8") def get_stats(self, host, port): # stuff that's always ignored, aren't 'stats' @@ -118,7 +118,7 @@ def collect(self): hosts = self.config.get('hosts') # Convert a string config value to be an array - if isinstance(hosts, basestring): + if isinstance(hosts, str): hosts = [hosts] for host in hosts: diff --git a/src/diamond/collector.py b/src/diamond/collector.py index 8e5727814..2856f7169 100644 --- a/src/diamond/collector.py +++ b/src/diamond/collector.py @@ -16,7 +16,7 @@ from diamond.metric import Metric from diamond.utils.config import load_config -from error import DiamondException +from . error import DiamondException # Detect the architecture of the system and set the counters for MAX_VALUES # appropriately. Otherwise, rolling over counters will cause incorrect or @@ -52,7 +52,10 @@ def get_hostname(config, method=None): proc = subprocess.Popen(config['hostname'], shell=True, stdout=subprocess.PIPE) - hostname = proc.communicate()[0].strip() + hostname = proc.communicate()[0] + if isinstance(hostname, bytes): + hostname = hostname.decode("utf8") + hostname = hostname.strip() if proc.returncode != 0: raise subprocess.CalledProcessError(proc.returncode, config['hostname']) @@ -144,7 +147,7 @@ def str_to_bool(value): Converts string truthy/falsey strings to a bool Empty strings are false """ - if isinstance(value, basestring): + if isinstance(value, str): value = value.strip().lower() if value in ['true', 't', 'yes', 'y']: return True @@ -222,7 +225,7 @@ def process_config(self): event """ if 'byte_unit' in self.config: - if isinstance(self.config['byte_unit'], basestring): + if isinstance(self.config['byte_unit'], str): self.config['byte_unit'] = self.config['byte_unit'].split() if 'enabled' in self.config: @@ -569,8 +572,13 @@ def run_command(self, args): if str_to_bool(self.config['use_sudo']): command.insert(0, self.config['sudo_cmd']) - return subprocess.Popen(command, - stdout=subprocess.PIPE).communicate() + p = subprocess.Popen(command, stdout=subprocess.PIPE) + stdout, stderr = p.communicate() + if isinstance(stdout, bytes): + stdout = stdout.decode("utf8") + if isinstance(stderr, bytes): + stderr = stderr.decode("utf8") + return stdout, stderr except OSError: self.log.exception("Unable to run %s", command) return None diff --git a/src/diamond/gmetric.py b/src/diamond/gmetric.py index d35ab5a2e..ab3a3534d 100644 --- a/src/diamond/gmetric.py +++ b/src/diamond/gmetric.py @@ -93,7 +93,7 @@ def send(self, NAME, VAL, TYPE='', UNITS='', SLOPE='both', TMAX, DMAX, GROUP) - # print msg + # print(msg) self.socket.sendto(meta_msg, self.hostport) self.socket.sendto(data_msg, self.hostport) diff --git a/src/diamond/handler/archive.py b/src/diamond/handler/archive.py index 0157cb6d3..f41f8c107 100644 --- a/src/diamond/handler/archive.py +++ b/src/diamond/handler/archive.py @@ -5,7 +5,7 @@ every night and remove after 7 days. """ -from Handler import Handler +from . Handler import Handler import logging import logging.handlers diff --git a/src/diamond/handler/cloudwatch.py b/src/diamond/handler/cloudwatch.py index d5491c87a..9a9897200 100644 --- a/src/diamond/handler/cloudwatch.py +++ b/src/diamond/handler/cloudwatch.py @@ -41,7 +41,7 @@ import sys import datetime -from Handler import Handler +from . Handler import Handler from configobj import Section try: @@ -254,10 +254,10 @@ def send_metrics_to_cloudwatch(self, rule, metric, dimensions): rule['namespace'], str(metric.value), str(dimensions)) - except AttributeError, e: + except AttributeError as e: self.log.error( "CloudWatch: Failed publishing - %s ", str(e)) - except Exception, e: # Rough connection re-try logic. + except Exception as e: # Rough connection re-try logic. self.log.error( "CloudWatch: Failed publishing - %s\n%s ", str(e), diff --git a/src/diamond/handler/datadog.py b/src/diamond/handler/datadog.py index d25be756c..35e740dce 100644 --- a/src/diamond/handler/datadog.py +++ b/src/diamond/handler/datadog.py @@ -10,7 +10,7 @@ """ -from Handler import Handler +from . Handler import Handler import logging from collections import deque diff --git a/src/diamond/handler/g_metric.py b/src/diamond/handler/g_metric.py index 9d41804af..4d7ae23d9 100644 --- a/src/diamond/handler/g_metric.py +++ b/src/diamond/handler/g_metric.py @@ -5,7 +5,7 @@ [Ganglia Monitoring System](http://ganglia.sourceforge.net/) """ -from Handler import Handler +from . Handler import Handler import logging try: import gmetric diff --git a/src/diamond/handler/graphite.py b/src/diamond/handler/graphite.py index 006a26a7c..1c5d39507 100644 --- a/src/diamond/handler/graphite.py +++ b/src/diamond/handler/graphite.py @@ -16,7 +16,7 @@ """ -from Handler import Handler +from . Handler import Handler import socket import time @@ -202,7 +202,7 @@ def _connect(self): connection_struct = (self.host, self.port) try: addrinfo = socket.getaddrinfo(self.host, self.port, 0, stream) - except socket.gaierror, ex: + except socket.gaierror as ex: self.log.error("GraphiteHandler: Error looking up graphite host" " '%s' - %s", self.host, ex) @@ -242,7 +242,7 @@ def _connect(self): "graphite server %s:%d.", self.host, self.port) self.last_connect_timestamp = time.time() - except Exception, ex: + except Exception as ex: # Log Error self._throttle_error("GraphiteHandler: Failed to connect to " "%s:%i. %s.", self.host, self.port, ex) diff --git a/src/diamond/handler/graphitepickle.py b/src/diamond/handler/graphitepickle.py index edda9c5e8..888f0562b 100644 --- a/src/diamond/handler/graphitepickle.py +++ b/src/diamond/handler/graphitepickle.py @@ -18,7 +18,7 @@ import struct -from graphite import GraphiteHandler +from . graphite import GraphiteHandler try: import cPickle as pickle diff --git a/src/diamond/handler/hostedgraphite.py b/src/diamond/handler/hostedgraphite.py index 01289d6b1..287d889af 100644 --- a/src/diamond/handler/hostedgraphite.py +++ b/src/diamond/handler/hostedgraphite.py @@ -16,8 +16,8 @@ """ -from Handler import Handler -from graphite import GraphiteHandler +from . Handler import Handler +from . graphite import GraphiteHandler class HostedGraphiteHandler(Handler): diff --git a/src/diamond/handler/httpHandler.py b/src/diamond/handler/httpHandler.py index f3bfefbe5..c8b1c926e 100755 --- a/src/diamond/handler/httpHandler.py +++ b/src/diamond/handler/httpHandler.py @@ -3,10 +3,18 @@ """ Send metrics to a http endpoint via POST + +#### Configuration +Enable this handler + + * handlers = diamond.handler.httpHandler.HttpPostHandler + + * url = http://www.example.com/endpoint + """ -from Handler import Handler -import urllib2 +from . Handler import Handler +from . pycompat import Request, urlopen class HttpPostHandler(Handler): @@ -56,6 +64,6 @@ def flush(self): self.post() def post(self): - req = urllib2.Request(self.url, "\n".join(self.metrics)) - urllib2.urlopen(req) + req = Request(self.url, "\n".join(self.metrics)) + urlopen(req) self.metrics = [] diff --git a/src/diamond/handler/influxdbHandler.py b/src/diamond/handler/influxdbHandler.py index 1fa8463cb..0757177d9 100644 --- a/src/diamond/handler/influxdbHandler.py +++ b/src/diamond/handler/influxdbHandler.py @@ -29,7 +29,7 @@ """ import time -from Handler import Handler +from . Handler import Handler try: from influxdb.client import InfluxDBClient @@ -207,7 +207,7 @@ def _connect(self): self.log.debug("InfluxdbHandler: Established connection to " "%s:%d/%s.", self.hostname, self.port, self.database) - except Exception, ex: + except Exception as ex: # Log Error self._throttle_error("InfluxdbHandler: Failed to connect to " "%s:%d/%s. %s", diff --git a/src/diamond/handler/libratohandler.py b/src/diamond/handler/libratohandler.py index c080ded31..18cfced0a 100644 --- a/src/diamond/handler/libratohandler.py +++ b/src/diamond/handler/libratohandler.py @@ -12,7 +12,7 @@ """ -from Handler import Handler +from . Handler import Handler import logging import time import re @@ -48,7 +48,7 @@ def __init__(self, config=None): # If a user leaves off the ending comma, cast to a array for them include_filters = self.config['include_filters'] - if isinstance(include_filters, basestring): + if isinstance(include_filters, str): include_filters = [include_filters] self.include_reg = re.compile(r'(?:%s)' % '|'.join(include_filters)) diff --git a/src/diamond/handler/logentries_diamond.py b/src/diamond/handler/logentries_diamond.py index f145b2e68..473b856da 100644 --- a/src/diamond/handler/logentries_diamond.py +++ b/src/diamond/handler/logentries_diamond.py @@ -5,11 +5,11 @@ based on data in real time. """ -from Handler import Handler +from . Handler import Handler import logging -import urllib2 import json from collections import deque +from diamond.pycompat import URLError, urlopen, Request class LogentriesDiamondHandler(Handler): @@ -75,9 +75,9 @@ def _send(self): metric = self.queue.popleft() topic, value, timestamp = str(metric).split() msg = json.dumps({"event": {topic: value}}) - req = urllib2.Request("https://js.logentries.com/v1/logs/" + - self.log_token, msg) + req = Request("https://js.logentries.com/v1/logs/" + + self.log_token, msg) try: - urllib2.urlopen(req) - except urllib2.URLError, e: + urlopen(req) + except URLError as e: logging.error("Can't send log message to Logentries %s", e) diff --git a/src/diamond/handler/mqtt.py b/src/diamond/handler/mqtt.py index fc60ffa7a..1d6558ec5 100644 --- a/src/diamond/handler/mqtt.py +++ b/src/diamond/handler/mqtt.py @@ -59,7 +59,7 @@ """ -from Handler import Handler +from . Handler import Handler from diamond.collector import get_hostname import os HAVE_SSL = True diff --git a/src/diamond/handler/multigraphite.py b/src/diamond/handler/multigraphite.py index 4d8f8e55f..3755673e5 100644 --- a/src/diamond/handler/multigraphite.py +++ b/src/diamond/handler/multigraphite.py @@ -6,8 +6,8 @@ Specify them as a list of hosts divided by comma. """ -from Handler import Handler -from graphite import GraphiteHandler +from . Handler import Handler +from . graphite import GraphiteHandler from copy import deepcopy diff --git a/src/diamond/handler/multigraphitepickle.py b/src/diamond/handler/multigraphitepickle.py index 3c49d4320..4f1bfc24f 100644 --- a/src/diamond/handler/multigraphitepickle.py +++ b/src/diamond/handler/multigraphitepickle.py @@ -6,8 +6,8 @@ servers. Specify them as a list of hosts divided by comma. """ -from Handler import Handler -from graphitepickle import GraphitePickleHandler +from . Handler import Handler +from . graphitepickle import GraphitePickleHandler from copy import deepcopy diff --git a/src/diamond/handler/mysql.py b/src/diamond/handler/mysql.py index fc1b8a718..bf0c6cea7 100644 --- a/src/diamond/handler/mysql.py +++ b/src/diamond/handler/mysql.py @@ -4,7 +4,7 @@ Insert the collected values into a mysql table """ -from Handler import Handler +from . Handler import Handler import MySQLdb @@ -83,7 +83,7 @@ def _send(self, data): (data[0], data[2], data[1])) cursor.close() self.conn.commit() - except BaseException, e: + except BaseException as e: # Log Error self.log.error("MySQLHandler: Failed sending data. %s.", e) # Attempt to restablish connection diff --git a/src/diamond/handler/null.py b/src/diamond/handler/null.py index c593dd852..411aca0ed 100644 --- a/src/diamond/handler/null.py +++ b/src/diamond/handler/null.py @@ -4,7 +4,7 @@ Output the collected values to the debug log channel. """ -from Handler import Handler +from . Handler import Handler class NullHandler(Handler): diff --git a/src/diamond/handler/queue.py b/src/diamond/handler/queue.py index 8ebf44107..9d1a9c786 100644 --- a/src/diamond/handler/queue.py +++ b/src/diamond/handler/queue.py @@ -5,7 +5,7 @@ do not try to use it as a normal handler """ -from Handler import Handler +from . Handler import Handler import Queue diff --git a/src/diamond/handler/rabbitmq_pubsub.py b/src/diamond/handler/rabbitmq_pubsub.py index 58b936d16..229907b34 100644 --- a/src/diamond/handler/rabbitmq_pubsub.py +++ b/src/diamond/handler/rabbitmq_pubsub.py @@ -4,7 +4,7 @@ Output the collected values to RabitMQ pub/sub channel """ -from Handler import Handler +from . Handler import Handler import time try: @@ -156,7 +156,7 @@ def _bind(self, rmq_server): durable=self.rmq_durable) # Reset reconnect_interval after a successful connection self.reconnect_interval = 1 - except Exception, exception: + except Exception as exception: self.log.debug("Caught exception in _bind: %s", exception) if rmq_server in self.connections.keys(): self._unbind(rmq_server) @@ -200,7 +200,7 @@ def process(self, metric): channel = self.channels[rmq_server] channel.basic_publish(exchange=self.rmq_exchange, routing_key='', body="%s" % metric) - except Exception, exception: + except Exception as exception: self.log.error( "Failed publishing to %s, attempting reconnect", rmq_server) diff --git a/src/diamond/handler/rabbitmq_topic.py b/src/diamond/handler/rabbitmq_topic.py index 664a2a442..2909c3417 100644 --- a/src/diamond/handler/rabbitmq_topic.py +++ b/src/diamond/handler/rabbitmq_topic.py @@ -6,7 +6,7 @@ the metric path """ -from Handler import Handler +from . Handler import Handler try: import pika diff --git a/src/diamond/handler/riemann.py b/src/diamond/handler/riemann.py index c8d392e08..fa07e3356 100644 --- a/src/diamond/handler/riemann.py +++ b/src/diamond/handler/riemann.py @@ -18,7 +18,7 @@ """ -from Handler import Handler +from . Handler import Handler import logging try: @@ -87,7 +87,7 @@ def process(self, metric): event = self._metric_to_riemann_event(metric) try: self.client.send_event(event) - except Exception, e: + except Exception as e: self.log.error( "RiemannHandler: Error sending event to Riemann: %s", e) diff --git a/src/diamond/handler/rrdtool.py b/src/diamond/handler/rrdtool.py index 032b18166..a51ccedf7 100644 --- a/src/diamond/handler/rrdtool.py +++ b/src/diamond/handler/rrdtool.py @@ -7,9 +7,8 @@ import os import re import subprocess -import Queue - -from Handler import Handler +from diamond.pycompat import Queue, Empty +from . Handler import Handler # # Constants for RRD file creation. @@ -160,7 +159,7 @@ def process(self, metric): def _queue(self, filename, timestamp, value): if filename not in self._queues: - queue = Queue.Queue() + queue = Queue() self._queues[filename] = queue else: queue = self._queues[filename] @@ -199,7 +198,7 @@ def _flush_queue(self, filename): if timestamp not in updates: updates[timestamp] = [] updates[timestamp].append(value) - except Queue.Empty: + except Empty: break # Save the last update time. diff --git a/src/diamond/handler/sentry.py b/src/diamond/handler/sentry.py index 7f8eed9f4..e39686db2 100644 --- a/src/diamond/handler/sentry.py +++ b/src/diamond/handler/sentry.py @@ -31,7 +31,7 @@ import logging import re -from Handler import Handler +from . Handler import Handler from diamond.collector import get_hostname from configobj import Section @@ -333,7 +333,7 @@ def compile_section(self, section): # init rule try: return Rule(**kwargs) - except InvalidRule, err: + except InvalidRule as err: self.log.error(str(err)) def configure_sentry_errors(self): diff --git a/src/diamond/handler/signalfx.py b/src/diamond/handler/signalfx.py index d1d7eb787..51246c97c 100644 --- a/src/diamond/handler/signalfx.py +++ b/src/diamond/handler/signalfx.py @@ -3,11 +3,6 @@ """ Send metrics to signalfx -#### Dependencies - - * urllib2 - - #### Configuration Enable this handler @@ -18,12 +13,12 @@ posting """ -from Handler import Handler +from . Handler import Handler from diamond.util import get_diamond_version +from diamond.pycompat import URLError, urlopen, Request import json import logging import time -import urllib2 class SignalfxHandler(Handler): @@ -127,13 +122,13 @@ def _send(self): self.metrics = [] postBody = json.dumps(postDictionary) logging.debug("Body is %s", postBody) - req = urllib2.Request(self.url, postBody, - {"Content-type": "application/json", - "X-SF-TOKEN": self.auth_token, - "User-Agent": self.user_agent()}) + req = Request(self.url, postBody, + {"Content-type": "application/json", + "X-SF-TOKEN": self.auth_token, + "User-Agent": self.user_agent()}) self.resetBatchTimeout() try: - urllib2.urlopen(req) - except urllib2.URLError: + urlopen(req) + except URLError: logging.exception("Unable to post signalfx metrics") return diff --git a/src/diamond/handler/stats_d.py b/src/diamond/handler/stats_d.py index 83f7a2816..3e60465e8 100644 --- a/src/diamond/handler/stats_d.py +++ b/src/diamond/handler/stats_d.py @@ -27,7 +27,7 @@ """ -from Handler import Handler +from . Handler import Handler import logging try: import statsd diff --git a/src/diamond/handler/statsite.py b/src/diamond/handler/statsite.py index 5b76c0fec..2be0493a2 100644 --- a/src/diamond/handler/statsite.py +++ b/src/diamond/handler/statsite.py @@ -52,7 +52,7 @@ """ -from Handler import Handler +from . Handler import Handler import socket @@ -148,7 +148,7 @@ def _send(self, data): self.socket.sendall(data) # Done break - except socket.error, e: + except socket.error as e: # Log Error self.log.error("StatsiteHandler: Failed sending data. %s.", e) # Attempt to restablish connection @@ -183,7 +183,7 @@ def _connect(self): # Log self.log.debug("Established connection to statsite server %s:%d", self.host, self.port) - except Exception, ex: + except Exception as ex: # Log Error self.log.error("StatsiteHandler: Failed to connect to %s:%i. %s", self.host, self.port, ex) diff --git a/src/diamond/handler/test/testgraphitehandler.py b/src/diamond/handler/test/testgraphitehandler.py index be6067a45..70728cdd8 100644 --- a/src/diamond/handler/test/testgraphitehandler.py +++ b/src/diamond/handler/test/testgraphitehandler.py @@ -5,9 +5,9 @@ import time from test import unittest -from mock import Mock -from mock import patch -from mock import call +from test import Mock +from test import patch +from test import call import configobj diff --git a/src/diamond/handler/test/testriemann.py b/src/diamond/handler/test/testriemann.py index 404ef7fe8..c99055f89 100644 --- a/src/diamond/handler/test/testriemann.py +++ b/src/diamond/handler/test/testriemann.py @@ -4,8 +4,8 @@ from test import unittest from test import run_only -from mock import Mock -from mock import patch +from test import Mock +from test import patch from mock import call import configobj diff --git a/src/diamond/handler/test/teststatsdhandler.py b/src/diamond/handler/test/teststatsdhandler.py index e660f6fb9..859c1c307 100644 --- a/src/diamond/handler/test/teststatsdhandler.py +++ b/src/diamond/handler/test/teststatsdhandler.py @@ -4,7 +4,8 @@ from test import unittest from test import run_only -from mock import patch +from test import patch +from test import ANY import configobj diff --git a/src/diamond/handler/test/testtsdb.py b/src/diamond/handler/test/testtsdb.py index e9cf6b721..3d11e97e0 100644 --- a/src/diamond/handler/test/testtsdb.py +++ b/src/diamond/handler/test/testtsdb.py @@ -3,7 +3,7 @@ ########################################################################## from test import unittest -from mock import Mock +from test import Mock import configobj @@ -427,5 +427,5 @@ def test_with_invalid_tag(self): try: TSDBHandler(config) fail("Expected an exception") - except Exception, e: + except Exception as e: assert(e) diff --git a/src/diamond/handler/tsdb.py b/src/diamond/handler/tsdb.py index 9ba4be42e..aa1180502 100644 --- a/src/diamond/handler/tsdb.py +++ b/src/diamond/handler/tsdb.py @@ -1,4 +1,4 @@ -# coding=utf-8 +# coding=utf-8 """ Send metrics to a [OpenTSDB](http://opentsdb.net/) server. @@ -50,7 +50,7 @@ ' """ -from Handler import Handler +from . Handler import Handler from diamond.metric import Metric import socket @@ -77,7 +77,7 @@ def __init__(self, config=None): self.timeout = int(self.config['timeout']) self.metric_format = str(self.config['format']) self.tags = "" - if isinstance(self.config['tags'], basestring): + if isinstance(self.config['tags'], str): self.tags = self.config['tags'] elif isinstance(self.config['tags'], list): for tag in self.config['tags']: @@ -187,7 +187,7 @@ def _send(self, data): self.socket.sendall(data) # Done break - except socket.error, e: + except socket.error as e: # Log Error self.log.error("TSDBHandler: Failed sending data. %s.", e) # Attempt to restablish connection @@ -217,7 +217,7 @@ def _connect(self): # Log self.log.debug("Established connection to TSDB server %s:%d", self.host, self.port) - except Exception, ex: + except Exception as ex: # Log Error self.log.error("TSDBHandler: Failed to connect to %s:%i. %s", self.host, self.port, ex) diff --git a/src/diamond/handler/zmq_pubsub.py b/src/diamond/handler/zmq_pubsub.py index 30bd0cad9..92f76ecaa 100644 --- a/src/diamond/handler/zmq_pubsub.py +++ b/src/diamond/handler/zmq_pubsub.py @@ -4,7 +4,7 @@ Output the collected values to a Zer0MQ pub/sub channel """ -from Handler import Handler +from . Handler import Handler try: import zmq diff --git a/src/diamond/metric.py b/src/diamond/metric.py index 66d82515f..91b361387 100644 --- a/src/diamond/metric.py +++ b/src/diamond/metric.py @@ -3,7 +3,8 @@ import time import re import logging -from error import DiamondException +from . error import DiamondException +from diamond.pycompat import long class Metric(object): diff --git a/src/diamond/pycompat.py b/src/diamond/pycompat.py new file mode 100644 index 000000000..c815d34db --- /dev/null +++ b/src/diamond/pycompat.py @@ -0,0 +1,18 @@ +# coding=utf-8 +URLOPEN = "diamond.pycompat.urlopen" +try: + from urllib2 import HTTPError, Request, urlopen, URLError + from urllib import urlencode, quote + from urlparse import urljoin, urlparse + from Queue import Empty, Full, Queue +except ImportError: + from urllib.request import HTTPError, Request, URLError, urlopen + from urllib.parse import urlencode, quote, urljoin, urlparse + from queue import Full, Empty, Queue + +try: + long = long + unicode = unicode +except NameError: + long = int + unicode = str diff --git a/src/diamond/server.py b/src/diamond/server.py index c4a3c77ae..75f86bd35 100644 --- a/src/diamond/server.py +++ b/src/diamond/server.py @@ -86,7 +86,7 @@ def run(self): handlers_path = self.config['server']['handlers_path'] # Make an list if not one - if isinstance(handlers_path, basestring): + if isinstance(handlers_path, str): handlers_path = handlers_path.split(',') handlers_path = map(str.strip, handlers_path) self.config['server']['handlers_path'] = handlers_path @@ -98,7 +98,7 @@ def run(self): sys.exit(1) handlers = self.config['server'].get('handlers') - if isinstance(handlers, basestring): + if isinstance(handlers, str): handlers = [handlers] # Prevent the Queue Handler from being a normal handler @@ -146,7 +146,7 @@ def run(self): ############################################################## running_collectors = [] - for collector, config in self.config['collectors'].iteritems(): + for collector, config in self.config['collectors'].items(): if config.get('enabled', False) is not True: continue running_collectors.append(collector) diff --git a/src/diamond/test/testcollector.py b/src/diamond/test/testcollector.py index d48a146f8..32ecf89e6 100644 --- a/src/diamond/test/testcollector.py +++ b/src/diamond/test/testcollector.py @@ -2,7 +2,7 @@ # coding=utf-8 ########################################################################## -from mock import patch +from test import patch from test import unittest import configobj diff --git a/src/diamond/test/testmetric.py b/src/diamond/test/testmetric.py index 365c3f742..e870ebd04 100644 --- a/src/diamond/test/testmetric.py +++ b/src/diamond/test/testmetric.py @@ -101,7 +101,7 @@ def test_issue_723(self): '0.0000997143369909', ] - for precision in xrange(0, 100): + for precision in range(0, 100): for m in metrics: metric = Metric('test.723', m, timestamp=0) diff --git a/src/diamond/utils/classes.py b/src/diamond/utils/classes.py index ed99d776e..1f756c5f1 100644 --- a/src/diamond/utils/classes.py +++ b/src/diamond/utils/classes.py @@ -40,7 +40,7 @@ def load_dynamic_class(fqn, subclass): """ Dynamically load fqn class and verify it's a subclass of subclass """ - if not isinstance(fqn, basestring): + if not isinstance(fqn, str): return fqn cls = load_class_from_name(fqn) @@ -58,7 +58,7 @@ def load_handlers(config, handler_names): handlers = [] - if isinstance(handler_names, basestring): + if isinstance(handler_names, str): handler_names = [handler_names] for handler in handler_names: @@ -119,7 +119,7 @@ def load_collectors_from_paths(paths): if paths is None: return - if isinstance(paths, basestring): + if isinstance(paths, str): paths = paths.split(',') paths = map(str.strip, paths) @@ -157,7 +157,7 @@ def load_collectors_from_paths(paths): try: # Import the module mod = imp.load_module(modname, fp, pathname, description) - except (KeyboardInterrupt, SystemExit), err: + except (KeyboardInterrupt, SystemExit) as err: logger.error( "System or keyboard interrupt " "while loading module %s" diff --git a/src/diamond/utils/config.py b/src/diamond/utils/config.py index 7a5cd2d67..bbf828015 100644 --- a/src/diamond/utils/config.py +++ b/src/diamond/utils/config.py @@ -9,7 +9,7 @@ def str_to_bool(value): Converts string truthy/falsey strings to a bool Empty strings are false """ - if isinstance(value, basestring): + if isinstance(value, str): value = value.strip().lower() if value in ['true', 't', 'yes', 'y']: return True @@ -101,7 +101,7 @@ def load_config(configfile): try: newconfig = configobj.ConfigObj(cfgfile) - except Exception, e: + except Exception as e: raise Exception("Failed to load config file %s due to %s" % (cfgfile, e)) diff --git a/src/diamond/utils/log.py b/src/diamond/utils/log.py index 70f8be52e..f9e94c8d9 100644 --- a/src/diamond/utils/log.py +++ b/src/diamond/utils/log.py @@ -58,7 +58,7 @@ def setup_logging(configfile, stdout=False): streamHandler.setLevel(rootLogLevel) log.addHandler(streamHandler) - except Exception, e: + except Exception as e: sys.stderr.write("Error occurs when initialize logging: ") sys.stderr.write(str(e)) sys.stderr.write(os.linesep) diff --git a/test.py b/test.py index fea9dd01e..c5e477bdc 100755 --- a/test.py +++ b/test.py @@ -24,13 +24,27 @@ try: from cStringIO import StringIO except ImportError: - from StringIO import StringIO + try: + from StringIO import StringIO + except ImportError: + from io import StringIO try: from setproctitle import setproctitle except ImportError: setproctitle = None +try: + from mock import ANY, call, MagicMock, Mock, mock_open, patch +except ImportError: + from unittest.mock import ANY, call, MagicMock, Mock, mock_open, patch + +try: # py3k way + import builtins + BUILTIN_OPEN = "builtins.open" +except ImportError: # py2.x way + BUILTIN_OPEN = "__builtin__.open" + sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__)))) sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), 'src'))) @@ -110,7 +124,7 @@ def getFixturePath(self, fixture_name): path = os.path.join(self.getFixtureDirPath(), fixture_name) if not os.access(path, os.R_OK): - print "Missing Fixture " + path + print("Missing Fixture " + path) return path def getFixture(self, fixture_name): @@ -124,7 +138,7 @@ def getFixtures(self): return fixtures def getPickledResults(self, results_name): - with open(self.getFixturePath(results_name), 'r') as f: + with open(self.getFixturePath(results_name), 'rb') as f: return pickle.load(f) def setPickledResults(self, results_name, data): @@ -137,11 +151,11 @@ def assertUnpublished(self, mock, key, value, expected_value=0): def assertPublished(self, mock, key, value, expected_value=1): if type(mock) is list: for m in mock: - calls = (filter(lambda x: x[0][0] == key, m.call_args_list)) + calls = list(filter(lambda x: x[0][0] == key, m.call_args_list)) if len(calls) > 0: break else: - calls = filter(lambda x: x[0][0] == key, mock.call_args_list) + calls = list(filter(lambda x: x[0][0] == key, mock.call_args_list)) actual_value = len(calls) message = '%s: actual number of calls %d, expected %d' % ( @@ -173,7 +187,7 @@ def assertUnpublishedMany(self, mock, dict, expected_value=0): return self.assertPublishedMany(mock, dict, expected_value) def assertPublishedMany(self, mock, dict, expected_value=1): - for key, value in dict.iteritems(): + for key, value in dict.items(): self.assertPublished(mock, key, value, expected_value) if type(mock) is list: @@ -186,8 +200,8 @@ def assertUnpublishedMetric(self, mock, key, value, expected_value=0): return self.assertPublishedMetric(mock, key, value, expected_value) def assertPublishedMetric(self, mock, key, value, expected_value=1): - calls = filter(lambda x: x[0][0].path.find(key) != -1, - mock.call_args_list) + calls = list(filter(lambda x: x[0][0].path.find(key) != -1, + mock.call_args_list)) actual_value = len(calls) message = '%s: actual number of calls %d, expected %d' % ( @@ -219,7 +233,7 @@ def assertUnpublishedMetricMany(self, mock, dict, expected_value=0): return self.assertPublishedMetricMany(mock, dict, expected_value) def assertPublishedMetricMany(self, mock, dict, expected_value=1): - for key, value in dict.iteritems(): + for key, value in dict.items(): self.assertPublishedMetric(mock, key, value, expected_value) mock.reset_mock() @@ -245,8 +259,8 @@ def getCollectorTests(path): locals(), ['*']) except Exception: - print "Failed to import module: %s. %s" % ( - modname, traceback.format_exc()) + print("Failed to import module: %s. %s" % ( + modname, traceback.format_exc())) continue for f in os.listdir(path):