From 34a8ccfd98ff88898716141e91d49fc88a5e1d2b Mon Sep 17 00:00:00 2001 From: Hannu Valtonen Date: Mon, 24 Aug 2015 10:42:43 +0300 Subject: [PATCH 01/33] py3k: change all print statements to be functions This is a first step towards achieving compatibility with py2.6/2.7 and py3.3+ out of the same codebase. --- bin/diamond | 25 +++++++++-------- bin/diamond-setup | 43 +++++++++++++++-------------- build_doc.py | 23 +++++++-------- src/collectors/netapp/netapp.py | 4 +-- src/collectors/netapp/netappDisk.py | 2 +- src/diamond/gmetric.py | 2 +- test.py | 6 ++-- 7 files changed, 55 insertions(+), 50 deletions(-) diff --git a/bin/diamond b/bin/diamond index 9e29d7aba..3cef1ef71 100755 --- a/bin/diamond +++ b/bin/diamond @@ -1,5 +1,6 @@ #!/usr/bin/env python # coding=utf-8 +from __future__ import print_function import os import sys @@ -110,7 +111,7 @@ def main(): gid = -1 if options.version: - print "Diamond version %s" % (get_diamond_version()) + print("Diamond version %s" % (get_diamond_version())) sys.exit(0) # Initialize Config @@ -118,8 +119,8 @@ def main(): if os.path.exists(options.configfile): config = configobj.ConfigObj(options.configfile) else: - print >> sys.stderr, "ERROR: Config file: %s does not exist." % ( - options.configfile) + print("ERROR: Config file: %s does not exist." % ( + options.configfile), file=sys.stderr) parser.print_help(sys.stderr) sys.exit(1) @@ -159,11 +160,11 @@ def main(): # Pid is not real os.unlink(options.pidfile) pid = None - print >> sys.stderr, ( - "WARN: Bogus pid file was found. I deleted it.") + print("WARN: Bogus pid file was found. I deleted it.", + file=sys.stderr) else: - print >> sys.stderr, ( - "ERROR: Pidfile exists. Server already running?") + print("ERROR: Pidfile exists. Server already running?", + file=sys.stderr) sys.exit(1) # Get final GIDs @@ -187,7 +188,8 @@ def main(): try: pf = file(options.pidfile, 'w+') except IOError, e: - print >> sys.stderr, "Failed to write PID file: %s" % (e) + print("Failed to write PID file: %s" % (e), + file=sys.stderr) sys.exit(1) pf.write("%s\n" % pid) pf.close() @@ -212,7 +214,8 @@ def main(): os.setuid(uid) except Exception, e: - print >> sys.stderr, "ERROR: Failed to set UID/GID. %s" % (e) + print("ERROR: Failed to set UID/GID. %s" % (e), + file=sys.stderr) sys.exit(1) # Log @@ -237,7 +240,7 @@ def main(): # Exit first paren sys.exit(0) except OSError, e: - print >> sys.stderr, "Failed to fork process." % (e) + print("Failed to fork process." % (e), file=sys.stderr) sys.exit(1) # Decouple from parent environmen os.setsid() @@ -249,7 +252,7 @@ def main(): # Exit second paren sys.exit(0) except OSError, e: - print >> sys.stderr, "Failed to fork process." % (e) + print("Failed to fork process." % (e), file=sys.stderr) sys.exit(1) # Close file descriptors so that we can detach sys.stdout.close() diff --git a/bin/diamond-setup b/bin/diamond-setup index 9bf9c2cb4..45eb0730e 100755 --- a/bin/diamond-setup +++ b/bin/diamond-setup @@ -1,5 +1,6 @@ #!/usr/bin/env python ########################################################################## +from __future__ import print_function import os import sys @@ -63,10 +64,10 @@ def getCollectors(path): break except TypeError: continue - # print "Imported module: %s %s" % (modname, cls.__name__) + # print("Imported module: %s %s" % (modname, cls.__name__)) except Exception: - print "Failed to import module: %s. %s" % ( - modname, traceback.format_exc()) + print("Failed to import module: %s. %s" % ( + modname, traceback.format_exc())) collectors[modname] = False continue @@ -133,9 +134,9 @@ def configureKey(key): except NotImplementedError: return - print "\n" + print("\n") if key in default_conf_help: - print default_conf_help[key] + print(default_conf_help[key]) val = raw_input(key + ' [' + user_val + ']: ') # Empty user input? Default to current value @@ -177,18 +178,18 @@ if __name__ == "__main__": if os.path.exists(options.configfile): config = ConfigObj(os.path.abspath(options.configfile)) else: - print >> sys.stderr, "ERROR: Config file: %s does not exist." % ( - options.configfile) - print >> sys.stderr, ("Please run python config.py -c" - + " /path/to/diamond.conf") + print("ERROR: Config file: %s does not exist." % ( + options.configfile), file=sys.stderr) + print("Please run python config.py -c /path/to/diamond.conf", + file=sys.stderr) parser.print_help(sys.stderr) sys.exit(1) if not options.dump: - print '' - print 'I will be over writing files in' - print config['server']['collectors_config_path'] - print 'Please type yes to continue' + print('') + print('I will be over writing files in') + print(config['server']['collectors_config_path']) + print('Please type yes to continue') val = raw_input('Are you sure? ') if val != 'yes': @@ -225,7 +226,7 @@ if __name__ == "__main__": obj = cls(config=config, handlers={}) if options.dump: - print collector + " " + str(obj.config) + print(collector + " " + str(obj.config)) continue default_conf = obj.get_default_config() @@ -241,9 +242,9 @@ if __name__ == "__main__": config_keys['instance_prefix'] = False config_keys['interval'] = False - print "*" * 60 - print "\n\t\tNow configuring " + collector - print collectors[collector].__doc__ + print("*" * 60) + print("\n\t\tNow configuring " + collector) + print(collectors[collector].__doc__) print "(%s)" % collector configureKey('enabled') @@ -256,12 +257,12 @@ if __name__ == "__main__": config_file.write() - except IOError, (errno, strerror): - print "I/O error({0}): {1}".format(errno, strerror) + except IOError as err: + print("I/O error({0}): {1}".format(err.errno, err.strerror)) except KeyboardInterrupt: - print + print("") sys.exit() except: continue if not foundcollector: - print "Collector not found." + print("Collector not found.") diff --git a/build_doc.py b/build_doc.py index 10cc41443..0a95220cc 100755 --- a/build_doc.py +++ b/build_doc.py @@ -2,6 +2,7 @@ # coding=utf-8 ########################################################################## +from __future__ import print_function import configobj import optparse import os @@ -56,8 +57,8 @@ def getCollectors(path): if cls.__name__ not in collectors: collectors[cls.__name__] = module except Exception: - print "Failed to import module: %s. %s" % ( - modname, traceback.format_exc()) + print("Failed to import module: %s. %s" % ( + modname, traceback.format_exc())) collectors[modname] = False elif os.path.isdir(cPath): @@ -91,8 +92,8 @@ def getHandlers(path, name=None): if cls.__name__ not in handlers: handlers[cls.__name__] = module except Exception: - print "Failed to import module: %s. %s" % ( - modname, traceback.format_exc()) + print("Failed to import module: %s. %s" % ( + modname, traceback.format_exc())) handlers[modname] = False elif os.path.isdir(cPath): @@ -110,7 +111,7 @@ def writeDocString(docFile, name, doc): docFile.write("%s\n" % (name)) docFile.write("=====\n") if doc is None: - print "No __doc__ string for %s!" % name + print("No __doc__ string for %s!" % name) docFile.write("%s\n" % doc) @@ -150,7 +151,7 @@ def writeDoc(items, type_name, doc_path): if item.startswith('Test'): continue - print "Processing %s..." % (item) + print("Processing %s..." % (item)) if not hasattr(items[item], item): continue @@ -173,7 +174,7 @@ def writeDoc(items, type_name, doc_path): if type_name is "Handler": os.remove(tmpfile[1]) except Exception, e: - print "Caught Exception %s" % e + print("Caught Exception %s" % e) docFile = open(os.path.join(doc_path, item + ".md"), 'w') @@ -228,10 +229,10 @@ def writeDoc(items, type_name, doc_path): if os.path.exists(options.configfile): config = configobj.ConfigObj(os.path.abspath(options.configfile)) else: - print >> sys.stderr, "ERROR: Config file: %s does not exist." % ( - options.configfile) - print >> sys.stderr, ("Please run python config.py -c " + - "/path/to/diamond.conf") + print("ERROR: Config file: %s does not exist." % ( + options.configfile), file=sys.stderr) + print("Please run python config.py -c /path/to/diamond.conf", + file=sys.stderr) parser.print_help(sys.stderr) sys.exit(1) diff --git a/src/collectors/netapp/netapp.py b/src/collectors/netapp/netapp.py index 414e92802..8287670dd 100644 --- a/src/collectors/netapp/netapp.py +++ b/src/collectors/netapp/netapp.py @@ -319,8 +319,8 @@ def collect(self, device, ip, user, password): res = server.invoke_elem(query) if(res.results_status() == "failed"): - print "Connection to filer %s failed; %s" % ( - device, res.results_reason()) + print("Connection to filer %s failed; %s" % ( + device, res.results_reason())) return num_records = res.child_get_int("records") diff --git a/src/collectors/netapp/netappDisk.py b/src/collectors/netapp/netappDisk.py index ef48f5d42..dd0dd51b6 100644 --- a/src/collectors/netapp/netappDisk.py +++ b/src/collectors/netapp/netappDisk.py @@ -273,7 +273,7 @@ def get_netapp_elem(self, netapp_api=None, sub_element=None): self.log.error( 'While using netapp API failed to retrieve ' 'disk-list-info for netapp filer %s' % self.device) - print netapp_data.sprintf() + print(netapp_data.sprintf()) return netapp_xml = \ ET.fromstring(netapp_data.sprintf()).find(sub_element) diff --git a/src/diamond/gmetric.py b/src/diamond/gmetric.py index d35ab5a2e..ab3a3534d 100644 --- a/src/diamond/gmetric.py +++ b/src/diamond/gmetric.py @@ -93,7 +93,7 @@ def send(self, NAME, VAL, TYPE='', UNITS='', SLOPE='both', TMAX, DMAX, GROUP) - # print msg + # print(msg) self.socket.sendto(meta_msg, self.hostport) self.socket.sendto(data_msg, self.hostport) diff --git a/test.py b/test.py index fea9dd01e..1481bb8f7 100755 --- a/test.py +++ b/test.py @@ -110,7 +110,7 @@ def getFixturePath(self, fixture_name): path = os.path.join(self.getFixtureDirPath(), fixture_name) if not os.access(path, os.R_OK): - print "Missing Fixture " + path + print("Missing Fixture " + path) return path def getFixture(self, fixture_name): @@ -245,8 +245,8 @@ def getCollectorTests(path): locals(), ['*']) except Exception: - print "Failed to import module: %s. %s" % ( - modname, traceback.format_exc()) + print("Failed to import module: %s. %s" % ( + modname, traceback.format_exc())) continue for f in os.listdir(path): From 3ee58a79605a4ed1f3846a4beed586eb93a0efbe Mon Sep 17 00:00:00 2001 From: Hannu Valtonen Date: Mon, 24 Aug 2015 13:56:39 +0300 Subject: [PATCH 02/33] py3k: Change exception syntax from old style commas to use the as syntax --- bin/diamond | 18 +++++++++--------- build_doc.py | 2 +- src/collectors/beanstalkd/beanstalkd.py | 2 +- src/collectors/bind/bind.py | 2 +- src/collectors/ceph/ceph.py | 4 ++-- src/collectors/cephstats/cephstats.py | 2 +- src/collectors/chronyd/chronyd.py | 2 +- src/collectors/diskspace/diskspace.py | 2 +- src/collectors/drbd/drbd.py | 2 +- src/collectors/dropwizard/dropwizard.py | 2 +- src/collectors/dseopscenter/dseopscenter.py | 4 ++-- src/collectors/elasticsearch/elasticsearch.py | 2 +- src/collectors/elb/test/testelb.py | 2 +- src/collectors/endecadgraph/endecadgraph.py | 4 ++-- src/collectors/flume/flume.py | 6 +++--- src/collectors/haproxy/haproxy.py | 6 +++--- src/collectors/http/http.py | 4 ++-- src/collectors/httpd/httpd.py | 2 +- src/collectors/jbossapi/jbossapi.py | 2 +- src/collectors/jcollectd/jcollectd.py | 6 +++--- src/collectors/kafkastat/kafkastat.py | 2 +- src/collectors/mesos/mesos.py | 2 +- src/collectors/mongodb/mongodb.py | 3 +-- src/collectors/monit/monit.py | 2 +- src/collectors/mysqlstat/mysql55.py | 2 +- src/collectors/mysqlstat/mysqlstat.py | 8 ++++---- .../nagiosperfdata/nagiosperfdata.py | 2 +- src/collectors/nginx/nginx.py | 4 ++-- src/collectors/openldap/openldap.py | 2 +- src/collectors/openvpn/openvpn.py | 2 +- src/collectors/ossec/ossec.py | 2 +- src/collectors/phpfpm/phpfpm.py | 4 ++-- src/collectors/postgres/postgres.py | 2 +- .../processresources/processresources.py | 2 +- .../puppetdashboard/puppetdashboard.py | 4 ++-- src/collectors/puppetdb/puppetdb.py | 2 +- src/collectors/rabbitmq/rabbitmq.py | 4 ++-- src/collectors/redisstat/redisstat.py | 2 +- src/collectors/resqueweb/resqueweb.py | 4 ++-- src/collectors/sidekiqweb/sidekiqweb.py | 4 ++-- src/collectors/solr/solr.py | 2 +- src/collectors/squid/squid.py | 2 +- src/collectors/tokumx/tokumx.py | 4 ++-- src/collectors/uptime/uptime.py | 2 +- src/collectors/userscripts/userscripts.py | 2 +- .../websitemonitor/websitemonitor.py | 6 +++--- src/diamond/handler/cloudwatch.py | 4 ++-- src/diamond/handler/graphite.py | 4 ++-- src/diamond/handler/influxdbHandler.py | 2 +- src/diamond/handler/logentries_diamond.py | 2 +- src/diamond/handler/mysql.py | 2 +- src/diamond/handler/rabbitmq_pubsub.py | 4 ++-- src/diamond/handler/riemann.py | 2 +- src/diamond/handler/sentry.py | 2 +- src/diamond/handler/statsite.py | 4 ++-- src/diamond/handler/test/testtsdb.py | 2 +- src/diamond/handler/tsdb.py | 6 +++--- src/diamond/utils/classes.py | 2 +- src/diamond/utils/config.py | 2 +- src/diamond/utils/log.py | 2 +- 60 files changed, 96 insertions(+), 97 deletions(-) diff --git a/bin/diamond b/bin/diamond index 3cef1ef71..d29959341 100755 --- a/bin/diamond +++ b/bin/diamond @@ -128,10 +128,10 @@ def main(): log = setup_logging(options.configfile, options.log_stdout) # Pass the exit up stream rather then handle it as an general exception - except SystemExit, e: + except SystemExit as e: raise SystemExit - except Exception, e: + except Exception as e: import traceback sys.stderr.write("Unhandled exception: %s" % str(e)) sys.stderr.write("traceback: %s" % traceback.format_exc()) @@ -187,7 +187,7 @@ def main(): pid = str(os.getpid()) try: pf = file(options.pidfile, 'w+') - except IOError, e: + except IOError as e: print("Failed to write PID file: %s" % (e), file=sys.stderr) sys.exit(1) @@ -213,7 +213,7 @@ def main(): # Set UID os.setuid(uid) - except Exception, e: + except Exception as e: print("ERROR: Failed to set UID/GID. %s" % (e), file=sys.stderr) sys.exit(1) @@ -239,7 +239,7 @@ def main(): if pid > 0: # Exit first paren sys.exit(0) - except OSError, e: + except OSError as e: print("Failed to fork process." % (e), file=sys.stderr) sys.exit(1) # Decouple from parent environmen @@ -251,7 +251,7 @@ def main(): if pid > 0: # Exit second paren sys.exit(0) - except OSError, e: + except OSError as e: print("Failed to fork process." % (e), file=sys.stderr) sys.exit(1) # Close file descriptors so that we can detach @@ -272,7 +272,7 @@ def main(): pid = str(os.getpid()) try: pf = file(options.pidfile, 'w+') - except IOError, e: + except IOError as e: log.error("Failed to write child PID file: %s" % (e)) sys.exit(1) pf.write("%s\n" % pid) @@ -320,10 +320,10 @@ def main(): server.run() # Pass the exit up stream rather then handle it as an general exception - except SystemExit, e: + except SystemExit as e: raise SystemExit - except Exception, e: + except Exception as e: import traceback log.error("Unhandled exception: %s" % str(e)) log.error("traceback: %s" % traceback.format_exc()) diff --git a/build_doc.py b/build_doc.py index 0a95220cc..7afb370e3 100755 --- a/build_doc.py +++ b/build_doc.py @@ -173,7 +173,7 @@ def writeDoc(items, type_name, doc_path): default_options = obj.get_default_config() if type_name is "Handler": os.remove(tmpfile[1]) - except Exception, e: + except Exception as e: print("Caught Exception %s" % e) docFile = open(os.path.join(doc_path, item + ".md"), 'w') diff --git a/src/collectors/beanstalkd/beanstalkd.py b/src/collectors/beanstalkd/beanstalkd.py index 27ba7c199..b1156e71e 100644 --- a/src/collectors/beanstalkd/beanstalkd.py +++ b/src/collectors/beanstalkd/beanstalkd.py @@ -51,7 +51,7 @@ def _get_stats(self): try: connection = beanstalkc.Connection(self.config['host'], int(self.config['port'])) - except beanstalkc.BeanstalkcException, e: + except beanstalkc.BeanstalkcException as e: self.log.error("Couldn't connect to beanstalkd: %s", e) return {} diff --git a/src/collectors/bind/bind.py b/src/collectors/bind/bind.py index fa033ec35..59a0e0857 100644 --- a/src/collectors/bind/bind.py +++ b/src/collectors/bind/bind.py @@ -76,7 +76,7 @@ def collect(self): try: req = urllib2.urlopen('http://%s:%d/' % ( self.config['host'], int(self.config['port']))) - except Exception, e: + except Exception as e: self.log.error('Couldnt connect to bind: %s', e) return {} diff --git a/src/collectors/ceph/ceph.py b/src/collectors/ceph/ceph.py index 4aea17e32..6dff5cd9f 100644 --- a/src/collectors/ceph/ceph.py +++ b/src/collectors/ceph/ceph.py @@ -108,7 +108,7 @@ def _get_stats_from_socket(self, name): 'perf', 'dump', ]) - except subprocess.CalledProcessError, err: + except subprocess.CalledProcessError as err: self.log.info('Could not get stats from %s: %s', name, err) self.log.exception('Could not get stats from %s' % name) @@ -116,7 +116,7 @@ def _get_stats_from_socket(self, name): try: json_data = json.loads(json_blob) - except Exception, err: + except Exception as err: self.log.info('Could not parse stats from %s: %s', name, err) self.log.exception('Could not parse stats from %s' % name) diff --git a/src/collectors/cephstats/cephstats.py b/src/collectors/cephstats/cephstats.py index 2f54c9b81..2d31dfd60 100644 --- a/src/collectors/cephstats/cephstats.py +++ b/src/collectors/cephstats/cephstats.py @@ -68,7 +68,7 @@ def _get_stats(self): """ try: output = subprocess.check_output(['ceph', '-s']) - except subprocess.CalledProcessError, err: + except subprocess.CalledProcessError as err: self.log.info( 'Could not get stats: %s' % err) self.log.exception('Could not get stats') diff --git a/src/collectors/chronyd/chronyd.py b/src/collectors/chronyd/chronyd.py index 3b0b7939e..cb4e4a086 100644 --- a/src/collectors/chronyd/chronyd.py +++ b/src/collectors/chronyd/chronyd.py @@ -78,7 +78,7 @@ def collect(self): try: value = diamond.convertor.time.convert(offset, unit, 'ms') - except NotImplementedError, e: + except NotImplementedError as e: self.log.error('Unable to convert %s%s: %s', offset, unit, e) continue diff --git a/src/collectors/diskspace/diskspace.py b/src/collectors/diskspace/diskspace.py index a79526b46..6e3d1c60b 100644 --- a/src/collectors/diskspace/diskspace.py +++ b/src/collectors/diskspace/diskspace.py @@ -201,7 +201,7 @@ def collect(self): if hasattr(os, 'statvfs'): # POSIX try: data = os.statvfs(info['mount_point']) - except OSError, e: + except OSError as e: self.log.exception(e) continue diff --git a/src/collectors/drbd/drbd.py b/src/collectors/drbd/drbd.py index 103f20f10..c7ade8dfa 100644 --- a/src/collectors/drbd/drbd.py +++ b/src/collectors/drbd/drbd.py @@ -75,7 +75,7 @@ def collect(self): else: continue statusfile.close() - except IOError, errormsg: + except IOError as errormsg: self.log.error("Can't read DRBD status file: {0}".format(errormsg)) return diff --git a/src/collectors/dropwizard/dropwizard.py b/src/collectors/dropwizard/dropwizard.py index b116bdc2d..2af875287 100644 --- a/src/collectors/dropwizard/dropwizard.py +++ b/src/collectors/dropwizard/dropwizard.py @@ -46,7 +46,7 @@ def collect(self): self.config['host'], int(self.config['port'])) try: response = urllib2.urlopen(url) - except urllib2.HTTPError, err: + except urllib2.HTTPError as err: self.log.error("%s: %s", url, err) return diff --git a/src/collectors/dseopscenter/dseopscenter.py b/src/collectors/dseopscenter/dseopscenter.py index dce08f1ab..1217101f9 100644 --- a/src/collectors/dseopscenter/dseopscenter.py +++ b/src/collectors/dseopscenter/dseopscenter.py @@ -120,7 +120,7 @@ def _get_schema(self): self.config['cluster_id']) try: response = urllib2.urlopen(url) - except Exception, err: + except Exception as err: self.log.error('%s: %s', url, err) return False @@ -158,7 +158,7 @@ def _get(self, start, end, step=60): try: response = urllib2.urlopen(url) - except Exception, err: + except Exception as err: self.log.error('%s: %s', url, err) return False diff --git a/src/collectors/elasticsearch/elasticsearch.py b/src/collectors/elasticsearch/elasticsearch.py index 534241b36..17ac0d1e2 100755 --- a/src/collectors/elasticsearch/elasticsearch.py +++ b/src/collectors/elasticsearch/elasticsearch.py @@ -117,7 +117,7 @@ def _get(self, scheme, host, port, path, assert_key=None): '%s:%s' % (self.config['user'], self.config['password'])) request.add_header("Authorization", "Basic %s" % base64string) response = urllib2.urlopen(request) - except Exception, err: + except Exception as err: self.log.error("%s: %s" % (url, err)) return False diff --git a/src/collectors/elb/test/testelb.py b/src/collectors/elb/test/testelb.py index 19a293768..ee3b47d20 100644 --- a/src/collectors/elb/test/testelb.py +++ b/src/collectors/elb/test/testelb.py @@ -203,7 +203,7 @@ def assertRaisesAndContains(excClass, contains_str, callableObj, *args, **kwargs): try: callableObj(*args, **kwargs) - except excClass, e: + except excClass as e: msg = str(e) if contains_str in msg: return diff --git a/src/collectors/endecadgraph/endecadgraph.py b/src/collectors/endecadgraph/endecadgraph.py index 3ee022cdc..86a31cc9b 100644 --- a/src/collectors/endecadgraph/endecadgraph.py +++ b/src/collectors/endecadgraph/endecadgraph.py @@ -109,14 +109,14 @@ def walkXML(context, elemList): processElem(elem, elemList) elif event == 'end': elemList.pop() - except Exception, e: + except Exception as e: self.log.error('Something went wrong: %s', e) url = 'http://%s:%d/admin?op=stats' % (self.config['host'], self.config['port']) try: xml = urllib2.urlopen(url, timeout=self.config['timeout']).read() - except Exception, e: + except Exception as e: self.log.error('Could not connect to endeca on %s: %s' % (url, e)) return {} diff --git a/src/collectors/flume/flume.py b/src/collectors/flume/flume.py index 06584eda0..efcebe5f7 100644 --- a/src/collectors/flume/flume.py +++ b/src/collectors/flume/flume.py @@ -83,14 +83,14 @@ def collect(self): try: j = json.loads(resp.read()) resp.close() - except Exception, e: + except Exception as e: resp.close() self.log.error('Cannot load json data: %s', e) return None - except urllib2.URLError, e: + except urllib2.URLError as e: self.log.error('Failed to open url: %s', e) return None - except Exception, e: + except Exception as e: self.log.error('Unknown error opening url: %s', e) return None diff --git a/src/collectors/haproxy/haproxy.py b/src/collectors/haproxy/haproxy.py index e681e5dc3..f4e1963bf 100644 --- a/src/collectors/haproxy/haproxy.py +++ b/src/collectors/haproxy/haproxy.py @@ -68,7 +68,7 @@ def http_get_csv_data(self, section=None): try: handle = urllib2.urlopen(req) return handle.readlines() - except Exception, e: + except Exception as e: if not hasattr(e, 'code') or e.code != 401: self.log.error("Error retrieving HAProxy stats. %s", e) return metrics @@ -104,7 +104,7 @@ def http_get_csv_data(self, section=None): handle = urllib2.urlopen(req) metrics = handle.readlines() return metrics - except IOError, e: + except IOError as e: # here we shouldn't fail if the USER/PASS is right self.log.error("Error retrieving HAProxy stats. " + "(Invalid username or password?) %s", e) @@ -122,7 +122,7 @@ def unix_get_csv_data(self): if not buf: break data += buf - except socket.error, e: + except socket.error as e: self.log.error("Error retrieving HAProxy stats. %s", e) return [] diff --git a/src/collectors/http/http.py b/src/collectors/http/http.py index 282154a59..1f5c73e4a 100644 --- a/src/collectors/http/http.py +++ b/src/collectors/http/http.py @@ -86,8 +86,8 @@ def collect(self): metric_name + '.size', len(the_page)) - except IOError, e: + except IOError as e: self.log.error("Unable to open %s", self.config['req_url']) - except Exception, e: + except Exception as e: self.log.error("Unknown error opening url: %s", e) diff --git a/src/collectors/httpd/httpd.py b/src/collectors/httpd/httpd.py index 4ceeab48c..f4b589a09 100644 --- a/src/collectors/httpd/httpd.py +++ b/src/collectors/httpd/httpd.py @@ -93,7 +93,7 @@ def collect(self): break url = headers['location'] connection.close() - except Exception, e: + except Exception as e: self.log.error( "Error retrieving HTTPD stats for host %s:%s, url '%s': %s", service_host, str(service_port), url, e) diff --git a/src/collectors/jbossapi/jbossapi.py b/src/collectors/jbossapi/jbossapi.py index 70d52c4dc..f3646ae0b 100644 --- a/src/collectors/jbossapi/jbossapi.py +++ b/src/collectors/jbossapi/jbossapi.py @@ -357,7 +357,7 @@ def get_data(self, op_type, current_host, current_port, current_proto, stdout=subprocess.PIPE ).communicate()[0] output = json.loads(attributes) - except Exception, e: + except Exception as e: self.log.error("JbossApiCollector: There was an exception %s", e) output = '' return output diff --git a/src/collectors/jcollectd/jcollectd.py b/src/collectors/jcollectd/jcollectd.py index edeed256c..b651c25ac 100644 --- a/src/collectors/jcollectd/jcollectd.py +++ b/src/collectors/jcollectd/jcollectd.py @@ -131,9 +131,9 @@ def run(self): try: items = rdr.interpret(poll_interval=self.poll_interval) self.send_to_collector(items) - except ValueError, e: + except ValueError as e: self.log.warn('Dropping bad packet: {0}'.format(e)) - except Exception, e: + except Exception as e: self.log.error('caught exception: type={0}, exc={1}'.format(type(e), e)) @@ -149,7 +149,7 @@ def send_to_collector(self, items): self.queue.put(metric) except Queue.Full: self.log.error('Queue to collector is FULL') - except Exception, e: + except Exception as e: self.log.error('B00M! type={0}, exception={1}'.format(type(e), e)) diff --git a/src/collectors/kafkastat/kafkastat.py b/src/collectors/kafkastat/kafkastat.py index fefd3ab9f..d71a0ab5a 100644 --- a/src/collectors/kafkastat/kafkastat.py +++ b/src/collectors/kafkastat/kafkastat.py @@ -68,7 +68,7 @@ def _get(self, path, query_args=None): try: response = urllib2.urlopen(url) - except urllib2.URLError, err: + except urllib2.URLError as err: self.log.error("%s: %s", url, err) return None diff --git a/src/collectors/mesos/mesos.py b/src/collectors/mesos/mesos.py index 596cf8401..8c1e23c1b 100644 --- a/src/collectors/mesos/mesos.py +++ b/src/collectors/mesos/mesos.py @@ -200,7 +200,7 @@ def _get(self, path): url = self._get_url(path) try: response = urllib2.urlopen(url) - except Exception, err: + except Exception as err: self.log.error("%s: %s", url, err) return False diff --git a/src/collectors/mongodb/mongodb.py b/src/collectors/mongodb/mongodb.py index 8c3126ad1..3557626d7 100644 --- a/src/collectors/mongodb/mongodb.py +++ b/src/collectors/mongodb/mongodb.py @@ -175,7 +175,7 @@ def collect(self): ssl=self.config['ssl'], read_preference=ReadPreference.SECONDARY, ) - except Exception, e: + except Exception as e: self.log.error('Couldnt connect to mongodb: %s', e) continue @@ -183,7 +183,6 @@ def collect(self): if user: try: conn.admin.authenticate(user, passwd) - except Exception, e: self.log.error( 'User auth given, but could not autheticate' + ' with host: %s, err: %s' % (host, e)) diff --git a/src/collectors/monit/monit.py b/src/collectors/monit/monit.py index c3bbd5936..abd3e6d92 100644 --- a/src/collectors/monit/monit.py +++ b/src/collectors/monit/monit.py @@ -55,7 +55,7 @@ def collect(self): self.config['user'], self.config['passwd'])).replace('\n', '') request.add_header("Authorization", "Basic %s" % base64string) response = urllib2.urlopen(request) - except urllib2.HTTPError, err: + except urllib2.HTTPError as err: self.log.error("%s: %s", err, url) return diff --git a/src/collectors/mysqlstat/mysql55.py b/src/collectors/mysqlstat/mysql55.py index 3be99a4ea..397edbc5a 100644 --- a/src/collectors/mysqlstat/mysql55.py +++ b/src/collectors/mysqlstat/mysql55.py @@ -133,7 +133,7 @@ def connect(self, params): try: self.db = MySQLdb.connect(**params) - except MySQLError, e: + except MySQLError as e: self.log.error('MySQLPerfCollector couldnt connect to database %s', e) return {} diff --git a/src/collectors/mysqlstat/mysqlstat.py b/src/collectors/mysqlstat/mysqlstat.py index 74969dae1..3fcd1db3a 100644 --- a/src/collectors/mysqlstat/mysqlstat.py +++ b/src/collectors/mysqlstat/mysqlstat.py @@ -296,7 +296,7 @@ def get_db_stats(self, query): try: cursor.execute(query) return cursor.fetchall() - except MySQLError, e: + except MySQLError as e: self.log.error('MySQLCollector could not get db stats', e) return () @@ -304,7 +304,7 @@ def connect(self, params): try: self.db = MySQLdb.connect(**params) self.log.debug('MySQLCollector: Connected to database.') - except MySQLError, e: + except MySQLError as e: self.log.error('MySQLCollector couldnt connect to database %s', e) return False return True @@ -403,7 +403,7 @@ def get_stats(self, params): for key in todo: self.log.debug("MySQLCollector: %s regexp not matched " + "in innodb status", key) - except Exception, innodb_status_error: + except Exception as innodb_status_error: self.log.error('MySQLCollector: Couldnt get engine innodb ' + 'status, check user permissions: %s', innodb_status_error) @@ -472,7 +472,7 @@ def collect(self): try: metrics = self.get_stats(params=params) - except Exception, e: + except Exception as e: try: self.disconnect() except MySQLdb.ProgrammingError: diff --git a/src/collectors/nagiosperfdata/nagiosperfdata.py b/src/collectors/nagiosperfdata/nagiosperfdata.py index 238bb7ee8..6079639e3 100644 --- a/src/collectors/nagiosperfdata/nagiosperfdata.py +++ b/src/collectors/nagiosperfdata/nagiosperfdata.py @@ -201,7 +201,7 @@ def _process_file(self, path): self._process_line(line) os.remove(path) - except IOError, ex: + except IOError as ex: self.log.error("Could not open file `{path}': {error}".format( path=path, error=ex.strerror)) diff --git a/src/collectors/nginx/nginx.py b/src/collectors/nginx/nginx.py index 6e843f64b..8272fab07 100644 --- a/src/collectors/nginx/nginx.py +++ b/src/collectors/nginx/nginx.py @@ -229,7 +229,7 @@ def collect(self): else: self.collect_nginx(handle) - except IOError, e: + except IOError as e: self.log.error("Unable to open %s" % url) - except Exception, e: + except Exception as e: self.log.error("Unknown error opening url: %s", e) diff --git a/src/collectors/openldap/openldap.py b/src/collectors/openldap/openldap.py index 9c972e72c..0a24b18e1 100644 --- a/src/collectors/openldap/openldap.py +++ b/src/collectors/openldap/openldap.py @@ -160,7 +160,7 @@ def collect(self): datapoints = self.get_datapoints(ldap_url, self.config['username'], self.config['password']) - except Exception, e: + except Exception as e: self.log.error('Unable to query %s: %s' % (ldap_url, e)) return {} diff --git a/src/collectors/openvpn/openvpn.py b/src/collectors/openvpn/openvpn.py index 68c5584a9..dcc31c9c8 100644 --- a/src/collectors/openvpn/openvpn.py +++ b/src/collectors/openvpn/openvpn.py @@ -165,7 +165,7 @@ def collect_tcp(self, uri): # Bye server.close() - except socket.error, e: + except socket.error as e: self.log.error('OpenVPN management connection error: %s', e) return diff --git a/src/collectors/ossec/ossec.py b/src/collectors/ossec/ossec.py index 05c9e314a..9c9da1cef 100644 --- a/src/collectors/ossec/ossec.py +++ b/src/collectors/ossec/ossec.py @@ -54,7 +54,7 @@ def collect(self): try: p = subprocess.Popen(command, stdout=subprocess.PIPE) res = p.communicate()[0] - except Exception, e: + except Exception as e: self.log.error('Unable to exec cmd: %s, because %s' % (' '.join(command), str(e))) return diff --git a/src/collectors/phpfpm/phpfpm.py b/src/collectors/phpfpm/phpfpm.py index 636535cb5..4c2763c8e 100644 --- a/src/collectors/phpfpm/phpfpm.py +++ b/src/collectors/phpfpm/phpfpm.py @@ -75,13 +75,13 @@ def collect(self): response = urllib2.urlopen("http://%s:%s/%s?json" % ( self.config['host'], int(self.config['port']), self.config['uri'])) - except Exception, e: + except Exception as e: self.log.error('Couldnt connect to php-fpm status page: %s', e) return {} try: j = json.loads(response.read()) - except Exception, e: + except Exception as e: self.log.error('Couldnt parse json: %s', e) return {} diff --git a/src/collectors/postgres/postgres.py b/src/collectors/postgres/postgres.py index a2c19db5d..613d4339a 100644 --- a/src/collectors/postgres/postgres.py +++ b/src/collectors/postgres/postgres.py @@ -170,7 +170,7 @@ def _connect(self, database=None): try: conn = psycopg2.connect(**conn_args) - except Exception, e: + except Exception as e: self.log.error(e) raise e diff --git a/src/collectors/processresources/processresources.py b/src/collectors/processresources/processresources.py index 618a43dc5..05d7636d9 100644 --- a/src/collectors/processresources/processresources.py +++ b/src/collectors/processresources/processresources.py @@ -181,7 +181,7 @@ def collect_process_info(self, process): uptime = time.time() - get_value(process, 'create_time') pi.update({'uptime': uptime}) self.save_process_info(pg_name, pi) - except psutil.NoSuchProcess, e: + except psutil.NoSuchProcess as e: self.log.info("Process exited while trying to get info: %s", e) def collect(self): diff --git a/src/collectors/puppetdashboard/puppetdashboard.py b/src/collectors/puppetdashboard/puppetdashboard.py index 81ca10804..b07e38159 100644 --- a/src/collectors/puppetdashboard/puppetdashboard.py +++ b/src/collectors/puppetdashboard/puppetdashboard.py @@ -42,7 +42,7 @@ def collect(self): try: response = urllib2.urlopen("http://%s:%s/" % ( self.config['host'], int(self.config['port']))) - except Exception, e: + except Exception as e: self.log.error('Couldnt connect to puppet-dashboard: %s', e) return {} @@ -59,5 +59,5 @@ def collect(self): results = r.groupdict() self.publish(results['key'], results['count']) - except Exception, e: + except Exception as e: self.log.error('Couldnt parse the output: %s', e) diff --git a/src/collectors/puppetdb/puppetdb.py b/src/collectors/puppetdb/puppetdb.py index b9ad400ce..34320bc1b 100644 --- a/src/collectors/puppetdb/puppetdb.py +++ b/src/collectors/puppetdb/puppetdb.py @@ -91,7 +91,7 @@ def fetch_metrics(self, url): url = "http://%s:%s/%s" % ( self.config['host'], int(self.config['port']), url) response = urllib2.urlopen(url) - except Exception, e: + except Exception as e: self.log.error('Couldn\'t connect to puppetdb: %s -> %s', url, e) return {} return json.load(response) diff --git a/src/collectors/rabbitmq/rabbitmq.py b/src/collectors/rabbitmq/rabbitmq.py index 8084fa7d4..c9e94ae8b 100644 --- a/src/collectors/rabbitmq/rabbitmq.py +++ b/src/collectors/rabbitmq/rabbitmq.py @@ -66,7 +66,7 @@ def get_queue(self, vhost, queue_name): try: queue = self.do_call(path) return queue or None - except Exception, e: + except Exception as e: self.log.error('Error querying queue %s/%s: %s' % ( vhost, queue_name, e )) @@ -80,7 +80,7 @@ def get_queues(self, vhost): try: queues = self.do_call(path) return queues or [] - except Exception, e: + except Exception as e: self.log.error('Error querying queues %s: %s' % ( vhost, e )) diff --git a/src/collectors/redisstat/redisstat.py b/src/collectors/redisstat/redisstat.py index 35fb8304d..9ca94736c 100644 --- a/src/collectors/redisstat/redisstat.py +++ b/src/collectors/redisstat/redisstat.py @@ -221,7 +221,7 @@ def _client(self, host, port, unix_socket, auth): unix_socket_path=unix_socket) cli.ping() return cli - except Exception, ex: + except Exception as ex: self.log.error("RedisCollector: failed to connect to %s:%i. %s.", unix_socket or host, port, ex) diff --git a/src/collectors/resqueweb/resqueweb.py b/src/collectors/resqueweb/resqueweb.py index 8258d0d6c..1b7606e20 100644 --- a/src/collectors/resqueweb/resqueweb.py +++ b/src/collectors/resqueweb/resqueweb.py @@ -37,7 +37,7 @@ def collect(self): try: response = urllib2.urlopen("http://%s:%s/stats.txt" % ( self.config['host'], int(self.config['port']))) - except Exception, e: + except Exception as e: self.log.error('Couldnt connect to resque-web: %s', e) return {} @@ -59,5 +59,5 @@ def collect(self): else: self.publish("queue.%s.current" % queue, count) - except Exception, e: + except Exception as e: self.log.error('Couldnt parse the queue: %s', e) diff --git a/src/collectors/sidekiqweb/sidekiqweb.py b/src/collectors/sidekiqweb/sidekiqweb.py index 1a9acd833..52ef9be1b 100644 --- a/src/collectors/sidekiqweb/sidekiqweb.py +++ b/src/collectors/sidekiqweb/sidekiqweb.py @@ -44,13 +44,13 @@ def collect(self): try: response = urllib2.urlopen("http://%s:%s/dashboard/stats" % ( self.config['host'], int(self.config['port']))) - except Exception, e: + except Exception as e: self.log.error('Couldnt connect to sidekiq-web: %s', e) return {} try: j = json.loads(response.read()) - except Exception, e: + except Exception as e: self.log.error('Couldnt parse json: %s', e) return {} diff --git a/src/collectors/solr/solr.py b/src/collectors/solr/solr.py index d08ecbc7d..43d5f4c3f 100644 --- a/src/collectors/solr/solr.py +++ b/src/collectors/solr/solr.py @@ -76,7 +76,7 @@ def _get(self, path): self.config['host'], int(self.config['port']), path) try: response = urllib2.urlopen(url) - except Exception, err: + except Exception as err: self.log.error("%s: %s", url, err) return False diff --git a/src/collectors/squid/squid.py b/src/collectors/squid/squid.py index 90697d006..d5afe3119 100644 --- a/src/collectors/squid/squid.py +++ b/src/collectors/squid/squid.py @@ -79,7 +79,7 @@ def _getData(self, host, port): if not data: break fulldata = fulldata + data - except Exception, e: + except Exception as e: self.log.error('Couldnt connect to squid: %s', e) return None squid_sock.close() diff --git a/src/collectors/tokumx/tokumx.py b/src/collectors/tokumx/tokumx.py index fd7123939..736a2d3e8 100644 --- a/src/collectors/tokumx/tokumx.py +++ b/src/collectors/tokumx/tokumx.py @@ -125,7 +125,7 @@ def collect(self): network_timeout=self.config['network_timeout'], read_preference=ReadPreference.SECONDARY, ) - except Exception, e: + except Exception as e: self.log.error('Couldnt connect to mongodb: %s', e) continue @@ -133,7 +133,7 @@ def collect(self): if user: try: conn.admin.authenticate(user, passwd) - except Exception, e: + except Exception as e: self.log.error( 'User auth given, but could not autheticate' + ' with host: %s, err: %s' % (host, e)) diff --git a/src/collectors/uptime/uptime.py b/src/collectors/uptime/uptime.py index 02a0c55ec..4525ff3d1 100644 --- a/src/collectors/uptime/uptime.py +++ b/src/collectors/uptime/uptime.py @@ -42,7 +42,7 @@ def read(self): fd.close() v = float(uptime.split()[0].strip()) return convertor.time.convert(v, 's', self.config['metric_name']) - except Exception, e: + except Exception as e: self.log.error('Unable to read uptime from %s: %s' % (self.PROC, e)) return None diff --git a/src/collectors/userscripts/userscripts.py b/src/collectors/userscripts/userscripts.py index 7f2d77ec7..4e2174d02 100644 --- a/src/collectors/userscripts/userscripts.py +++ b/src/collectors/userscripts/userscripts.py @@ -71,7 +71,7 @@ def collect(self): stdout=subprocess.PIPE, stderr=subprocess.PIPE) (out, err) = proc.communicate() - except subprocess.CalledProcessError, e: + except subprocess.CalledProcessError as e: self.log.error("%s error launching: %s; skipping" % (absolutescriptpath, e)) continue diff --git a/src/collectors/websitemonitor/websitemonitor.py b/src/collectors/websitemonitor/websitemonitor.py index a75371548..af0723469 100644 --- a/src/collectors/websitemonitor/websitemonitor.py +++ b/src/collectors/websitemonitor/websitemonitor.py @@ -60,7 +60,7 @@ def collect(self): self.publish('response_time.%s' % (resp.code), rt, metric_type='COUNTER') # urllib2 will puke on non HTTP 200/OK URLs - except urllib2.URLError, e: + except urllib2.URLError as e: if e.code != 200: # time in seconds since epoch as a floating number end_time = time.time() @@ -71,8 +71,8 @@ def collect(self): self.publish('response_time.%s' % (e.code), rt, metric_type='COUNTER') - except IOError, e: + except IOError as e: self.log.error('Unable to open %s' % (self.config['URL'])) - except Exception, e: + except Exception as e: self.log.error("Unknown error opening url: %s", e) diff --git a/src/diamond/handler/cloudwatch.py b/src/diamond/handler/cloudwatch.py index d5491c87a..ebdf0d862 100644 --- a/src/diamond/handler/cloudwatch.py +++ b/src/diamond/handler/cloudwatch.py @@ -254,10 +254,10 @@ def send_metrics_to_cloudwatch(self, rule, metric, dimensions): rule['namespace'], str(metric.value), str(dimensions)) - except AttributeError, e: + except AttributeError as e: self.log.error( "CloudWatch: Failed publishing - %s ", str(e)) - except Exception, e: # Rough connection re-try logic. + except Exception as e: # Rough connection re-try logic. self.log.error( "CloudWatch: Failed publishing - %s\n%s ", str(e), diff --git a/src/diamond/handler/graphite.py b/src/diamond/handler/graphite.py index 006a26a7c..87d0cae1e 100644 --- a/src/diamond/handler/graphite.py +++ b/src/diamond/handler/graphite.py @@ -202,7 +202,7 @@ def _connect(self): connection_struct = (self.host, self.port) try: addrinfo = socket.getaddrinfo(self.host, self.port, 0, stream) - except socket.gaierror, ex: + except socket.gaierror as ex: self.log.error("GraphiteHandler: Error looking up graphite host" " '%s' - %s", self.host, ex) @@ -242,7 +242,7 @@ def _connect(self): "graphite server %s:%d.", self.host, self.port) self.last_connect_timestamp = time.time() - except Exception, ex: + except Exception as ex: # Log Error self._throttle_error("GraphiteHandler: Failed to connect to " "%s:%i. %s.", self.host, self.port, ex) diff --git a/src/diamond/handler/influxdbHandler.py b/src/diamond/handler/influxdbHandler.py index 1fa8463cb..7f3af4b3c 100644 --- a/src/diamond/handler/influxdbHandler.py +++ b/src/diamond/handler/influxdbHandler.py @@ -207,7 +207,7 @@ def _connect(self): self.log.debug("InfluxdbHandler: Established connection to " "%s:%d/%s.", self.hostname, self.port, self.database) - except Exception, ex: + except Exception as ex: # Log Error self._throttle_error("InfluxdbHandler: Failed to connect to " "%s:%d/%s. %s", diff --git a/src/diamond/handler/logentries_diamond.py b/src/diamond/handler/logentries_diamond.py index f145b2e68..59cd81593 100644 --- a/src/diamond/handler/logentries_diamond.py +++ b/src/diamond/handler/logentries_diamond.py @@ -79,5 +79,5 @@ def _send(self): self.log_token, msg) try: urllib2.urlopen(req) - except urllib2.URLError, e: + except urllib2.URLError as e: logging.error("Can't send log message to Logentries %s", e) diff --git a/src/diamond/handler/mysql.py b/src/diamond/handler/mysql.py index fc1b8a718..ee4e188ab 100644 --- a/src/diamond/handler/mysql.py +++ b/src/diamond/handler/mysql.py @@ -83,7 +83,7 @@ def _send(self, data): (data[0], data[2], data[1])) cursor.close() self.conn.commit() - except BaseException, e: + except BaseException as e: # Log Error self.log.error("MySQLHandler: Failed sending data. %s.", e) # Attempt to restablish connection diff --git a/src/diamond/handler/rabbitmq_pubsub.py b/src/diamond/handler/rabbitmq_pubsub.py index 58b936d16..974824f59 100644 --- a/src/diamond/handler/rabbitmq_pubsub.py +++ b/src/diamond/handler/rabbitmq_pubsub.py @@ -156,7 +156,7 @@ def _bind(self, rmq_server): durable=self.rmq_durable) # Reset reconnect_interval after a successful connection self.reconnect_interval = 1 - except Exception, exception: + except Exception as exception: self.log.debug("Caught exception in _bind: %s", exception) if rmq_server in self.connections.keys(): self._unbind(rmq_server) @@ -200,7 +200,7 @@ def process(self, metric): channel = self.channels[rmq_server] channel.basic_publish(exchange=self.rmq_exchange, routing_key='', body="%s" % metric) - except Exception, exception: + except Exception as exception: self.log.error( "Failed publishing to %s, attempting reconnect", rmq_server) diff --git a/src/diamond/handler/riemann.py b/src/diamond/handler/riemann.py index c8d392e08..02943dc7a 100644 --- a/src/diamond/handler/riemann.py +++ b/src/diamond/handler/riemann.py @@ -87,7 +87,7 @@ def process(self, metric): event = self._metric_to_riemann_event(metric) try: self.client.send_event(event) - except Exception, e: + except Exception as e: self.log.error( "RiemannHandler: Error sending event to Riemann: %s", e) diff --git a/src/diamond/handler/sentry.py b/src/diamond/handler/sentry.py index 7f8eed9f4..821d74643 100644 --- a/src/diamond/handler/sentry.py +++ b/src/diamond/handler/sentry.py @@ -333,7 +333,7 @@ def compile_section(self, section): # init rule try: return Rule(**kwargs) - except InvalidRule, err: + except InvalidRule as err: self.log.error(str(err)) def configure_sentry_errors(self): diff --git a/src/diamond/handler/statsite.py b/src/diamond/handler/statsite.py index 5b76c0fec..60f18381f 100644 --- a/src/diamond/handler/statsite.py +++ b/src/diamond/handler/statsite.py @@ -148,7 +148,7 @@ def _send(self, data): self.socket.sendall(data) # Done break - except socket.error, e: + except socket.error as e: # Log Error self.log.error("StatsiteHandler: Failed sending data. %s.", e) # Attempt to restablish connection @@ -183,7 +183,7 @@ def _connect(self): # Log self.log.debug("Established connection to statsite server %s:%d", self.host, self.port) - except Exception, ex: + except Exception as ex: # Log Error self.log.error("StatsiteHandler: Failed to connect to %s:%i. %s", self.host, self.port, ex) diff --git a/src/diamond/handler/test/testtsdb.py b/src/diamond/handler/test/testtsdb.py index e9cf6b721..8e7133a01 100644 --- a/src/diamond/handler/test/testtsdb.py +++ b/src/diamond/handler/test/testtsdb.py @@ -427,5 +427,5 @@ def test_with_invalid_tag(self): try: TSDBHandler(config) fail("Expected an exception") - except Exception, e: + except Exception as e: assert(e) diff --git a/src/diamond/handler/tsdb.py b/src/diamond/handler/tsdb.py index 9ba4be42e..c67471733 100644 --- a/src/diamond/handler/tsdb.py +++ b/src/diamond/handler/tsdb.py @@ -1,4 +1,4 @@ -# coding=utf-8 +# coding=utf-8 """ Send metrics to a [OpenTSDB](http://opentsdb.net/) server. @@ -187,7 +187,7 @@ def _send(self, data): self.socket.sendall(data) # Done break - except socket.error, e: + except socket.error as e: # Log Error self.log.error("TSDBHandler: Failed sending data. %s.", e) # Attempt to restablish connection @@ -217,7 +217,7 @@ def _connect(self): # Log self.log.debug("Established connection to TSDB server %s:%d", self.host, self.port) - except Exception, ex: + except Exception as ex: # Log Error self.log.error("TSDBHandler: Failed to connect to %s:%i. %s", self.host, self.port, ex) diff --git a/src/diamond/utils/classes.py b/src/diamond/utils/classes.py index ed99d776e..51613cf11 100644 --- a/src/diamond/utils/classes.py +++ b/src/diamond/utils/classes.py @@ -157,7 +157,7 @@ def load_collectors_from_paths(paths): try: # Import the module mod = imp.load_module(modname, fp, pathname, description) - except (KeyboardInterrupt, SystemExit), err: + except (KeyboardInterrupt, SystemExit) as err: logger.error( "System or keyboard interrupt " "while loading module %s" diff --git a/src/diamond/utils/config.py b/src/diamond/utils/config.py index 7a5cd2d67..f7dec11e8 100644 --- a/src/diamond/utils/config.py +++ b/src/diamond/utils/config.py @@ -101,7 +101,7 @@ def load_config(configfile): try: newconfig = configobj.ConfigObj(cfgfile) - except Exception, e: + except Exception as e: raise Exception("Failed to load config file %s due to %s" % (cfgfile, e)) diff --git a/src/diamond/utils/log.py b/src/diamond/utils/log.py index 70f8be52e..f9e94c8d9 100644 --- a/src/diamond/utils/log.py +++ b/src/diamond/utils/log.py @@ -58,7 +58,7 @@ def setup_logging(configfile, stdout=False): streamHandler.setLevel(rootLogLevel) log.addHandler(streamHandler) - except Exception, e: + except Exception as e: sys.stderr.write("Error occurs when initialize logging: ") sys.stderr.write(str(e)) sys.stderr.write(os.linesep) From a4572f3ddaf4ddc74faa3c5bf149311ef8133ddf Mon Sep 17 00:00:00 2001 From: Hannu Valtonen Date: Mon, 24 Aug 2015 14:08:36 +0300 Subject: [PATCH 03/33] tests: import Mock and patch from the test base file --- src/collectors/aerospike/test/testaerospike27.py | 5 +++-- src/collectors/aerospike/test/testaerospike39.py | 5 +++-- src/collectors/amavis/test/testamavis.py | 2 +- src/collectors/apcupsd/test/testapcupsd.py | 4 ++-- src/collectors/aurora/test/testaurora.py | 4 ++-- src/collectors/beanstalkd/test/testbeanstalkd.py | 4 ++-- src/collectors/bind/test/testbind.py | 4 ++-- src/collectors/ceph/test/testceph.py | 3 ++- src/collectors/chronyd/test/testchronyd.py | 4 ++-- src/collectors/conntrack/test/testconntrack.py | 4 ++-- src/collectors/cpu/test/testcpu.py | 4 ++-- .../cpuacct_cgroup/test/testcpuacct_cgroup.py | 2 +- src/collectors/darner/test/testdarner.py | 4 ++-- src/collectors/diskspace/test/testdiskspace.py | 4 ++-- src/collectors/disktemp/test/testdisktemp.py | 4 ++-- src/collectors/diskusage/test/testdiskusage.py | 4 ++-- src/collectors/dropwizard/test/testdropwizard.py | 4 ++-- src/collectors/dseopscenter/test/testdseopscenter.py | 4 ++-- .../elasticsearch/test/testelasticsearch.py | 4 ++-- src/collectors/elb/test/testelb.py | 4 ++-- src/collectors/endecadgraph/test/testendecadgraph.py | 2 +- src/collectors/etcdstat/test/test_etcdstat.py | 4 ++-- .../tests/testeventstoreprojections.py | 3 ++- src/collectors/example/test/testexample.py | 2 +- src/collectors/exim/test/testexim.py | 4 ++-- src/collectors/filestat/test/testfilestat.py | 4 ++-- src/collectors/flume/test/testflume.py | 4 ++-- src/collectors/gridengine/test/testgridengine.py | 2 +- src/collectors/hadoop/test/testhadoop.py | 2 +- src/collectors/haproxy/test/testhaproxy.py | 4 ++-- src/collectors/http/test/testhttp.py | 4 ++-- src/collectors/httpd/test/testhttpd.py | 4 ++-- src/collectors/httpjson/test/testhttpjson.py | 4 ++-- src/collectors/interrupt/test/testinterrupt.py | 4 ++-- src/collectors/interrupt/test/testsoft.py | 4 ++-- src/collectors/ip/test/testip.py | 4 ++-- src/collectors/ipmisensor/test/testipmisensor.py | 4 ++-- src/collectors/ipvs/test/testipvs.py | 4 ++-- src/collectors/jolokia/test/testcassandra_jolokia.py | 2 +- src/collectors/jolokia/test/testjolokia.py | 4 ++-- .../test/testkafka_consumer_lag.py | 3 ++- src/collectors/kafkastat/test/testkafka.py | 2 +- src/collectors/ksm/test/testksm.py | 4 ++-- src/collectors/kvm/test/testkvm.py | 4 ++-- src/collectors/lmsensors/test/testlmsensors.py | 2 +- src/collectors/loadavg/test/testloadavg.py | 4 ++-- src/collectors/mdstat/test/test_mdstat.py | 4 ++-- src/collectors/memcached/test/testmemcached.py | 4 ++-- src/collectors/memory/test/testmemory.py | 4 ++-- .../memory_cgroup/test/testmemory_cgroup.py | 4 ++-- .../memory_docker/test/testmemory_docker.py | 4 ++-- src/collectors/mesos/test/testmesos.py | 4 ++-- src/collectors/mesos_cgroup/test/testmesos_cgroup.py | 5 +++-- src/collectors/mogilefs/test/testmogilefs.py | 5 +++-- src/collectors/mongodb/mongodb.py | 1 + src/collectors/mongodb/test/testmongodb.py | 2 +- src/collectors/monit/test/testmonit.py | 4 ++-- src/collectors/mountstats/test/testmountstats.py | 5 +++-- src/collectors/mysqlstat/test/testmysql.py | 4 ++-- src/collectors/nagios/test/testnagios.py | 4 ++-- .../nagiosperfdata/test/testnagiosperfdata.py | 2 +- src/collectors/netstat/test/testnetstat.py | 2 +- src/collectors/network/test/testnetwork.py | 4 ++-- src/collectors/nfacct/test/testnfacct.py | 4 ++-- src/collectors/nfs/test/testnfs.py | 4 ++-- src/collectors/nfsd/test/testnfsd.py | 4 ++-- src/collectors/nginx/test/testnginx.py | 4 ++-- src/collectors/ntp/test/testntp.py | 4 ++-- src/collectors/ntpd/test/testntpd.py | 4 ++-- src/collectors/numa/test/testnuma.py | 4 ++-- src/collectors/nvidia_gpu/test/testnvidia_gpu.py | 3 ++- src/collectors/onewire/test/testonewire.py | 2 +- .../test/testopenstackswiftrecon.py | 2 +- src/collectors/openvpn/test/testopenvpn.py | 2 +- src/collectors/openvz/test/testopenvz.py | 4 ++-- src/collectors/pgbouncer/test/testpgbouncer.py | 2 +- src/collectors/pgq/test/testpgq.py | 3 ++- src/collectors/phpfpm/test/testphpfpm.py | 4 ++-- src/collectors/ping/test/testping.py | 4 ++-- src/collectors/portstat/tests/test_portstat.py | 4 +++- src/collectors/postfix/test/testpostfix.py | 4 ++-- src/collectors/postgres/test/testpostgres.py | 4 ++-- src/collectors/postqueue/test/testpostqueue.py | 4 ++-- src/collectors/powerdns/test/testpowerdns.py | 4 ++-- src/collectors/proc/test/testproc.py | 4 ++-- .../processresources/test/testprocessresources.py | 2 +- src/collectors/puppetagent/test/testpuppetagent.py | 2 +- .../puppetdashboard/test/testpuppetdashboard.py | 4 ++-- src/collectors/rabbitmq/test/testrabbitmq.py | 4 ++-- src/collectors/redisstat/test/testredisstat.py | 5 +++-- src/collectors/resqueweb/test/testresqueweb.py | 4 ++-- src/collectors/scribe/test/test_scribe.py | 2 +- src/collectors/sidekiq/test/testsidekiq.py | 4 ++-- src/collectors/sidekiqweb/test/testsidekiqweb.py | 4 ++-- src/collectors/slabinfo/test/testslabinfo.py | 4 ++-- src/collectors/slony/test/testslony.py | 2 +- src/collectors/smart/test/testsmart.py | 4 ++-- src/collectors/snmpraw/test/testsnmpraw.py | 4 ++-- src/collectors/sockstat/test/testsockstat.py | 4 ++-- src/collectors/solr/test/testsolr.py | 3 ++- src/collectors/squid/test/testsquid.py | 4 ++-- src/collectors/supervisord/test/testsupervisord.py | 4 ++-- src/collectors/tcp/test/testtcp.py | 4 ++-- src/collectors/tokumx/test/testtokumx.py | 2 +- src/collectors/twemproxy/test/testtwemproxy.py | 4 ++-- src/collectors/udp/test/testudp.py | 4 ++-- src/collectors/unbound/test/testunbound.py | 4 ++-- src/collectors/ups/test/testups.py | 4 ++-- src/collectors/uptime/test/test_uptime.py | 4 ++-- src/collectors/users/test/testusers.py | 2 +- src/collectors/userscripts/test/testuserscripts.py | 2 +- src/collectors/varnish/test/testvarnish.py | 4 ++-- src/collectors/vmstat/test/testvmstat.py | 4 ++-- .../websitemonitor/test/testwebsitemonitor.py | 2 +- src/collectors/xen_collector/test/testxen.py | 4 ++-- src/collectors/xfs/test/testxfs.py | 4 ++-- src/diamond/handler/test/testgraphitehandler.py | 4 ++-- src/diamond/handler/test/testriemann.py | 4 ++-- src/diamond/handler/test/teststatsdhandler.py | 3 ++- src/diamond/handler/test/testtsdb.py | 2 +- src/diamond/test/testcollector.py | 2 +- test.py | 12 +++++++++++- 122 files changed, 233 insertions(+), 207 deletions(-) diff --git a/src/collectors/aerospike/test/testaerospike27.py b/src/collectors/aerospike/test/testaerospike27.py index 1ff6b2c60..342e9a649 100644 --- a/src/collectors/aerospike/test/testaerospike27.py +++ b/src/collectors/aerospike/test/testaerospike27.py @@ -5,8 +5,9 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock, call -from mock import patch +from mock import call +from test import Mock +from test import patch from diamond.collector import Collector diff --git a/src/collectors/aerospike/test/testaerospike39.py b/src/collectors/aerospike/test/testaerospike39.py index 1319b80eb..d75d40f7b 100644 --- a/src/collectors/aerospike/test/testaerospike39.py +++ b/src/collectors/aerospike/test/testaerospike39.py @@ -5,8 +5,9 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock, call -from mock import patch +from mock import call +from test import Mock +from test import patch from diamond.collector import Collector diff --git a/src/collectors/amavis/test/testamavis.py b/src/collectors/amavis/test/testamavis.py index cf40ddc02..38829b163 100644 --- a/src/collectors/amavis/test/testamavis.py +++ b/src/collectors/amavis/test/testamavis.py @@ -7,7 +7,7 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import patch +from test import patch import amavis from diamond.collector import Collector diff --git a/src/collectors/apcupsd/test/testapcupsd.py b/src/collectors/apcupsd/test/testapcupsd.py index ff3d28367..d74e5406c 100644 --- a/src/collectors/apcupsd/test/testapcupsd.py +++ b/src/collectors/apcupsd/test/testapcupsd.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from apcupsd import ApcupsdCollector diff --git a/src/collectors/aurora/test/testaurora.py b/src/collectors/aurora/test/testaurora.py index 5b4fe017a..b0fdf29c0 100644 --- a/src/collectors/aurora/test/testaurora.py +++ b/src/collectors/aurora/test/testaurora.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector diff --git a/src/collectors/beanstalkd/test/testbeanstalkd.py b/src/collectors/beanstalkd/test/testbeanstalkd.py index a6122ff3e..c8d80e212 100644 --- a/src/collectors/beanstalkd/test/testbeanstalkd.py +++ b/src/collectors/beanstalkd/test/testbeanstalkd.py @@ -6,8 +6,8 @@ from test import get_collector_config from test import unittest from test import run_only -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from beanstalkd import BeanstalkdCollector diff --git a/src/collectors/bind/test/testbind.py b/src/collectors/bind/test/testbind.py index 10f31714d..e2d12e6cd 100644 --- a/src/collectors/bind/test/testbind.py +++ b/src/collectors/bind/test/testbind.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from bind import BindCollector diff --git a/src/collectors/ceph/test/testceph.py b/src/collectors/ceph/test/testceph.py index 8095f7e17..3c611ecef 100644 --- a/src/collectors/ceph/test/testceph.py +++ b/src/collectors/ceph/test/testceph.py @@ -12,7 +12,8 @@ from test import get_collector_config from test import unittest from test import run_only -from mock import patch, call +from test import patch +from mock import call from diamond.collector import Collector import ceph diff --git a/src/collectors/chronyd/test/testchronyd.py b/src/collectors/chronyd/test/testchronyd.py index e819d3162..1fc47c086 100644 --- a/src/collectors/chronyd/test/testchronyd.py +++ b/src/collectors/chronyd/test/testchronyd.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from chronyd import ChronydCollector diff --git a/src/collectors/conntrack/test/testconntrack.py b/src/collectors/conntrack/test/testconntrack.py index c98b362f4..e15eb5bf6 100644 --- a/src/collectors/conntrack/test/testconntrack.py +++ b/src/collectors/conntrack/test/testconntrack.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from conntrack import ConnTrackCollector diff --git a/src/collectors/cpu/test/testcpu.py b/src/collectors/cpu/test/testcpu.py index 44336e20b..34ac47209 100644 --- a/src/collectors/cpu/test/testcpu.py +++ b/src/collectors/cpu/test/testcpu.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch try: from cStringIO import StringIO diff --git a/src/collectors/cpuacct_cgroup/test/testcpuacct_cgroup.py b/src/collectors/cpuacct_cgroup/test/testcpuacct_cgroup.py index 4e2430262..b152c727b 100644 --- a/src/collectors/cpuacct_cgroup/test/testcpuacct_cgroup.py +++ b/src/collectors/cpuacct_cgroup/test/testcpuacct_cgroup.py @@ -5,7 +5,7 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import patch +from test import patch try: from cStringIO import StringIO diff --git a/src/collectors/darner/test/testdarner.py b/src/collectors/darner/test/testdarner.py index e301f3681..bac2ba097 100644 --- a/src/collectors/darner/test/testdarner.py +++ b/src/collectors/darner/test/testdarner.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from darner import DarnerCollector diff --git a/src/collectors/diskspace/test/testdiskspace.py b/src/collectors/diskspace/test/testdiskspace.py index 522f47ad3..8968abc0d 100644 --- a/src/collectors/diskspace/test/testdiskspace.py +++ b/src/collectors/diskspace/test/testdiskspace.py @@ -6,8 +6,8 @@ from test import get_collector_config from test import unittest from test import run_only -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from diskspace import DiskSpaceCollector diff --git a/src/collectors/disktemp/test/testdisktemp.py b/src/collectors/disktemp/test/testdisktemp.py index 4616218eb..2b7c85a0c 100644 --- a/src/collectors/disktemp/test/testdisktemp.py +++ b/src/collectors/disktemp/test/testdisktemp.py @@ -4,8 +4,8 @@ from test import CollectorTestCase from test import get_collector_config -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from disktemp import DiskTemperatureCollector diff --git a/src/collectors/diskusage/test/testdiskusage.py b/src/collectors/diskusage/test/testdiskusage.py index ff5030048..626b1941d 100644 --- a/src/collectors/diskusage/test/testdiskusage.py +++ b/src/collectors/diskusage/test/testdiskusage.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from diskusage import DiskUsageCollector diff --git a/src/collectors/dropwizard/test/testdropwizard.py b/src/collectors/dropwizard/test/testdropwizard.py index d143b2821..d4246bda0 100644 --- a/src/collectors/dropwizard/test/testdropwizard.py +++ b/src/collectors/dropwizard/test/testdropwizard.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector diff --git a/src/collectors/dseopscenter/test/testdseopscenter.py b/src/collectors/dseopscenter/test/testdseopscenter.py index c9f95d351..f654fe749 100644 --- a/src/collectors/dseopscenter/test/testdseopscenter.py +++ b/src/collectors/dseopscenter/test/testdseopscenter.py @@ -4,8 +4,8 @@ from test import CollectorTestCase from test import get_collector_config -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector diff --git a/src/collectors/elasticsearch/test/testelasticsearch.py b/src/collectors/elasticsearch/test/testelasticsearch.py index 814114d19..80cf410b0 100644 --- a/src/collectors/elasticsearch/test/testelasticsearch.py +++ b/src/collectors/elasticsearch/test/testelasticsearch.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector diff --git a/src/collectors/elb/test/testelb.py b/src/collectors/elb/test/testelb.py index ee3b47d20..a88af31bf 100644 --- a/src/collectors/elb/test/testelb.py +++ b/src/collectors/elb/test/testelb.py @@ -7,9 +7,9 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import patch +from test import patch from test import run_only -from mock import Mock +from test import Mock from diamond.collector import Collector from elb import ElbCollector diff --git a/src/collectors/endecadgraph/test/testendecadgraph.py b/src/collectors/endecadgraph/test/testendecadgraph.py index 3ff538192..1d12f688a 100644 --- a/src/collectors/endecadgraph/test/testendecadgraph.py +++ b/src/collectors/endecadgraph/test/testendecadgraph.py @@ -4,7 +4,7 @@ from test import CollectorTestCase from test import get_collector_config -from mock import patch +from test import patch from diamond.collector import Collector from endecadgraph import EndecaDgraphCollector diff --git a/src/collectors/etcdstat/test/test_etcdstat.py b/src/collectors/etcdstat/test/test_etcdstat.py index abb9d9711..8a2a6358a 100644 --- a/src/collectors/etcdstat/test/test_etcdstat.py +++ b/src/collectors/etcdstat/test/test_etcdstat.py @@ -6,8 +6,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import patch -from mock import Mock +from test import patch +from test import Mock from etcdstat import EtcdCollector diff --git a/src/collectors/eventstoreprojections/tests/testeventstoreprojections.py b/src/collectors/eventstoreprojections/tests/testeventstoreprojections.py index 211560be6..33d89be0f 100644 --- a/src/collectors/eventstoreprojections/tests/testeventstoreprojections.py +++ b/src/collectors/eventstoreprojections/tests/testeventstoreprojections.py @@ -5,7 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import call, patch +from test import patch +from mock import call from diamond.collector import Collector diff --git a/src/collectors/example/test/testexample.py b/src/collectors/example/test/testexample.py index 4af108657..51c6346fd 100644 --- a/src/collectors/example/test/testexample.py +++ b/src/collectors/example/test/testexample.py @@ -5,7 +5,7 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import patch +from test import patch from diamond.collector import Collector from example import ExampleCollector diff --git a/src/collectors/exim/test/testexim.py b/src/collectors/exim/test/testexim.py index 650147d99..48d91445c 100644 --- a/src/collectors/exim/test/testexim.py +++ b/src/collectors/exim/test/testexim.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from exim import EximCollector diff --git a/src/collectors/filestat/test/testfilestat.py b/src/collectors/filestat/test/testfilestat.py index 42f18eadf..c54feecfd 100644 --- a/src/collectors/filestat/test/testfilestat.py +++ b/src/collectors/filestat/test/testfilestat.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch try: from cStringIO import StringIO diff --git a/src/collectors/flume/test/testflume.py b/src/collectors/flume/test/testflume.py index b19cc51f5..5d38a68b9 100644 --- a/src/collectors/flume/test/testflume.py +++ b/src/collectors/flume/test/testflume.py @@ -4,8 +4,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import patch -from mock import Mock +from test import patch +from test import Mock from diamond.collector import Collector from flume import FlumeCollector diff --git a/src/collectors/gridengine/test/testgridengine.py b/src/collectors/gridengine/test/testgridengine.py index c84c434e5..870854868 100644 --- a/src/collectors/gridengine/test/testgridengine.py +++ b/src/collectors/gridengine/test/testgridengine.py @@ -3,7 +3,7 @@ from test import CollectorTestCase from test import get_collector_config -from mock import patch +from test import patch import os from diamond.collector import Collector diff --git a/src/collectors/hadoop/test/testhadoop.py b/src/collectors/hadoop/test/testhadoop.py index 66efdb17f..f9c9058dc 100644 --- a/src/collectors/hadoop/test/testhadoop.py +++ b/src/collectors/hadoop/test/testhadoop.py @@ -5,7 +5,7 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import patch +from test import patch from diamond.collector import Collector from hadoop import HadoopCollector diff --git a/src/collectors/haproxy/test/testhaproxy.py b/src/collectors/haproxy/test/testhaproxy.py index d3b98c065..6fd713ee3 100644 --- a/src/collectors/haproxy/test/testhaproxy.py +++ b/src/collectors/haproxy/test/testhaproxy.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from haproxy import HAProxyCollector diff --git a/src/collectors/http/test/testhttp.py b/src/collectors/http/test/testhttp.py index 3f4158ab5..74570edf6 100644 --- a/src/collectors/http/test/testhttp.py +++ b/src/collectors/http/test/testhttp.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector diff --git a/src/collectors/httpd/test/testhttpd.py b/src/collectors/httpd/test/testhttpd.py index 643795d4e..866b0a2ed 100644 --- a/src/collectors/httpd/test/testhttpd.py +++ b/src/collectors/httpd/test/testhttpd.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from httpd import HttpdCollector diff --git a/src/collectors/httpjson/test/testhttpjson.py b/src/collectors/httpjson/test/testhttpjson.py index e2dea204c..acc1a6cff 100644 --- a/src/collectors/httpjson/test/testhttpjson.py +++ b/src/collectors/httpjson/test/testhttpjson.py @@ -3,8 +3,8 @@ ########################################################################## from test import CollectorTestCase from test import get_collector_config -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from httpjson import HTTPJSONCollector diff --git a/src/collectors/interrupt/test/testinterrupt.py b/src/collectors/interrupt/test/testinterrupt.py index 3f978c373..99907f642 100644 --- a/src/collectors/interrupt/test/testinterrupt.py +++ b/src/collectors/interrupt/test/testinterrupt.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch try: from cStringIO import StringIO diff --git a/src/collectors/interrupt/test/testsoft.py b/src/collectors/interrupt/test/testsoft.py index 7c9eb2afd..aa4e523e3 100644 --- a/src/collectors/interrupt/test/testsoft.py +++ b/src/collectors/interrupt/test/testsoft.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch try: from cStringIO import StringIO diff --git a/src/collectors/ip/test/testip.py b/src/collectors/ip/test/testip.py index 4ac373d92..e18e4920d 100644 --- a/src/collectors/ip/test/testip.py +++ b/src/collectors/ip/test/testip.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch try: from cStringIO import StringIO diff --git a/src/collectors/ipmisensor/test/testipmisensor.py b/src/collectors/ipmisensor/test/testipmisensor.py index 649fb2cf3..c1c970bc8 100644 --- a/src/collectors/ipmisensor/test/testipmisensor.py +++ b/src/collectors/ipmisensor/test/testipmisensor.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from ipmisensor import IPMISensorCollector diff --git a/src/collectors/ipvs/test/testipvs.py b/src/collectors/ipvs/test/testipvs.py index b0e1c188b..95471734a 100644 --- a/src/collectors/ipvs/test/testipvs.py +++ b/src/collectors/ipvs/test/testipvs.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from ipvs import IPVSCollector diff --git a/src/collectors/jolokia/test/testcassandra_jolokia.py b/src/collectors/jolokia/test/testcassandra_jolokia.py index a2cd6831c..3ad9ffd73 100644 --- a/src/collectors/jolokia/test/testcassandra_jolokia.py +++ b/src/collectors/jolokia/test/testcassandra_jolokia.py @@ -5,7 +5,7 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import patch +from test import patch from diamond.collector import Collector diff --git a/src/collectors/jolokia/test/testjolokia.py b/src/collectors/jolokia/test/testjolokia.py index 4af23123c..63d65b13f 100644 --- a/src/collectors/jolokia/test/testjolokia.py +++ b/src/collectors/jolokia/test/testjolokia.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch import re from diamond.collector import Collector diff --git a/src/collectors/kafka_consumer_lag/test/testkafka_consumer_lag.py b/src/collectors/kafka_consumer_lag/test/testkafka_consumer_lag.py index 54be46a1b..dff0311a5 100644 --- a/src/collectors/kafka_consumer_lag/test/testkafka_consumer_lag.py +++ b/src/collectors/kafka_consumer_lag/test/testkafka_consumer_lag.py @@ -4,7 +4,8 @@ from test import CollectorTestCase from test import get_collector_config -from mock import patch, Mock +from test import patch +from test import Mock from diamond.collector import Collector from kafka_consumer_lag import KafkaConsumerLagCollector diff --git a/src/collectors/kafkastat/test/testkafka.py b/src/collectors/kafkastat/test/testkafka.py index b00ddeebc..6ff62ded8 100644 --- a/src/collectors/kafkastat/test/testkafka.py +++ b/src/collectors/kafkastat/test/testkafka.py @@ -13,7 +13,7 @@ from test import get_collector_config from test import run_only from test import unittest -from mock import patch +from test import patch from diamond.collector import Collector from kafkastat import KafkaCollector diff --git a/src/collectors/ksm/test/testksm.py b/src/collectors/ksm/test/testksm.py index 8f4e2fd9e..bd2177fd4 100644 --- a/src/collectors/ksm/test/testksm.py +++ b/src/collectors/ksm/test/testksm.py @@ -7,8 +7,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from ksm import KSMCollector diff --git a/src/collectors/kvm/test/testkvm.py b/src/collectors/kvm/test/testkvm.py index f1f8cd951..319b84780 100644 --- a/src/collectors/kvm/test/testkvm.py +++ b/src/collectors/kvm/test/testkvm.py @@ -7,8 +7,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch try: from cStringIO import StringIO diff --git a/src/collectors/lmsensors/test/testlmsensors.py b/src/collectors/lmsensors/test/testlmsensors.py index 9c7060f71..356480af1 100644 --- a/src/collectors/lmsensors/test/testlmsensors.py +++ b/src/collectors/lmsensors/test/testlmsensors.py @@ -5,7 +5,7 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import patch +from test import patch from diamond.collector import Collector from lmsensors import LMSensorsCollector diff --git a/src/collectors/loadavg/test/testloadavg.py b/src/collectors/loadavg/test/testloadavg.py index 6f237019e..dd7bc734e 100644 --- a/src/collectors/loadavg/test/testloadavg.py +++ b/src/collectors/loadavg/test/testloadavg.py @@ -6,8 +6,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch try: from cStringIO import StringIO diff --git a/src/collectors/mdstat/test/test_mdstat.py b/src/collectors/mdstat/test/test_mdstat.py index 94f724b23..976df091f 100644 --- a/src/collectors/mdstat/test/test_mdstat.py +++ b/src/collectors/mdstat/test/test_mdstat.py @@ -6,8 +6,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from mdstat import MdStatCollector diff --git a/src/collectors/memcached/test/testmemcached.py b/src/collectors/memcached/test/testmemcached.py index 1fd69cc25..39eefb507 100644 --- a/src/collectors/memcached/test/testmemcached.py +++ b/src/collectors/memcached/test/testmemcached.py @@ -6,8 +6,8 @@ from test import get_collector_config from test import unittest from mock import MagicMock -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from memcached import MemcachedCollector diff --git a/src/collectors/memory/test/testmemory.py b/src/collectors/memory/test/testmemory.py index 4ea45e808..5ad1f2229 100644 --- a/src/collectors/memory/test/testmemory.py +++ b/src/collectors/memory/test/testmemory.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch try: from cStringIO import StringIO diff --git a/src/collectors/memory_cgroup/test/testmemory_cgroup.py b/src/collectors/memory_cgroup/test/testmemory_cgroup.py index 3981e1595..8176cc1b8 100644 --- a/src/collectors/memory_cgroup/test/testmemory_cgroup.py +++ b/src/collectors/memory_cgroup/test/testmemory_cgroup.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch try: from cStringIO import StringIO diff --git a/src/collectors/memory_docker/test/testmemory_docker.py b/src/collectors/memory_docker/test/testmemory_docker.py index 00cdfa5b4..8317ed1cb 100644 --- a/src/collectors/memory_docker/test/testmemory_docker.py +++ b/src/collectors/memory_docker/test/testmemory_docker.py @@ -6,8 +6,8 @@ from test import get_collector_config from test import unittest from test import run_only -from mock import Mock -from mock import patch +from test import Mock +from test import patch try: from cStringIO import StringIO diff --git a/src/collectors/mesos/test/testmesos.py b/src/collectors/mesos/test/testmesos.py index bb140fa34..89398615f 100644 --- a/src/collectors/mesos/test/testmesos.py +++ b/src/collectors/mesos/test/testmesos.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector diff --git a/src/collectors/mesos_cgroup/test/testmesos_cgroup.py b/src/collectors/mesos_cgroup/test/testmesos_cgroup.py index 152cc9d6f..c2236b237 100644 --- a/src/collectors/mesos_cgroup/test/testmesos_cgroup.py +++ b/src/collectors/mesos_cgroup/test/testmesos_cgroup.py @@ -5,8 +5,9 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import MagicMock, Mock, mock_open -from mock import patch +from mock import MagicMock, mock_open +from test import patch +from test import Mock from diamond.collector import Collector diff --git a/src/collectors/mogilefs/test/testmogilefs.py b/src/collectors/mogilefs/test/testmogilefs.py index 8787e165a..e13b07153 100644 --- a/src/collectors/mogilefs/test/testmogilefs.py +++ b/src/collectors/mogilefs/test/testmogilefs.py @@ -5,8 +5,9 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock, call -from mock import patch +from mock import call +from test import Mock +from test import patch from diamond.collector import Collector diff --git a/src/collectors/mongodb/mongodb.py b/src/collectors/mongodb/mongodb.py index 3557626d7..3fbb1420b 100644 --- a/src/collectors/mongodb/mongodb.py +++ b/src/collectors/mongodb/mongodb.py @@ -183,6 +183,7 @@ def collect(self): if user: try: conn.admin.authenticate(user, passwd) + except Exception as e: self.log.error( 'User auth given, but could not autheticate' + ' with host: %s, err: %s' % (host, e)) diff --git a/src/collectors/mongodb/test/testmongodb.py b/src/collectors/mongodb/test/testmongodb.py index 340e192f4..3c9896910 100644 --- a/src/collectors/mongodb/test/testmongodb.py +++ b/src/collectors/mongodb/test/testmongodb.py @@ -7,7 +7,7 @@ from test import unittest from test import run_only from mock import MagicMock -from mock import patch +from test import patch from mock import call from diamond.collector import Collector diff --git a/src/collectors/monit/test/testmonit.py b/src/collectors/monit/test/testmonit.py index d7ac85a03..480bcb624 100644 --- a/src/collectors/monit/test/testmonit.py +++ b/src/collectors/monit/test/testmonit.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector diff --git a/src/collectors/mountstats/test/testmountstats.py b/src/collectors/mountstats/test/testmountstats.py index e006b74c9..1bf51ef2b 100644 --- a/src/collectors/mountstats/test/testmountstats.py +++ b/src/collectors/mountstats/test/testmountstats.py @@ -4,8 +4,9 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import MagicMock, Mock -from mock import patch +from mock import MagicMock +from test import patch +from test import Mock from diamond.collector import Collector from mountstats import MountStatsCollector diff --git a/src/collectors/mysqlstat/test/testmysql.py b/src/collectors/mysqlstat/test/testmysql.py index 3ac98d1f4..138079813 100644 --- a/src/collectors/mysqlstat/test/testmysql.py +++ b/src/collectors/mysqlstat/test/testmysql.py @@ -6,8 +6,8 @@ from test import get_collector_config from test import unittest from test import run_only -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from mysqlstat import MySQLCollector diff --git a/src/collectors/nagios/test/testnagios.py b/src/collectors/nagios/test/testnagios.py index b5878d64c..fca470745 100644 --- a/src/collectors/nagios/test/testnagios.py +++ b/src/collectors/nagios/test/testnagios.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from nagios import NagiosStatsCollector diff --git a/src/collectors/nagiosperfdata/test/testnagiosperfdata.py b/src/collectors/nagiosperfdata/test/testnagiosperfdata.py index 101adc441..8d7a96cd8 100644 --- a/src/collectors/nagiosperfdata/test/testnagiosperfdata.py +++ b/src/collectors/nagiosperfdata/test/testnagiosperfdata.py @@ -3,7 +3,7 @@ from test import CollectorTestCase from test import get_collector_config -from mock import patch +from test import patch import os from diamond.collector import Collector diff --git a/src/collectors/netstat/test/testnetstat.py b/src/collectors/netstat/test/testnetstat.py index 4c316ac94..f4b22d4e1 100644 --- a/src/collectors/netstat/test/testnetstat.py +++ b/src/collectors/netstat/test/testnetstat.py @@ -5,7 +5,7 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import patch +from test import patch from diamond.collector import Collector from netstat import NetstatCollector diff --git a/src/collectors/network/test/testnetwork.py b/src/collectors/network/test/testnetwork.py index 24d5873f5..ead42d586 100644 --- a/src/collectors/network/test/testnetwork.py +++ b/src/collectors/network/test/testnetwork.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch try: from cStringIO import StringIO diff --git a/src/collectors/nfacct/test/testnfacct.py b/src/collectors/nfacct/test/testnfacct.py index 97d98731c..7576d681d 100644 --- a/src/collectors/nfacct/test/testnfacct.py +++ b/src/collectors/nfacct/test/testnfacct.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from nfacct import NetfilterAccountingCollector diff --git a/src/collectors/nfs/test/testnfs.py b/src/collectors/nfs/test/testnfs.py index 55b9c0078..634067c1b 100644 --- a/src/collectors/nfs/test/testnfs.py +++ b/src/collectors/nfs/test/testnfs.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch try: from cStringIO import StringIO diff --git a/src/collectors/nfsd/test/testnfsd.py b/src/collectors/nfsd/test/testnfsd.py index f4de04989..a7f4467c6 100644 --- a/src/collectors/nfsd/test/testnfsd.py +++ b/src/collectors/nfsd/test/testnfsd.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch try: from cStringIO import StringIO diff --git a/src/collectors/nginx/test/testnginx.py b/src/collectors/nginx/test/testnginx.py index 4c015fb88..3e2ab360c 100644 --- a/src/collectors/nginx/test/testnginx.py +++ b/src/collectors/nginx/test/testnginx.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector diff --git a/src/collectors/ntp/test/testntp.py b/src/collectors/ntp/test/testntp.py index 60120f65e..4366698a8 100644 --- a/src/collectors/ntp/test/testntp.py +++ b/src/collectors/ntp/test/testntp.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector diff --git a/src/collectors/ntpd/test/testntpd.py b/src/collectors/ntpd/test/testntpd.py index 7fe3070e2..dbfd75ef4 100644 --- a/src/collectors/ntpd/test/testntpd.py +++ b/src/collectors/ntpd/test/testntpd.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector diff --git a/src/collectors/numa/test/testnuma.py b/src/collectors/numa/test/testnuma.py index 9b12afaa3..b56bddef4 100644 --- a/src/collectors/numa/test/testnuma.py +++ b/src/collectors/numa/test/testnuma.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import patch -from mock import Mock +from test import patch +from test import Mock from diamond.collector import Collector from numa import NumaCollector diff --git a/src/collectors/nvidia_gpu/test/testnvidia_gpu.py b/src/collectors/nvidia_gpu/test/testnvidia_gpu.py index 3c64123b4..4a10e5733 100644 --- a/src/collectors/nvidia_gpu/test/testnvidia_gpu.py +++ b/src/collectors/nvidia_gpu/test/testnvidia_gpu.py @@ -5,7 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import patch, Mock +from test import patch +from test import Mock from diamond.collector import Collector from nvidia_gpu import NvidiaGPUCollector diff --git a/src/collectors/onewire/test/testonewire.py b/src/collectors/onewire/test/testonewire.py index 597125a44..7b5a90e32 100644 --- a/src/collectors/onewire/test/testonewire.py +++ b/src/collectors/onewire/test/testonewire.py @@ -6,7 +6,7 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import patch +from test import patch from diamond.collector import Collector from onewire import OneWireCollector diff --git a/src/collectors/openstackswiftrecon/test/testopenstackswiftrecon.py b/src/collectors/openstackswiftrecon/test/testopenstackswiftrecon.py index e98e7aa81..44970f7bd 100644 --- a/src/collectors/openstackswiftrecon/test/testopenstackswiftrecon.py +++ b/src/collectors/openstackswiftrecon/test/testopenstackswiftrecon.py @@ -6,7 +6,7 @@ from test import get_collector_config from test import unittest -from mock import patch, Mock +from test import patch, Mock from diamond.collector import Collector from openstackswiftrecon import OpenstackSwiftReconCollector diff --git a/src/collectors/openvpn/test/testopenvpn.py b/src/collectors/openvpn/test/testopenvpn.py index d496a0a92..1a8e38d36 100644 --- a/src/collectors/openvpn/test/testopenvpn.py +++ b/src/collectors/openvpn/test/testopenvpn.py @@ -5,7 +5,7 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import patch +from test import patch from diamond.collector import Collector from openvpn import OpenVPNCollector diff --git a/src/collectors/openvz/test/testopenvz.py b/src/collectors/openvz/test/testopenvz.py index a008e8153..8f081b5b0 100644 --- a/src/collectors/openvz/test/testopenvz.py +++ b/src/collectors/openvz/test/testopenvz.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector diff --git a/src/collectors/pgbouncer/test/testpgbouncer.py b/src/collectors/pgbouncer/test/testpgbouncer.py index 15b963de1..3ea799e18 100644 --- a/src/collectors/pgbouncer/test/testpgbouncer.py +++ b/src/collectors/pgbouncer/test/testpgbouncer.py @@ -6,7 +6,7 @@ from test import get_collector_config from test import run_only from test import unittest -from mock import patch +from test import patch from pgbouncer import PgbouncerCollector diff --git a/src/collectors/pgq/test/testpgq.py b/src/collectors/pgq/test/testpgq.py index 862760251..153b84777 100644 --- a/src/collectors/pgq/test/testpgq.py +++ b/src/collectors/pgq/test/testpgq.py @@ -1,6 +1,7 @@ #!/usr/bin/python from test import CollectorTestCase, get_collector_config -from mock import MagicMock, patch +from test import patch +from mock import MagicMock from pgq import PgQCollector diff --git a/src/collectors/phpfpm/test/testphpfpm.py b/src/collectors/phpfpm/test/testphpfpm.py index ab8dea909..c6868106c 100644 --- a/src/collectors/phpfpm/test/testphpfpm.py +++ b/src/collectors/phpfpm/test/testphpfpm.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from phpfpm import PhpFpmCollector diff --git a/src/collectors/ping/test/testping.py b/src/collectors/ping/test/testping.py index 7a529d3f4..574ec8dc9 100644 --- a/src/collectors/ping/test/testping.py +++ b/src/collectors/ping/test/testping.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from ping import PingCollector diff --git a/src/collectors/portstat/tests/test_portstat.py b/src/collectors/portstat/tests/test_portstat.py index 187b52789..01b948f79 100644 --- a/src/collectors/portstat/tests/test_portstat.py +++ b/src/collectors/portstat/tests/test_portstat.py @@ -1,7 +1,9 @@ from test import CollectorTestCase from test import get_collector_config +from test import patch +from test import Mock -from mock import call, Mock, patch +from mock import call from unittest import TestCase from diamond.collector import Collector diff --git a/src/collectors/postfix/test/testpostfix.py b/src/collectors/postfix/test/testpostfix.py index 8dd7bd5a5..b0511d386 100644 --- a/src/collectors/postfix/test/testpostfix.py +++ b/src/collectors/postfix/test/testpostfix.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from postfix import PostfixCollector diff --git a/src/collectors/postgres/test/testpostgres.py b/src/collectors/postgres/test/testpostgres.py index 866145196..fa124ae25 100644 --- a/src/collectors/postgres/test/testpostgres.py +++ b/src/collectors/postgres/test/testpostgres.py @@ -4,8 +4,8 @@ from test import CollectorTestCase from test import get_collector_config - -from mock import patch, Mock +from test import patch +from test import Mock from postgres import PostgresqlCollector diff --git a/src/collectors/postqueue/test/testpostqueue.py b/src/collectors/postqueue/test/testpostqueue.py index b667b051b..95b5a411c 100644 --- a/src/collectors/postqueue/test/testpostqueue.py +++ b/src/collectors/postqueue/test/testpostqueue.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from postqueue import PostqueueCollector diff --git a/src/collectors/powerdns/test/testpowerdns.py b/src/collectors/powerdns/test/testpowerdns.py index 5a9c173fd..c0815deb7 100644 --- a/src/collectors/powerdns/test/testpowerdns.py +++ b/src/collectors/powerdns/test/testpowerdns.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from powerdns import PowerDNSCollector diff --git a/src/collectors/proc/test/testproc.py b/src/collectors/proc/test/testproc.py index 997d9ef6f..2592f47ae 100644 --- a/src/collectors/proc/test/testproc.py +++ b/src/collectors/proc/test/testproc.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch try: from cStringIO import StringIO diff --git a/src/collectors/processresources/test/testprocessresources.py b/src/collectors/processresources/test/testprocessresources.py index 7e4e14316..4c6b7e320 100644 --- a/src/collectors/processresources/test/testprocessresources.py +++ b/src/collectors/processresources/test/testprocessresources.py @@ -8,7 +8,7 @@ from test import get_collector_config from test import unittest from test import run_only -from mock import patch +from test import patch from diamond.collector import Collector from processresources import ProcessResourcesCollector diff --git a/src/collectors/puppetagent/test/testpuppetagent.py b/src/collectors/puppetagent/test/testpuppetagent.py index 5aa339caa..d22c77cff 100644 --- a/src/collectors/puppetagent/test/testpuppetagent.py +++ b/src/collectors/puppetagent/test/testpuppetagent.py @@ -6,7 +6,7 @@ from test import get_collector_config from test import unittest from test import run_only -from mock import patch +from test import patch from diamond.collector import Collector from puppetagent import PuppetAgentCollector diff --git a/src/collectors/puppetdashboard/test/testpuppetdashboard.py b/src/collectors/puppetdashboard/test/testpuppetdashboard.py index 04fcf7dad..8b080e594 100644 --- a/src/collectors/puppetdashboard/test/testpuppetdashboard.py +++ b/src/collectors/puppetdashboard/test/testpuppetdashboard.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from puppetdashboard import PuppetDashboardCollector diff --git a/src/collectors/rabbitmq/test/testrabbitmq.py b/src/collectors/rabbitmq/test/testrabbitmq.py index f5006a590..d1329e099 100644 --- a/src/collectors/rabbitmq/test/testrabbitmq.py +++ b/src/collectors/rabbitmq/test/testrabbitmq.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from mock import call from diamond.collector import Collector diff --git a/src/collectors/redisstat/test/testredisstat.py b/src/collectors/redisstat/test/testredisstat.py index 987473282..61fa3dbbf 100644 --- a/src/collectors/redisstat/test/testredisstat.py +++ b/src/collectors/redisstat/test/testredisstat.py @@ -6,8 +6,9 @@ from test import get_collector_config from test import unittest from test import run_only -from mock import Mock -from mock import patch, call +from test import Mock +from test import patch +from mock import call from diamond.collector import Collector from redisstat import RedisCollector diff --git a/src/collectors/resqueweb/test/testresqueweb.py b/src/collectors/resqueweb/test/testresqueweb.py index 5b492756f..7baa91046 100644 --- a/src/collectors/resqueweb/test/testresqueweb.py +++ b/src/collectors/resqueweb/test/testresqueweb.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from resqueweb import ResqueWebCollector diff --git a/src/collectors/scribe/test/test_scribe.py b/src/collectors/scribe/test/test_scribe.py index a4d218276..1fbbd3fc8 100644 --- a/src/collectors/scribe/test/test_scribe.py +++ b/src/collectors/scribe/test/test_scribe.py @@ -3,7 +3,7 @@ from test import CollectorTestCase from test import get_collector_config -from mock import Mock, patch +from test import Mock, patch from scribe import ScribeCollector diff --git a/src/collectors/sidekiq/test/testsidekiq.py b/src/collectors/sidekiq/test/testsidekiq.py index fb41aac82..d1b68d0d4 100644 --- a/src/collectors/sidekiq/test/testsidekiq.py +++ b/src/collectors/sidekiq/test/testsidekiq.py @@ -7,8 +7,8 @@ from test import get_collector_config from test import unittest from test import run_only -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from sidekiq import SidekiqCollector diff --git a/src/collectors/sidekiqweb/test/testsidekiqweb.py b/src/collectors/sidekiqweb/test/testsidekiqweb.py index cb87037de..4aff95486 100644 --- a/src/collectors/sidekiqweb/test/testsidekiqweb.py +++ b/src/collectors/sidekiqweb/test/testsidekiqweb.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from sidekiqweb import SidekiqWebCollector diff --git a/src/collectors/slabinfo/test/testslabinfo.py b/src/collectors/slabinfo/test/testslabinfo.py index 106514db4..58a99b1e1 100644 --- a/src/collectors/slabinfo/test/testslabinfo.py +++ b/src/collectors/slabinfo/test/testslabinfo.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch try: from cStringIO import StringIO diff --git a/src/collectors/slony/test/testslony.py b/src/collectors/slony/test/testslony.py index 17e9b15c4..13f957d72 100644 --- a/src/collectors/slony/test/testslony.py +++ b/src/collectors/slony/test/testslony.py @@ -5,7 +5,7 @@ from test import CollectorTestCase from test import get_collector_config from test import run_only -from mock import patch +from test import patch from slony import SlonyCollector diff --git a/src/collectors/smart/test/testsmart.py b/src/collectors/smart/test/testsmart.py index 72431bb43..4c534bfd3 100644 --- a/src/collectors/smart/test/testsmart.py +++ b/src/collectors/smart/test/testsmart.py @@ -5,9 +5,9 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock +from test import Mock from mock import call -from mock import patch +from test import patch from diamond.collector import Collector from smart import SmartCollector diff --git a/src/collectors/snmpraw/test/testsnmpraw.py b/src/collectors/snmpraw/test/testsnmpraw.py index 2dbe447d7..4a6e9b28e 100644 --- a/src/collectors/snmpraw/test/testsnmpraw.py +++ b/src/collectors/snmpraw/test/testsnmpraw.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from snmpraw import SNMPRawCollector from diamond.collector import Collector diff --git a/src/collectors/sockstat/test/testsockstat.py b/src/collectors/sockstat/test/testsockstat.py index bd8db36b5..30a85278a 100644 --- a/src/collectors/sockstat/test/testsockstat.py +++ b/src/collectors/sockstat/test/testsockstat.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from mock import call from collections import Iterator diff --git a/src/collectors/solr/test/testsolr.py b/src/collectors/solr/test/testsolr.py index c0d01752c..0f31f01a0 100644 --- a/src/collectors/solr/test/testsolr.py +++ b/src/collectors/solr/test/testsolr.py @@ -5,7 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import call, patch +from mock import call +from test import patch from diamond.collector import Collector diff --git a/src/collectors/squid/test/testsquid.py b/src/collectors/squid/test/testsquid.py index a44b55ba8..aeb5222e9 100644 --- a/src/collectors/squid/test/testsquid.py +++ b/src/collectors/squid/test/testsquid.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from squid import SquidCollector diff --git a/src/collectors/supervisord/test/testsupervisord.py b/src/collectors/supervisord/test/testsupervisord.py index bf5b2b512..c93a34d56 100644 --- a/src/collectors/supervisord/test/testsupervisord.py +++ b/src/collectors/supervisord/test/testsupervisord.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from supervisord import SupervisordCollector diff --git a/src/collectors/tcp/test/testtcp.py b/src/collectors/tcp/test/testtcp.py index addabf76b..658ea0702 100644 --- a/src/collectors/tcp/test/testtcp.py +++ b/src/collectors/tcp/test/testtcp.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch try: from cStringIO import StringIO diff --git a/src/collectors/tokumx/test/testtokumx.py b/src/collectors/tokumx/test/testtokumx.py index e43ef159a..263d3feab 100644 --- a/src/collectors/tokumx/test/testtokumx.py +++ b/src/collectors/tokumx/test/testtokumx.py @@ -7,7 +7,7 @@ from test import unittest from test import run_only from mock import MagicMock -from mock import patch +from test import patch from mock import call from diamond.collector import Collector diff --git a/src/collectors/twemproxy/test/testtwemproxy.py b/src/collectors/twemproxy/test/testtwemproxy.py index 6ba15d3c8..15b6066cd 100644 --- a/src/collectors/twemproxy/test/testtwemproxy.py +++ b/src/collectors/twemproxy/test/testtwemproxy.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from twemproxy import TwemproxyCollector diff --git a/src/collectors/udp/test/testudp.py b/src/collectors/udp/test/testudp.py index 7edcff02f..ae9dea063 100644 --- a/src/collectors/udp/test/testudp.py +++ b/src/collectors/udp/test/testudp.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch try: from cStringIO import StringIO diff --git a/src/collectors/unbound/test/testunbound.py b/src/collectors/unbound/test/testunbound.py index cf08767bf..5c6b79982 100644 --- a/src/collectors/unbound/test/testunbound.py +++ b/src/collectors/unbound/test/testunbound.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector diff --git a/src/collectors/ups/test/testups.py b/src/collectors/ups/test/testups.py index 00693278f..a3f7db2f5 100644 --- a/src/collectors/ups/test/testups.py +++ b/src/collectors/ups/test/testups.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from ups import UPSCollector diff --git a/src/collectors/uptime/test/test_uptime.py b/src/collectors/uptime/test/test_uptime.py index 9dc2bec29..05e1a6353 100644 --- a/src/collectors/uptime/test/test_uptime.py +++ b/src/collectors/uptime/test/test_uptime.py @@ -4,8 +4,8 @@ from test import CollectorTestCase from test import get_collector_config -from mock import Mock -from mock import patch +from test import Mock +from test import patch try: from cStringIO import StringIO diff --git a/src/collectors/users/test/testusers.py b/src/collectors/users/test/testusers.py index 3d85693b3..a44fde4b0 100644 --- a/src/collectors/users/test/testusers.py +++ b/src/collectors/users/test/testusers.py @@ -6,7 +6,7 @@ from test import get_collector_config from test import unittest from test import run_only -from mock import patch +from test import patch from diamond.collector import Collector from users import UsersCollector diff --git a/src/collectors/userscripts/test/testuserscripts.py b/src/collectors/userscripts/test/testuserscripts.py index fcb8bb77e..b87b73790 100644 --- a/src/collectors/userscripts/test/testuserscripts.py +++ b/src/collectors/userscripts/test/testuserscripts.py @@ -9,7 +9,7 @@ from test import get_collector_config from test import unittest from test import run_only -from mock import patch +from test import patch from diamond.collector import Collector from userscripts import UserScriptsCollector diff --git a/src/collectors/varnish/test/testvarnish.py b/src/collectors/varnish/test/testvarnish.py index 40bc8f4ae..354bf4a34 100644 --- a/src/collectors/varnish/test/testvarnish.py +++ b/src/collectors/varnish/test/testvarnish.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector diff --git a/src/collectors/vmstat/test/testvmstat.py b/src/collectors/vmstat/test/testvmstat.py index 81adbcdcb..1478dce15 100644 --- a/src/collectors/vmstat/test/testvmstat.py +++ b/src/collectors/vmstat/test/testvmstat.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch try: from cStringIO import StringIO diff --git a/src/collectors/websitemonitor/test/testwebsitemonitor.py b/src/collectors/websitemonitor/test/testwebsitemonitor.py index 6d5b56588..0dfdf1cfb 100644 --- a/src/collectors/websitemonitor/test/testwebsitemonitor.py +++ b/src/collectors/websitemonitor/test/testwebsitemonitor.py @@ -4,7 +4,7 @@ from test import CollectorTestCase from test import get_collector_config -from mock import patch +from test import patch from diamond.collector import Collector diff --git a/src/collectors/xen_collector/test/testxen.py b/src/collectors/xen_collector/test/testxen.py index aca58c007..4e5ce98f9 100644 --- a/src/collectors/xen_collector/test/testxen.py +++ b/src/collectors/xen_collector/test/testxen.py @@ -6,8 +6,8 @@ from test import get_collector_config from test import unittest from test import run_only -from mock import Mock -from mock import patch +from test import Mock +from test import patch from diamond.collector import Collector from xen_collector import XENCollector diff --git a/src/collectors/xfs/test/testxfs.py b/src/collectors/xfs/test/testxfs.py index 924a4a332..de708c1f2 100644 --- a/src/collectors/xfs/test/testxfs.py +++ b/src/collectors/xfs/test/testxfs.py @@ -5,8 +5,8 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch try: from cStringIO import StringIO diff --git a/src/diamond/handler/test/testgraphitehandler.py b/src/diamond/handler/test/testgraphitehandler.py index be6067a45..0b7fda3f8 100644 --- a/src/diamond/handler/test/testgraphitehandler.py +++ b/src/diamond/handler/test/testgraphitehandler.py @@ -5,8 +5,8 @@ import time from test import unittest -from mock import Mock -from mock import patch +from test import Mock +from test import patch from mock import call import configobj diff --git a/src/diamond/handler/test/testriemann.py b/src/diamond/handler/test/testriemann.py index 404ef7fe8..c99055f89 100644 --- a/src/diamond/handler/test/testriemann.py +++ b/src/diamond/handler/test/testriemann.py @@ -4,8 +4,8 @@ from test import unittest from test import run_only -from mock import Mock -from mock import patch +from test import Mock +from test import patch from mock import call import configobj diff --git a/src/diamond/handler/test/teststatsdhandler.py b/src/diamond/handler/test/teststatsdhandler.py index e660f6fb9..701188f21 100644 --- a/src/diamond/handler/test/teststatsdhandler.py +++ b/src/diamond/handler/test/teststatsdhandler.py @@ -4,7 +4,8 @@ from test import unittest from test import run_only -from mock import patch +from test import patch +from mock import ANY import configobj diff --git a/src/diamond/handler/test/testtsdb.py b/src/diamond/handler/test/testtsdb.py index 8e7133a01..3d11e97e0 100644 --- a/src/diamond/handler/test/testtsdb.py +++ b/src/diamond/handler/test/testtsdb.py @@ -3,7 +3,7 @@ ########################################################################## from test import unittest -from mock import Mock +from test import Mock import configobj diff --git a/src/diamond/test/testcollector.py b/src/diamond/test/testcollector.py index d48a146f8..32ecf89e6 100644 --- a/src/diamond/test/testcollector.py +++ b/src/diamond/test/testcollector.py @@ -2,7 +2,7 @@ # coding=utf-8 ########################################################################## -from mock import patch +from test import patch from test import unittest import configobj diff --git a/test.py b/test.py index 1481bb8f7..00d62a03d 100755 --- a/test.py +++ b/test.py @@ -24,13 +24,23 @@ try: from cStringIO import StringIO except ImportError: - from StringIO import StringIO + try: + from StringIO import StringIO + except ImportError: + from io import StringIO try: from setproctitle import setproctitle except ImportError: setproctitle = None +try: + from mock import Mock + from mock import patch +except ImportError: + from unittest.mock import Mock + from unittest.mock import patch + sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__)))) sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), 'src'))) From a74189555c7d706b8012baa8895e92fa60247704 Mon Sep 17 00:00:00 2001 From: Hannu Valtonen Date: Wed, 26 Aug 2015 14:22:55 +0300 Subject: [PATCH 04/33] py3k: add a pycompat helper file for imports --- src/diamond/pycompat.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 src/diamond/pycompat.py diff --git a/src/diamond/pycompat.py b/src/diamond/pycompat.py new file mode 100644 index 000000000..c815d34db --- /dev/null +++ b/src/diamond/pycompat.py @@ -0,0 +1,18 @@ +# coding=utf-8 +URLOPEN = "diamond.pycompat.urlopen" +try: + from urllib2 import HTTPError, Request, urlopen, URLError + from urllib import urlencode, quote + from urlparse import urljoin, urlparse + from Queue import Empty, Full, Queue +except ImportError: + from urllib.request import HTTPError, Request, URLError, urlopen + from urllib.parse import urlencode, quote, urljoin, urlparse + from queue import Full, Empty, Queue + +try: + long = long + unicode = unicode +except NameError: + long = int + unicode = str From 8e898fd5db1b0cb0d1ccbba9f17395f10a2eab18 Mon Sep 17 00:00:00 2001 From: Hannu Valtonen Date: Mon, 24 Aug 2015 14:13:03 +0300 Subject: [PATCH 05/33] tests: import exception types through a relative import Also get rid of basestring so tests can start to be run on py3k. --- src/diamond/collector.py | 6 +++--- src/diamond/metric.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/diamond/collector.py b/src/diamond/collector.py index 8e5727814..5c25743c3 100644 --- a/src/diamond/collector.py +++ b/src/diamond/collector.py @@ -16,7 +16,7 @@ from diamond.metric import Metric from diamond.utils.config import load_config -from error import DiamondException +from . error import DiamondException # Detect the architecture of the system and set the counters for MAX_VALUES # appropriately. Otherwise, rolling over counters will cause incorrect or @@ -144,7 +144,7 @@ def str_to_bool(value): Converts string truthy/falsey strings to a bool Empty strings are false """ - if isinstance(value, basestring): + if isinstance(value, str): value = value.strip().lower() if value in ['true', 't', 'yes', 'y']: return True @@ -222,7 +222,7 @@ def process_config(self): event """ if 'byte_unit' in self.config: - if isinstance(self.config['byte_unit'], basestring): + if isinstance(self.config['byte_unit'], str): self.config['byte_unit'] = self.config['byte_unit'].split() if 'enabled' in self.config: diff --git a/src/diamond/metric.py b/src/diamond/metric.py index 66d82515f..8570b7cca 100644 --- a/src/diamond/metric.py +++ b/src/diamond/metric.py @@ -3,7 +3,7 @@ import time import re import logging -from error import DiamondException +from . error import DiamondException class Metric(object): From c21a510a88b96ca54246d975e5efec287e2cc9ad Mon Sep 17 00:00:00 2001 From: Hannu Valtonen Date: Mon, 24 Aug 2015 14:14:30 +0300 Subject: [PATCH 06/33] py3k: replace all dict.iteritems() with .items() The performance impact of this should be fairly limited but it allows to run the same code with py3k. --- src/collectors/cpuacct_cgroup/cpuacct_cgroup.py | 4 ++-- src/collectors/diskspace/diskspace.py | 2 +- src/collectors/diskusage/diskusage.py | 4 ++-- src/collectors/elasticsearch/elasticsearch.py | 10 +++++----- src/collectors/flume/flume.py | 2 +- src/collectors/ipvs/ipvs.py | 8 ++++---- src/collectors/jolokia/jolokia.py | 4 ++-- src/collectors/kafkastat/kafkastat.py | 2 +- src/collectors/memory_cgroup/memory_cgroup.py | 4 ++-- .../memory_cgroup/test/testmemory_cgroup.py | 4 ++-- src/collectors/onewire/onewire.py | 6 +++--- .../openstackswiftrecon/openstackswiftrecon.py | 2 +- src/collectors/pgbouncer/pgbouncer.py | 6 +++--- src/collectors/pgq/pgq.py | 2 +- src/collectors/portstat/portstat.py | 4 ++-- src/collectors/postfix/postfix.py | 8 ++++---- src/collectors/postgres/postgres.py | 2 +- src/collectors/processresources/processresources.py | 8 ++++---- src/collectors/puppetagent/puppetagent.py | 4 ++-- src/collectors/slony/slony.py | 2 +- src/collectors/twemproxy/twemproxy.py | 12 ++++++------ src/collectors/unbound/unbound.py | 2 +- src/diamond/server.py | 2 +- test.py | 4 ++-- 24 files changed, 54 insertions(+), 54 deletions(-) diff --git a/src/collectors/cpuacct_cgroup/cpuacct_cgroup.py b/src/collectors/cpuacct_cgroup/cpuacct_cgroup.py index ed3ec2675..f15d02e73 100644 --- a/src/collectors/cpuacct_cgroup/cpuacct_cgroup.py +++ b/src/collectors/cpuacct_cgroup/cpuacct_cgroup.py @@ -62,8 +62,8 @@ def collect(self): stat_file.close() # create metrics from collected utimes and stimes for cgroups - for parent, cpuacct in results.iteritems(): - for key, value in cpuacct.iteritems(): + for parent, cpuacct in results.items(): + for key, value in cpuacct.items(): metric_name = '.'.join([parent, key]) self.publish(metric_name, value, metric_type='GAUGE') return True diff --git a/src/collectors/diskspace/diskspace.py b/src/collectors/diskspace/diskspace.py index 6e3d1c60b..330ace1bf 100644 --- a/src/collectors/diskspace/diskspace.py +++ b/src/collectors/diskspace/diskspace.py @@ -187,7 +187,7 @@ def collect(self): self.log.error('No diskspace metrics retrieved') return None - for info in results.itervalues(): + for key, info in results.items(): if info['device'] in labels: name = labels[info['device']] else: diff --git a/src/collectors/diskusage/diskusage.py b/src/collectors/diskusage/diskusage.py index 296209c46..015e02094 100644 --- a/src/collectors/diskusage/diskusage.py +++ b/src/collectors/diskusage/diskusage.py @@ -163,13 +163,13 @@ def collect(self): self.log.error('No diskspace metrics retrieved') return None - for key, info in results.iteritems(): + for key, info in results.items(): metrics = {} name = info['device'] if not reg.match(name): continue - for key, value in info.iteritems(): + for key, value in info.items(): if key == 'device': continue oldkey = key diff --git a/src/collectors/elasticsearch/elasticsearch.py b/src/collectors/elasticsearch/elasticsearch.py index 17ac0d1e2..739d95d51 100755 --- a/src/collectors/elasticsearch/elasticsearch.py +++ b/src/collectors/elasticsearch/elasticsearch.py @@ -135,13 +135,13 @@ def _get(self, scheme, host, port, path, assert_key=None): return doc def _copy_one_level(self, metrics, prefix, data, filter=lambda key: True): - for key, value in data.iteritems(): + for key, value in data.items(): if filter(key): metric_path = '%s.%s' % (prefix, key) self._set_or_sum_metric(metrics, metric_path, value) def _copy_two_level(self, metrics, prefix, data, filter=lambda key: True): - for key1, d1 in data.iteritems(): + for key1, d1 in data.items(): self._copy_one_level(metrics, '%s.%s' % (prefix, key1), d1, filter) def _index_metrics(self, metrics, prefix, index): @@ -236,7 +236,7 @@ def collect_instance_index_stats(self, scheme, host, port, metrics): else: return - for name, index in indices.iteritems(): + for name, index in indices.items(): self._index_metrics(metrics, 'indices.%s' % name, index['primaries']) @@ -367,7 +367,7 @@ def collect_instance(self, alias, scheme, host, port): if 'heap_used_percent' in mem: metrics['jvm.mem.heap_used_percent'] = mem['heap_used_percent'] - for pool, d in mem['pools'].iteritems(): + for pool, d in mem['pools'].items(): pool = pool.replace(' ', '_') metrics['jvm.mem.pools.%s.used' % pool] = d['used_in_bytes'] metrics['jvm.mem.pools.%s.max' % pool] = d['max_in_bytes'] @@ -377,7 +377,7 @@ def collect_instance(self, alias, scheme, host, port): gc = jvm['gc'] collection_count = 0 collection_time_in_millis = 0 - for collector, d in gc['collectors'].iteritems(): + for collector, d in gc['collectors'].items(): metrics['jvm.gc.collection.%s.count' % collector] = d[ 'collection_count'] collection_count += d['collection_count'] diff --git a/src/collectors/flume/flume.py b/src/collectors/flume/flume.py index efcebe5f7..3109803bc 100644 --- a/src/collectors/flume/flume.py +++ b/src/collectors/flume/flume.py @@ -94,7 +94,7 @@ def collect(self): self.log.error('Unknown error opening url: %s', e) return None - for comp in j.iteritems(): + for comp in j.items(): comp_name = comp[0] comp_items = comp[1] comp_type = comp_items['Type'] diff --git a/src/collectors/ipvs/ipvs.py b/src/collectors/ipvs/ipvs.py index e5a4dd868..3dddb76db 100644 --- a/src/collectors/ipvs/ipvs.py +++ b/src/collectors/ipvs/ipvs.py @@ -101,7 +101,7 @@ def collect(self): else: continue - for metric, column in columns.iteritems(): + for metric, column in columns.items(): metric_name = ".".join([external, backend, metric]) # metric_value = int(row[column]) value = row[column] @@ -135,7 +135,7 @@ def collect(self): if row[0] == "TCP" or row[0] == "UDP": if total: - for metric, value in total.iteritems(): + for metric, value in total.items(): self.publish( ".".join([external, "total", metric]), value) @@ -149,7 +149,7 @@ def collect(self): else: continue - for metric, column in columns.iteritems(): + for metric, column in columns.items(): metric_name = ".".join([external, backend, metric]) # metric_value = int(row[column]) value = row[column] @@ -167,5 +167,5 @@ def collect(self): self.publish(metric_name, metric_value) if total: - for metric, value in total.iteritems(): + for metric, value in total.items(): self.publish(".".join([external, "total", metric]), value) diff --git a/src/collectors/jolokia/jolokia.py b/src/collectors/jolokia/jolokia.py index 37c0adb89..17cda5b2e 100644 --- a/src/collectors/jolokia/jolokia.py +++ b/src/collectors/jolokia/jolokia.py @@ -202,7 +202,7 @@ def collect(self): # The reponse was totally empty, or not an expected format self.log.error('Unable to retrieve domain %s.', domain) continue - for k, v in mbeans.iteritems(): + for k, v in mbeans.items(): if self._check_mbean(k): self.collect_bean(k, v) @@ -287,7 +287,7 @@ def clean_up(self, text): return text def collect_bean(self, prefix, obj): - for k, v in obj.iteritems(): + for k, v in obj.items(): if type(v) in [int, float, long]: key = "%s.%s" % (prefix, k) key = self.clean_up(key) diff --git a/src/collectors/kafkastat/kafkastat.py b/src/collectors/kafkastat/kafkastat.py index d71a0ab5a..9d28c46f2 100644 --- a/src/collectors/kafkastat/kafkastat.py +++ b/src/collectors/kafkastat/kafkastat.py @@ -179,5 +179,5 @@ def collect(self): metrics.update(stats) # Publish stats - for metric, value in metrics.iteritems(): + for metric, value in metrics.items(): self.publish(metric, value) diff --git a/src/collectors/memory_cgroup/memory_cgroup.py b/src/collectors/memory_cgroup/memory_cgroup.py index abbc3f653..e07be6bd4 100644 --- a/src/collectors/memory_cgroup/memory_cgroup.py +++ b/src/collectors/memory_cgroup/memory_cgroup.py @@ -112,8 +112,8 @@ def collect(self): break # create metrics from collected utimes and stimes for cgroups - for parent, cpuacct in results.iteritems(): - for key, value in cpuacct.iteritems(): + for parent, cpuacct in results.items(): + for key, value in cpuacct.items(): metric_name = '.'.join([parent, key]) self.publish(metric_name, value, metric_type='GAUGE') return True diff --git a/src/collectors/memory_cgroup/test/testmemory_cgroup.py b/src/collectors/memory_cgroup/test/testmemory_cgroup.py index 8176cc1b8..c6ca6a959 100644 --- a/src/collectors/memory_cgroup/test/testmemory_cgroup.py +++ b/src/collectors/memory_cgroup/test/testmemory_cgroup.py @@ -112,9 +112,9 @@ def test_should_not_include_filtered_metrics(self, publish_mock): 'lxc.testcontainer.total_swap': 1, } [self.assertPublished(publish_mock, k, v) - for k, v in should_be_published.iteritems()] + for k, v in should_be_published.items()] [self.assertUnpublished(publish_mock, k, v) - for k, v in should_not_be_published.iteritems()] + for k, v in should_not_be_published.items()] if __name__ == "__main__": unittest.main() diff --git a/src/collectors/onewire/onewire.py b/src/collectors/onewire/onewire.py index d3c40d6fd..a0b3553f7 100644 --- a/src/collectors/onewire/onewire.py +++ b/src/collectors/onewire/onewire.py @@ -58,11 +58,11 @@ def collect(self): if '.' in ld: self.read_values(ld, self.config['scan'], metrics) - for oid, files in self.config.iteritems(): + for oid, files in self.config.items(): if oid[:3] == 'id:': self.read_values(oid[3:], files, metrics) - for fn, fv in metrics.iteritems(): + for fn, fv in metrics.items(): self.publish(fn, fv, 2) def read_values(self, oid, files, metrics): @@ -74,7 +74,7 @@ def read_values(self, oid, files, metrics): oid_path = os.path.join(self.config['owfs'], oid) oid = oid.replace('.', '_') - for fn, alias in files.iteritems(): + for fn, alias in files.items(): fv = os.path.join(oid_path, fn) if os.path.isfile(fv): try: diff --git a/src/collectors/openstackswiftrecon/openstackswiftrecon.py b/src/collectors/openstackswiftrecon/openstackswiftrecon.py index 51ad82a60..278ecf943 100644 --- a/src/collectors/openstackswiftrecon/openstackswiftrecon.py +++ b/src/collectors/openstackswiftrecon/openstackswiftrecon.py @@ -50,7 +50,7 @@ def get_default_config(self): def _process_cache(self, d, path=()): """Recusively walk a nested recon cache dict to obtain path/values""" - for k, v in d.iteritems(): + for k, v in d.items(): if not isinstance(v, dict): self.metrics.append((path + (k,), v)) else: diff --git a/src/collectors/pgbouncer/pgbouncer.py b/src/collectors/pgbouncer/pgbouncer.py index 2c174c0a8..05ee80940 100644 --- a/src/collectors/pgbouncer/pgbouncer.py +++ b/src/collectors/pgbouncer/pgbouncer.py @@ -83,15 +83,15 @@ def collect(self): } } - for name, instance in instances.iteritems(): + for name, instance in instances.items(): host = instance['host'] port = instance['port'] user = instance.get('user') or self.config['user'] password = instance.get('password') or self.config['password'] for database, stats in self._get_stats_by_database( - host, port, user, password).iteritems(): - for stat_name, stat_value in stats.iteritems(): + host, port, user, password).items(): + for stat_name, stat_value in stats.items(): self.publish( self._get_metric_name(name, database, stat_name), stat_value) diff --git a/src/collectors/pgq/pgq.py b/src/collectors/pgq/pgq.py index ad3e8358d..4e5a36c72 100644 --- a/src/collectors/pgq/pgq.py +++ b/src/collectors/pgq/pgq.py @@ -52,7 +52,7 @@ def collect(self): self.log.error('Unable to import module psycopg2') return None - for instance, configuration in self.config['instances'].iteritems(): + for instance, configuration in self.config['instances'].items(): connection = psycopg2.connect(configuration['dsn']) connection.set_isolation_level( psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT, diff --git a/src/collectors/portstat/portstat.py b/src/collectors/portstat/portstat.py index 634a0ed60..8a7f23b27 100644 --- a/src/collectors/portstat/portstat.py +++ b/src/collectors/portstat/portstat.py @@ -66,10 +66,10 @@ def collect(self): self.log.error('Unable to import module psutil') return {} - for port_name, port_cfg in self.ports.iteritems(): + for port_name, port_cfg in self.ports.items(): port = int(port_cfg['number']) stats = get_port_stats(port) - for stat_name, stat_value in stats.iteritems(): + for stat_name, stat_value in stats.items(): metric_name = '%s.%s' % (port_name, stat_name) self.publish(metric_name, stat_value) diff --git a/src/collectors/postfix/postfix.py b/src/collectors/postfix/postfix.py index 16bbbea6d..1c16eb202 100644 --- a/src/collectors/postfix/postfix.py +++ b/src/collectors/postfix/postfix.py @@ -101,7 +101,7 @@ def collect(self): return if str_to_bool(self.config['include_clients']) and u'clients' in data: - for client, value in data['clients'].iteritems(): + for client, value in data['clients'].items(): # translate dots to underscores in client names metric = u'.'.join(['clients', client.translate(DOTS_TO_UNDERS)]) @@ -114,8 +114,8 @@ def collect(self): if action not in data: continue - for sect, stats in data[action].iteritems(): - for status, value in stats.iteritems(): + for sect, stats in data[action].items(): + for status, value in stats.items(): metric = '.'.join([action, sect, status.translate(DOTS_TO_UNDERS)]) @@ -125,7 +125,7 @@ def collect(self): self.publish(metric, dvalue, precision=4) if u'local' in data: - for key, value in data[u'local'].iteritems(): + for key, value in data[u'local'].items(): metric = '.'.join(['local', key]) dvalue = self.derivative(metric, value) diff --git a/src/collectors/postgres/postgres.py b/src/collectors/postgres/postgres.py index 613d4339a..b9fd5aca8 100644 --- a/src/collectors/postgres/postgres.py +++ b/src/collectors/postgres/postgres.py @@ -220,7 +220,7 @@ def fetch(self, pg_version): # If row > length 2, assume each column name maps to # key => value else: - for key, value in row.iteritems(): + for key, value in row.items(): if key in ('datname', 'schemaname', 'relname', 'indexrelname', 'funcname',): continue diff --git a/src/collectors/processresources/processresources.py b/src/collectors/processresources/processresources.py index 05d7636d9..035017ede 100644 --- a/src/collectors/processresources/processresources.py +++ b/src/collectors/processresources/processresources.py @@ -81,7 +81,7 @@ def process_info(process, info_keys): if type(value) in [float, int]: results.update({key: value}) elif hasattr(value, '_asdict'): - for subkey, subvalue in value._asdict().iteritems(): + for subkey, subvalue in value._asdict().items(): results.update({"%s.%s" % (key, subkey): subvalue}) return results @@ -158,7 +158,7 @@ def get_default_config(self): return config def save_process_info(self, pg_name, process_info): - for key, value in process_info.iteritems(): + for key, value in process_info.items(): if key in self.processes_info[pg_name]: self.processes_info[pg_name][key] += value else: @@ -198,11 +198,11 @@ def collect(self): self.collect_process_info(process) # publish results - for pg_name, counters in self.processes_info.iteritems(): + for pg_name, counters in self.processes_info.items(): if counters: metrics = ( ("%s.%s" % (pg_name, key), value) - for key, value in counters.iteritems()) + for key, value in counters.items()) else: if self.processes[pg_name]['count_workers']: metrics = (('%s.workers_count' % pg_name, 0), ) diff --git a/src/collectors/puppetagent/puppetagent.py b/src/collectors/puppetagent/puppetagent.py index 579cc0ece..501bd7f9f 100644 --- a/src/collectors/puppetagent/puppetagent.py +++ b/src/collectors/puppetagent/puppetagent.py @@ -55,8 +55,8 @@ def collect(self): summary = self._get_summary() - for sect, data in summary.iteritems(): - for stat, value in data.iteritems(): + for sect, data in summary.items(): + for stat, value in data.items(): if value is None or isinstance(value, basestring): continue diff --git a/src/collectors/slony/slony.py b/src/collectors/slony/slony.py index 1a2ef9dc5..ae0c91eee 100644 --- a/src/collectors/slony/slony.py +++ b/src/collectors/slony/slony.py @@ -94,7 +94,7 @@ def collect(self): } } - for name, instance in instances.iteritems(): + for name, instance in instances.items(): host = self.config['host'] port = self.config['port'] user = instance.get('user') or self.config['user'] diff --git a/src/collectors/twemproxy/twemproxy.py b/src/collectors/twemproxy/twemproxy.py index 0df1b2b28..6ebf5cf87 100644 --- a/src/collectors/twemproxy/twemproxy.py +++ b/src/collectors/twemproxy/twemproxy.py @@ -113,17 +113,17 @@ def get_stats(self, host, port): stats = {} pools = {} - for stat, value in data.iteritems(): + for stat, value in data.items(): # Test if this is a pool if isinstance(value, dict): pool_name = stat.replace('.', '_') pools[pool_name] = {} - for pool_stat, pool_value in value.iteritems(): + for pool_stat, pool_value in value.items(): # Test if this is a pool server if isinstance(pool_value, dict): server_name = pool_stat.replace('.', '_') pools[pool_name][server_name] = {} - for server_stat, server_value in pool_value.iteritems(): + for server_stat, server_value in pool_value.items(): pools[pool_name][server_name][server_stat] = \ int(server_value) else: @@ -161,11 +161,11 @@ def collect(self): self.publish_counter(alias + "." + stat, stats[stat]) # Pool stats - for pool, pool_stats in pools.iteritems(): - for stat, stat_value in pool_stats.iteritems(): + for pool, pool_stats in pools.items(): + for stat, stat_value in pool_stats.items(): # Test if this is a pool server if isinstance(stat_value, dict): - for server_stat, server_value in stat_value.iteritems(): + for server_stat, server_value in stat_value.items(): if server_stat in self.GAUGES: self.publish_gauge( alias + ".pools." + pool + ".servers." + diff --git a/src/collectors/unbound/unbound.py b/src/collectors/unbound/unbound.py index ff17d53ad..177eb15d9 100644 --- a/src/collectors/unbound/unbound.py +++ b/src/collectors/unbound/unbound.py @@ -86,5 +86,5 @@ def collect(self): if include_hist: histogram = self.get_massaged_histogram(raw_histogram) - for intv, value in histogram.iteritems(): + for intv, value in histogram.items(): self.publish('histogram.' + intv, value) diff --git a/src/diamond/server.py b/src/diamond/server.py index c4a3c77ae..c94cf9e25 100644 --- a/src/diamond/server.py +++ b/src/diamond/server.py @@ -146,7 +146,7 @@ def run(self): ############################################################## running_collectors = [] - for collector, config in self.config['collectors'].iteritems(): + for collector, config in self.config['collectors'].items(): if config.get('enabled', False) is not True: continue running_collectors.append(collector) diff --git a/test.py b/test.py index 00d62a03d..ee6b4600e 100755 --- a/test.py +++ b/test.py @@ -183,7 +183,7 @@ def assertUnpublishedMany(self, mock, dict, expected_value=0): return self.assertPublishedMany(mock, dict, expected_value) def assertPublishedMany(self, mock, dict, expected_value=1): - for key, value in dict.iteritems(): + for key, value in dict.items(): self.assertPublished(mock, key, value, expected_value) if type(mock) is list: @@ -229,7 +229,7 @@ def assertUnpublishedMetricMany(self, mock, dict, expected_value=0): return self.assertPublishedMetricMany(mock, dict, expected_value) def assertPublishedMetricMany(self, mock, dict, expected_value=1): - for key, value in dict.iteritems(): + for key, value in dict.items(): self.assertPublishedMetric(mock, key, value, expected_value) mock.reset_mock() From 3afd21b8e77893d1a9614a3fbbb9ca5e14a16787 Mon Sep 17 00:00:00 2001 From: Hannu Valtonen Date: Mon, 24 Aug 2015 14:16:21 +0300 Subject: [PATCH 07/33] test: make filter iterators into lists for py3k compatibility --- test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test.py b/test.py index ee6b4600e..c90bcf6ae 100755 --- a/test.py +++ b/test.py @@ -147,11 +147,11 @@ def assertUnpublished(self, mock, key, value, expected_value=0): def assertPublished(self, mock, key, value, expected_value=1): if type(mock) is list: for m in mock: - calls = (filter(lambda x: x[0][0] == key, m.call_args_list)) + calls = list(filter(lambda x: x[0][0] == key, m.call_args_list)) if len(calls) > 0: break else: - calls = filter(lambda x: x[0][0] == key, mock.call_args_list) + calls = list(filter(lambda x: x[0][0] == key, mock.call_args_list)) actual_value = len(calls) message = '%s: actual number of calls %d, expected %d' % ( @@ -196,8 +196,8 @@ def assertUnpublishedMetric(self, mock, key, value, expected_value=0): return self.assertPublishedMetric(mock, key, value, expected_value) def assertPublishedMetric(self, mock, key, value, expected_value=1): - calls = filter(lambda x: x[0][0].path.find(key) != -1, - mock.call_args_list) + calls = list(filter(lambda x: x[0][0].path.find(key) != -1, + mock.call_args_list)) actual_value = len(calls) message = '%s: actual number of calls %d, expected %d' % ( From 8bd1acfc9a91a7f9ac5285e74e704cef9b5f5a26 Mon Sep 17 00:00:00 2001 From: Hannu Valtonen Date: Mon, 24 Aug 2015 14:19:07 +0300 Subject: [PATCH 08/33] py3k: replace xrange with range for compatibility None of the callsites is performance critical (the ranges in question are tiny) and this allows for the same code to be run on py3k. --- src/collectors/ip/ip.py | 2 +- src/collectors/solr/solr.py | 2 +- src/collectors/tcp/tcp.py | 2 +- src/collectors/udp/udp.py | 2 +- src/diamond/test/testmetric.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/collectors/ip/ip.py b/src/collectors/ip/ip.py index 5d916b4a8..3e70228d7 100644 --- a/src/collectors/ip/ip.py +++ b/src/collectors/ip/ip.py @@ -103,7 +103,7 @@ def collect(self): data = data.split() # Zip up the keys and values - for i in xrange(1, len(header)): + for i in range(1, len(header)): metrics[header[i]] = data[i] for metric_name in metrics.keys(): diff --git a/src/collectors/solr/solr.py b/src/collectors/solr/solr.py index 43d5f4c3f..1c247fee6 100644 --- a/src/collectors/solr/solr.py +++ b/src/collectors/solr/solr.py @@ -131,7 +131,7 @@ def collect(self): continue s = result['solr-mbeans'] - stats = dict((s[i], s[i + 1]) for i in xrange(0, len(s), 2)) + stats = dict((s[i], s[i + 1]) for i in range(0, len(s), 2)) if 'core' in self.config['stats']: core_searcher = stats["CORE"]["searcher"]["stats"] diff --git a/src/collectors/tcp/tcp.py b/src/collectors/tcp/tcp.py index a357dd65b..18b664c4b 100644 --- a/src/collectors/tcp/tcp.py +++ b/src/collectors/tcp/tcp.py @@ -260,7 +260,7 @@ def collect(self): header = header.split() data = data.split() - for i in xrange(1, len(header)): + for i in range(1, len(header)): metrics[header[i]] = data[i] for metric_name in metrics.keys(): diff --git a/src/collectors/udp/udp.py b/src/collectors/udp/udp.py index 8e4f04acc..d9872fc15 100644 --- a/src/collectors/udp/udp.py +++ b/src/collectors/udp/udp.py @@ -83,7 +83,7 @@ def collect(self): header = header.split() data = data.split() - for i in xrange(1, len(header)): + for i in range(1, len(header)): metrics[header[i]] = data[i] for metric_name in metrics.keys(): diff --git a/src/diamond/test/testmetric.py b/src/diamond/test/testmetric.py index 365c3f742..e870ebd04 100644 --- a/src/diamond/test/testmetric.py +++ b/src/diamond/test/testmetric.py @@ -101,7 +101,7 @@ def test_issue_723(self): '0.0000997143369909', ] - for precision in xrange(0, 100): + for precision in range(0, 100): for m in metrics: metric = Metric('test.723', m, timestamp=0) From f28b2352ed532dc5788ea7a04336a434920a641d Mon Sep 17 00:00:00 2001 From: Hannu Valtonen Date: Mon, 24 Aug 2015 14:23:03 +0300 Subject: [PATCH 09/33] py3k: Replace all uses of basestring with isinstance(x, str) The callsites seemed to all parse the config, making them just strings instead of possibly unicode string on py2.x --- bin/diamond-setup | 8 ++++---- src/collectors/conntrack/conntrack.py | 4 ++-- src/collectors/darner/darner.py | 2 +- src/collectors/diskspace/diskspace.py | 4 ++-- src/collectors/elasticsearch/elasticsearch.py | 2 +- src/collectors/filestat/filestat.py | 12 ++++++------ src/collectors/httpd/httpd.py | 2 +- src/collectors/jcollectd/collectd_network.py | 2 +- src/collectors/jolokia/jolokia.py | 2 +- src/collectors/memcached/memcached.py | 2 +- src/collectors/mongodb/mongodb.py | 2 +- src/collectors/mountstats/mountstats.py | 4 ++-- src/collectors/openvpn/openvpn.py | 2 +- src/collectors/puppetagent/puppetagent.py | 2 +- src/collectors/redisstat/redisstat.py | 2 +- src/collectors/twemproxy/twemproxy.py | 2 +- src/collectors/zookeeper/zookeeper.py | 2 +- src/diamond/handler/libratohandler.py | 2 +- src/diamond/server.py | 4 ++-- src/diamond/utils/classes.py | 6 +++--- src/diamond/utils/config.py | 2 +- test.py | 6 ++---- 22 files changed, 37 insertions(+), 39 deletions(-) diff --git a/bin/diamond-setup b/bin/diamond-setup index 45eb0730e..eb9f1a5e5 100755 --- a/bin/diamond-setup +++ b/bin/diamond-setup @@ -78,7 +78,7 @@ def getCollectors(path): def typeToString(key): - if isinstance(obj.config[key], basestring): + if isinstance(obj.config[key], str): user_val = obj.config[key] elif isinstance(obj.config[key], bool): user_val = str(obj.config[key]) @@ -95,7 +95,7 @@ def typeToString(key): def stringToType(key, val): if type(obj.config[key]) is type(val): config_file[key] = val - elif isinstance(obj.config[key], basestring): + elif isinstance(obj.config[key], str): if val.lower() == 'false': config_file[key] = False elif val.lower() == 'true': @@ -103,7 +103,7 @@ def stringToType(key, val): else: config_file[key] = val elif isinstance(obj.config[key], bool): - if isinstance(val, basestring): + if isinstance(val, str): config_file[key] = str_to_bool(val) else: config_file[key] = bool(val) @@ -117,7 +117,7 @@ def stringToType(key, val): def boolCheck(val): - if isinstance(val, basestring): + if isinstance(val, str): return str_to_bool(val) elif isinstance(val, bool): return val diff --git a/src/collectors/conntrack/conntrack.py b/src/collectors/conntrack/conntrack.py index c335545d1..5802e6a5a 100644 --- a/src/collectors/conntrack/conntrack.py +++ b/src/collectors/conntrack/conntrack.py @@ -50,12 +50,12 @@ def collect(self): collected = {} files = [] - if isinstance(self.config['dir'], basestring): + if isinstance(self.config['dir'], str): dirs = [d.strip() for d in self.config['dir'].split(',')] elif isinstance(self.config['dir'], list): dirs = self.config['dir'] - if isinstance(self.config['files'], basestring): + if isinstance(self.config['files'], str): files = [f.strip() for f in self.config['files'].split(',')] elif isinstance(self.config['files'], list): files = self.config['files'] diff --git a/src/collectors/darner/darner.py b/src/collectors/darner/darner.py index 33f69b10f..d40d9aa0b 100644 --- a/src/collectors/darner/darner.py +++ b/src/collectors/darner/darner.py @@ -117,7 +117,7 @@ def collect(self): hosts = self.config.get('hosts') # Convert a string config value to be an array - if isinstance(hosts, basestring): + if isinstance(hosts, str): hosts = [hosts] for host in hosts: diff --git a/src/collectors/diskspace/diskspace.py b/src/collectors/diskspace/diskspace.py index 330ace1bf..adff2f99d 100644 --- a/src/collectors/diskspace/diskspace.py +++ b/src/collectors/diskspace/diskspace.py @@ -78,7 +78,7 @@ def process_config(self): super(DiskSpaceCollector, self).process_config() # Precompile things self.exclude_filters = self.config['exclude_filters'] - if isinstance(self.exclude_filters, basestring): + if isinstance(self.exclude_filters, str): self.exclude_filters = [self.exclude_filters] if not self.exclude_filters: @@ -87,7 +87,7 @@ def process_config(self): self.exclude_reg = re.compile('|'.join(self.exclude_filters)) self.filesystems = [] - if isinstance(self.config['filesystems'], basestring): + if isinstance(self.config['filesystems'], str): for filesystem in self.config['filesystems'].split(','): self.filesystems.append(filesystem.strip()) elif isinstance(self.config['filesystems'], list): diff --git a/src/collectors/elasticsearch/elasticsearch.py b/src/collectors/elasticsearch/elasticsearch.py index 739d95d51..e9ec4d6a9 100755 --- a/src/collectors/elasticsearch/elasticsearch.py +++ b/src/collectors/elasticsearch/elasticsearch.py @@ -33,7 +33,7 @@ class ElasticSearchCollector(diamond.collector.Collector): def process_config(self): super(ElasticSearchCollector, self).process_config() instance_list = self.config['instances'] - if isinstance(instance_list, basestring): + if isinstance(instance_list, str): instance_list = [instance_list] if len(instance_list) == 0: diff --git a/src/collectors/filestat/filestat.py b/src/collectors/filestat/filestat.py index fe3a17557..becf5d021 100644 --- a/src/collectors/filestat/filestat.py +++ b/src/collectors/filestat/filestat.py @@ -116,13 +116,13 @@ def get_userlist(self): based on the variables user_include and user_exclude """ # convert user/group lists to arrays if strings - if isinstance(self.config['user_include'], basestring): + if isinstance(self.config['user_include'], str): self.config['user_include'] = self.config['user_include'].split() - if isinstance(self.config['user_exclude'], basestring): + if isinstance(self.config['user_exclude'], str): self.config['user_exclude'] = self.config['user_exclude'].split() - if isinstance(self.config['group_include'], basestring): + if isinstance(self.config['group_include'], str): self.config['group_include'] = self.config['group_include'].split() - if isinstance(self.config['group_exclude'], basestring): + if isinstance(self.config['group_exclude'], str): self.config['group_exclude'] = self.config['group_exclude'].split() rawusers = os.popen("lsof | awk '{ print $3 }' | sort | uniq -d" @@ -206,9 +206,9 @@ def get_typelist(self): typelist = [] # convert type list into arrays if strings - if isinstance(self.config['type_include'], basestring): + if isinstance(self.config['type_include'], str): self.config['type_include'] = self.config['type_include'].split() - if isinstance(self.config['type_exclude'], basestring): + if isinstance(self.config['type_exclude'], str): self.config['type_exclude'] = self.config['type_exclude'].split() # remove any not in include list diff --git a/src/collectors/httpd/httpd.py b/src/collectors/httpd/httpd.py index f4b589a09..81463725c 100644 --- a/src/collectors/httpd/httpd.py +++ b/src/collectors/httpd/httpd.py @@ -25,7 +25,7 @@ def process_config(self): self.config['urls'].append(self.config['url']) self.urls = {} - if isinstance(self.config['urls'], basestring): + if isinstance(self.config['urls'], str): self.config['urls'] = self.config['urls'].split(',') for url in self.config['urls']: diff --git a/src/collectors/jcollectd/collectd_network.py b/src/collectors/jcollectd/collectd_network.py index 4cda897d8..224066576 100644 --- a/src/collectors/jcollectd/collectd_network.py +++ b/src/collectors/jcollectd/collectd_network.py @@ -367,7 +367,7 @@ def interpret(self, iterable=None, poll_interval=0.2): if iterable is None: return None - if isinstance(iterable, basestring): + if isinstance(iterable, str): iterable = self.decode(poll_interval, iterable) return interpret_opcodes(iterable) diff --git a/src/collectors/jolokia/jolokia.py b/src/collectors/jolokia/jolokia.py index 17cda5b2e..c242bc1ee 100644 --- a/src/collectors/jolokia/jolokia.py +++ b/src/collectors/jolokia/jolokia.py @@ -120,7 +120,7 @@ def get_default_config(self): def __init__(self, *args, **kwargs): super(JolokiaCollector, self).__init__(*args, **kwargs) self.mbeans = [] - if isinstance(self.config['mbeans'], basestring): + if isinstance(self.config['mbeans'], str): for mbean in self.config['mbeans'].split('|'): self.mbeans.append(mbean.strip()) elif isinstance(self.config['mbeans'], list): diff --git a/src/collectors/memcached/memcached.py b/src/collectors/memcached/memcached.py index 22a9fbad5..00c021415 100644 --- a/src/collectors/memcached/memcached.py +++ b/src/collectors/memcached/memcached.py @@ -151,7 +151,7 @@ def collect(self): hosts = self.config.get('hosts') # Convert a string config value to be an array - if isinstance(hosts, basestring): + if isinstance(hosts, str): hosts = [hosts] for host in hosts: diff --git a/src/collectors/mongodb/mongodb.py b/src/collectors/mongodb/mongodb.py index 3fbb1420b..7e5f83040 100644 --- a/src/collectors/mongodb/mongodb.py +++ b/src/collectors/mongodb/mongodb.py @@ -115,7 +115,7 @@ def collect(self): hosts = self.config.get('hosts') # Convert a string config value to be an array - if isinstance(hosts, basestring): + if isinstance(hosts, str): hosts = [hosts] # we need this for backwards compatibility diff --git a/src/collectors/mountstats/mountstats.py b/src/collectors/mountstats/mountstats.py index f8d699657..abc9276ff 100755 --- a/src/collectors/mountstats/mountstats.py +++ b/src/collectors/mountstats/mountstats.py @@ -60,7 +60,7 @@ class MountStatsCollector(diamond.collector.Collector): def process_config(self): super(MountStatsCollector, self).process_config() self.exclude_filters = self.config['exclude_filters'] - if isinstance(self.exclude_filters, basestring): + if isinstance(self.exclude_filters, str): self.exclude_filters = [self.exclude_filters] if len(self.exclude_filters) > 0: @@ -69,7 +69,7 @@ def process_config(self): self.exclude_reg = None self.include_filters = self.config['include_filters'] - if isinstance(self.include_filters, basestring): + if isinstance(self.include_filters, str): self.include_filters = [self.include_filters] if len(self.include_filters) > 0: diff --git a/src/collectors/openvpn/openvpn.py b/src/collectors/openvpn/openvpn.py index dcc31c9c8..fe99b9f17 100644 --- a/src/collectors/openvpn/openvpn.py +++ b/src/collectors/openvpn/openvpn.py @@ -86,7 +86,7 @@ class Object(object): return parsed def collect(self): - if isinstance(self.config['instances'], basestring): + if isinstance(self.config['instances'], str): instances = [self.config['instances']] else: instances = self.config['instances'] diff --git a/src/collectors/puppetagent/puppetagent.py b/src/collectors/puppetagent/puppetagent.py index 501bd7f9f..d4c117495 100644 --- a/src/collectors/puppetagent/puppetagent.py +++ b/src/collectors/puppetagent/puppetagent.py @@ -57,7 +57,7 @@ def collect(self): for sect, data in summary.items(): for stat, value in data.items(): - if value is None or isinstance(value, basestring): + if value is None or isinstance(value, str): continue metric = '.'.join([sect, stat]) diff --git a/src/collectors/redisstat/redisstat.py b/src/collectors/redisstat/redisstat.py index 9ca94736c..a8d1f91c4 100644 --- a/src/collectors/redisstat/redisstat.py +++ b/src/collectors/redisstat/redisstat.py @@ -110,7 +110,7 @@ def process_config(self): super(RedisCollector, self).process_config() instance_list = self.config['instances'] # configobj make str of single-element list, let's convert - if isinstance(instance_list, basestring): + if isinstance(instance_list, str): instance_list = [instance_list] # process original single redis instance diff --git a/src/collectors/twemproxy/twemproxy.py b/src/collectors/twemproxy/twemproxy.py index 6ebf5cf87..58549de94 100644 --- a/src/collectors/twemproxy/twemproxy.py +++ b/src/collectors/twemproxy/twemproxy.py @@ -140,7 +140,7 @@ def collect(self): hosts = self.config.get('hosts') # Convert a string config value to be an array - if isinstance(hosts, basestring): + if isinstance(hosts, str): hosts = [hosts] for host in hosts: diff --git a/src/collectors/zookeeper/zookeeper.py b/src/collectors/zookeeper/zookeeper.py index b132ba25c..393e504d3 100644 --- a/src/collectors/zookeeper/zookeeper.py +++ b/src/collectors/zookeeper/zookeeper.py @@ -118,7 +118,7 @@ def collect(self): hosts = self.config.get('hosts') # Convert a string config value to be an array - if isinstance(hosts, basestring): + if isinstance(hosts, str): hosts = [hosts] for host in hosts: diff --git a/src/diamond/handler/libratohandler.py b/src/diamond/handler/libratohandler.py index c080ded31..f4a565cc0 100644 --- a/src/diamond/handler/libratohandler.py +++ b/src/diamond/handler/libratohandler.py @@ -48,7 +48,7 @@ def __init__(self, config=None): # If a user leaves off the ending comma, cast to a array for them include_filters = self.config['include_filters'] - if isinstance(include_filters, basestring): + if isinstance(include_filters, str): include_filters = [include_filters] self.include_reg = re.compile(r'(?:%s)' % '|'.join(include_filters)) diff --git a/src/diamond/server.py b/src/diamond/server.py index c94cf9e25..75f86bd35 100644 --- a/src/diamond/server.py +++ b/src/diamond/server.py @@ -86,7 +86,7 @@ def run(self): handlers_path = self.config['server']['handlers_path'] # Make an list if not one - if isinstance(handlers_path, basestring): + if isinstance(handlers_path, str): handlers_path = handlers_path.split(',') handlers_path = map(str.strip, handlers_path) self.config['server']['handlers_path'] = handlers_path @@ -98,7 +98,7 @@ def run(self): sys.exit(1) handlers = self.config['server'].get('handlers') - if isinstance(handlers, basestring): + if isinstance(handlers, str): handlers = [handlers] # Prevent the Queue Handler from being a normal handler diff --git a/src/diamond/utils/classes.py b/src/diamond/utils/classes.py index 51613cf11..1f756c5f1 100644 --- a/src/diamond/utils/classes.py +++ b/src/diamond/utils/classes.py @@ -40,7 +40,7 @@ def load_dynamic_class(fqn, subclass): """ Dynamically load fqn class and verify it's a subclass of subclass """ - if not isinstance(fqn, basestring): + if not isinstance(fqn, str): return fqn cls = load_class_from_name(fqn) @@ -58,7 +58,7 @@ def load_handlers(config, handler_names): handlers = [] - if isinstance(handler_names, basestring): + if isinstance(handler_names, str): handler_names = [handler_names] for handler in handler_names: @@ -119,7 +119,7 @@ def load_collectors_from_paths(paths): if paths is None: return - if isinstance(paths, basestring): + if isinstance(paths, str): paths = paths.split(',') paths = map(str.strip, paths) diff --git a/src/diamond/utils/config.py b/src/diamond/utils/config.py index f7dec11e8..bbf828015 100644 --- a/src/diamond/utils/config.py +++ b/src/diamond/utils/config.py @@ -9,7 +9,7 @@ def str_to_bool(value): Converts string truthy/falsey strings to a bool Empty strings are false """ - if isinstance(value, basestring): + if isinstance(value, str): value = value.strip().lower() if value in ['true', 't', 'yes', 'y']: return True diff --git a/test.py b/test.py index c90bcf6ae..792cd648a 100755 --- a/test.py +++ b/test.py @@ -35,11 +35,9 @@ setproctitle = None try: - from mock import Mock - from mock import patch + from mock import call, Mock, patch except ImportError: - from unittest.mock import Mock - from unittest.mock import patch + from unittest.mock import call, Mock, patch sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__)))) sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), From 0ed4d81b4ea80fdbb0262652cac747937eb27590 Mon Sep 17 00:00:00 2001 From: Hannu Valtonen Date: Mon, 24 Aug 2015 14:37:39 +0300 Subject: [PATCH 10/33] py3k: use string.ascii_letters since it also exists in py3k --- src/collectors/scribe/scribe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/collectors/scribe/scribe.py b/src/collectors/scribe/scribe.py index d31cef02e..31999598f 100644 --- a/src/collectors/scribe/scribe.py +++ b/src/collectors/scribe/scribe.py @@ -36,7 +36,7 @@ def get_default_config(self): def key_to_metric(self, key): """Replace all non-letter characters with underscores""" - return ''.join(l if l in string.letters else '_' for l in key) + return ''.join(l if l in string.ascii_letters else '_' for l in key) def get_scribe_ctrl_output(self): cmd = [self.config['scribe_ctrl_bin'], 'counters'] From 9addc26022a7c67ed039a265da9d7c8497a67b36 Mon Sep 17 00:00:00 2001 From: Hannu Valtonen Date: Mon, 24 Aug 2015 14:40:24 +0300 Subject: [PATCH 11/33] py3k: Replace the obsolete use of string.replace with .replace() --- src/collectors/ipvs/ipvs.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/collectors/ipvs/ipvs.py b/src/collectors/ipvs/ipvs.py index 3dddb76db..fd4961faf 100644 --- a/src/collectors/ipvs/ipvs.py +++ b/src/collectors/ipvs/ipvs.py @@ -12,7 +12,6 @@ import diamond.collector import subprocess import os -import string from diamond.collector import str_to_bool @@ -94,10 +93,10 @@ def collect(self): row = line.split() if row[0] == "TCP" or row[0] == "UDP": - external = row[0] + "_" + string.replace(row[1], ".", "_") + external = row[0] + "_" + row[1].replace(".", "_") backend = "total" elif row[0] == "->": - backend = string.replace(row[1], ".", "_") + backend = row[1].replace(".", "_") else: continue @@ -142,10 +141,10 @@ def collect(self): for k in columns.keys(): total[k] = 0.0 - external = row[0] + "_" + string.replace(row[1], ".", "_") + external = row[0] + "_" + row[1].replace(".", "_") continue elif row[0] == "->": - backend = string.replace(row[1], ".", "_") + backend = row[1].replace(".", "_") else: continue From d0e46bd8e01000233ce8f6ad6c59fecf91a019e8 Mon Sep 17 00:00:00 2001 From: Hannu Valtonen Date: Mon, 24 Aug 2015 14:45:35 +0300 Subject: [PATCH 12/33] py3k: Replace some more uses of call and magicmock from the one in base test --- src/collectors/mongodb/test/testmongodb.py | 4 ++-- src/collectors/mountstats/test/testmountstats.py | 2 +- src/collectors/pgq/test/testpgq.py | 2 +- src/collectors/portstat/tests/test_portstat.py | 2 +- src/collectors/smart/test/testsmart.py | 2 +- src/collectors/sockstat/test/testsockstat.py | 2 +- src/collectors/tokumx/test/testtokumx.py | 4 ++-- src/diamond/handler/test/testgraphitehandler.py | 2 +- test.py | 4 ++-- 9 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/collectors/mongodb/test/testmongodb.py b/src/collectors/mongodb/test/testmongodb.py index 3c9896910..b8b58d3f6 100644 --- a/src/collectors/mongodb/test/testmongodb.py +++ b/src/collectors/mongodb/test/testmongodb.py @@ -6,9 +6,9 @@ from test import get_collector_config from test import unittest from test import run_only -from mock import MagicMock +from test import MagicMock from test import patch -from mock import call +from test import call from diamond.collector import Collector from mongodb import MongoDBCollector diff --git a/src/collectors/mountstats/test/testmountstats.py b/src/collectors/mountstats/test/testmountstats.py index 1bf51ef2b..767e90c9a 100644 --- a/src/collectors/mountstats/test/testmountstats.py +++ b/src/collectors/mountstats/test/testmountstats.py @@ -4,7 +4,7 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest -from mock import MagicMock +from test import MagicMock from test import patch from test import Mock diff --git a/src/collectors/pgq/test/testpgq.py b/src/collectors/pgq/test/testpgq.py index 153b84777..40014376f 100644 --- a/src/collectors/pgq/test/testpgq.py +++ b/src/collectors/pgq/test/testpgq.py @@ -1,7 +1,7 @@ #!/usr/bin/python from test import CollectorTestCase, get_collector_config from test import patch -from mock import MagicMock +from test import MagicMock from pgq import PgQCollector diff --git a/src/collectors/portstat/tests/test_portstat.py b/src/collectors/portstat/tests/test_portstat.py index 01b948f79..992d2b975 100644 --- a/src/collectors/portstat/tests/test_portstat.py +++ b/src/collectors/portstat/tests/test_portstat.py @@ -2,8 +2,8 @@ from test import get_collector_config from test import patch from test import Mock +from test import call -from mock import call from unittest import TestCase from diamond.collector import Collector diff --git a/src/collectors/smart/test/testsmart.py b/src/collectors/smart/test/testsmart.py index 4c534bfd3..212d67416 100644 --- a/src/collectors/smart/test/testsmart.py +++ b/src/collectors/smart/test/testsmart.py @@ -6,7 +6,7 @@ from test import get_collector_config from test import unittest from test import Mock -from mock import call +from test import call from test import patch from diamond.collector import Collector diff --git a/src/collectors/sockstat/test/testsockstat.py b/src/collectors/sockstat/test/testsockstat.py index 30a85278a..585a35369 100644 --- a/src/collectors/sockstat/test/testsockstat.py +++ b/src/collectors/sockstat/test/testsockstat.py @@ -7,7 +7,7 @@ from test import unittest from test import Mock from test import patch -from mock import call +from test import call from collections import Iterator from diamond.collector import Collector diff --git a/src/collectors/tokumx/test/testtokumx.py b/src/collectors/tokumx/test/testtokumx.py index 263d3feab..69fc79d1c 100644 --- a/src/collectors/tokumx/test/testtokumx.py +++ b/src/collectors/tokumx/test/testtokumx.py @@ -6,9 +6,9 @@ from test import get_collector_config from test import unittest from test import run_only -from mock import MagicMock +from test import MagicMock from test import patch -from mock import call +from test import call from diamond.collector import Collector from tokumx import TokuMXCollector diff --git a/src/diamond/handler/test/testgraphitehandler.py b/src/diamond/handler/test/testgraphitehandler.py index 0b7fda3f8..70728cdd8 100644 --- a/src/diamond/handler/test/testgraphitehandler.py +++ b/src/diamond/handler/test/testgraphitehandler.py @@ -7,7 +7,7 @@ from test import unittest from test import Mock from test import patch -from mock import call +from test import call import configobj diff --git a/test.py b/test.py index 792cd648a..aee1e2ef1 100755 --- a/test.py +++ b/test.py @@ -35,9 +35,9 @@ setproctitle = None try: - from mock import call, Mock, patch + from mock import call, MagicMock, Mock, patch except ImportError: - from unittest.mock import call, Mock, patch + from unittest.mock import call, MagicMock, Mock, patch sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__)))) sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), From 4cb8a4cdc9034a31cf45d579f2de936f9122b76f Mon Sep 17 00:00:00 2001 From: Hannu Valtonen Date: Mon, 24 Aug 2015 15:02:10 +0300 Subject: [PATCH 13/33] tests: import StringIO from the base test file We now import the most suitable StringIO to the platform from a single place instead of from n files. --- src/collectors/cpu/cpu.py | 4 ++-- src/collectors/cpu/test/testcpu.py | 8 ++------ .../cpuacct_cgroup/test/testcpuacct_cgroup.py | 6 +----- src/collectors/filestat/test/testfilestat.py | 6 +----- src/collectors/interrupt/test/testinterrupt.py | 6 +----- src/collectors/interrupt/test/testsoft.py | 6 +----- src/collectors/ip/test/testip.py | 6 +----- src/collectors/kvm/test/testkvm.py | 6 +----- src/collectors/loadavg/test/testloadavg.py | 6 +----- src/collectors/memory/test/testmemory.py | 6 +----- .../memory_cgroup/test/testmemory_cgroup.py | 6 +----- .../memory_docker/test/testmemory_docker.py | 6 +----- src/collectors/mongodb/test/testmongodb.py | 6 ++++-- src/collectors/mountstats/mountstats.py | 12 ++++++------ src/collectors/network/test/testnetwork.py | 6 +----- src/collectors/nfs/test/testnfs.py | 6 +----- src/collectors/nfsd/test/testnfsd.py | 6 +----- src/collectors/proc/test/testproc.py | 6 +----- src/collectors/slabinfo/test/testslabinfo.py | 6 +----- src/collectors/tcp/test/testtcp.py | 6 +----- src/collectors/udp/test/testudp.py | 6 +----- src/collectors/uptime/test/test_uptime.py | 6 +----- src/collectors/vmstat/test/testvmstat.py | 6 +----- 23 files changed, 33 insertions(+), 111 deletions(-) diff --git a/src/collectors/cpu/cpu.py b/src/collectors/cpu/cpu.py index fdb93f9ce..6ada7855b 100644 --- a/src/collectors/cpu/cpu.py +++ b/src/collectors/cpu/cpu.py @@ -157,12 +157,12 @@ def cpu_delta_time(interval): ncpus > 0)): metrics[metric_name] = self.derivative( metric_name, - long(stats[s]), + int(stats[s]), self.MAX_VALUES[s]) / ncpus else: metrics[metric_name] = self.derivative( metric_name, - long(stats[s]), + int(stats[s]), self.MAX_VALUES[s]) # Check for a bug in xen where the idle time is doubled for guest diff --git a/src/collectors/cpu/test/testcpu.py b/src/collectors/cpu/test/testcpu.py index 34ac47209..89cfeef74 100644 --- a/src/collectors/cpu/test/testcpu.py +++ b/src/collectors/cpu/test/testcpu.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +i#!/usr/bin/python # coding=utf-8 ########################################################################## @@ -7,11 +7,7 @@ from test import unittest from test import Mock from test import patch - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from test import StringIO from diamond.collector import Collector from cpu import CPUCollector diff --git a/src/collectors/cpuacct_cgroup/test/testcpuacct_cgroup.py b/src/collectors/cpuacct_cgroup/test/testcpuacct_cgroup.py index b152c727b..cece2bb7f 100644 --- a/src/collectors/cpuacct_cgroup/test/testcpuacct_cgroup.py +++ b/src/collectors/cpuacct_cgroup/test/testcpuacct_cgroup.py @@ -6,11 +6,7 @@ from test import get_collector_config from test import unittest from test import patch - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from test import StringIO from diamond.collector import Collector from cpuacct_cgroup import CpuAcctCgroupCollector diff --git a/src/collectors/filestat/test/testfilestat.py b/src/collectors/filestat/test/testfilestat.py index c54feecfd..7413e35b6 100644 --- a/src/collectors/filestat/test/testfilestat.py +++ b/src/collectors/filestat/test/testfilestat.py @@ -7,11 +7,7 @@ from test import unittest from test import Mock from test import patch - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from test import StringIO from diamond.collector import Collector from filestat import FilestatCollector diff --git a/src/collectors/interrupt/test/testinterrupt.py b/src/collectors/interrupt/test/testinterrupt.py index 99907f642..6896214e7 100644 --- a/src/collectors/interrupt/test/testinterrupt.py +++ b/src/collectors/interrupt/test/testinterrupt.py @@ -7,11 +7,7 @@ from test import unittest from test import Mock from test import patch - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from test import StringIO from diamond.collector import Collector from interrupt import InterruptCollector diff --git a/src/collectors/interrupt/test/testsoft.py b/src/collectors/interrupt/test/testsoft.py index aa4e523e3..e36ad93ed 100644 --- a/src/collectors/interrupt/test/testsoft.py +++ b/src/collectors/interrupt/test/testsoft.py @@ -7,11 +7,7 @@ from test import unittest from test import Mock from test import patch - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from test import StringIO from diamond.collector import Collector from soft import SoftInterruptCollector diff --git a/src/collectors/ip/test/testip.py b/src/collectors/ip/test/testip.py index e18e4920d..f3d7fc191 100644 --- a/src/collectors/ip/test/testip.py +++ b/src/collectors/ip/test/testip.py @@ -7,11 +7,7 @@ from test import unittest from test import Mock from test import patch - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from test import StringIO from ip import IPCollector diff --git a/src/collectors/kvm/test/testkvm.py b/src/collectors/kvm/test/testkvm.py index 319b84780..f100333fd 100644 --- a/src/collectors/kvm/test/testkvm.py +++ b/src/collectors/kvm/test/testkvm.py @@ -9,11 +9,7 @@ from test import unittest from test import Mock from test import patch - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from test import StringIO from diamond.collector import Collector from kvm import KVMCollector diff --git a/src/collectors/loadavg/test/testloadavg.py b/src/collectors/loadavg/test/testloadavg.py index dd7bc734e..1f6963552 100644 --- a/src/collectors/loadavg/test/testloadavg.py +++ b/src/collectors/loadavg/test/testloadavg.py @@ -8,11 +8,7 @@ from test import unittest from test import Mock from test import patch - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from test import StringIO from diamond.collector import Collector from loadavg import LoadAverageCollector diff --git a/src/collectors/memory/test/testmemory.py b/src/collectors/memory/test/testmemory.py index 5ad1f2229..12a13f1d8 100644 --- a/src/collectors/memory/test/testmemory.py +++ b/src/collectors/memory/test/testmemory.py @@ -7,11 +7,7 @@ from test import unittest from test import Mock from test import patch - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from test import StringIO from diamond.collector import Collector from memory import MemoryCollector diff --git a/src/collectors/memory_cgroup/test/testmemory_cgroup.py b/src/collectors/memory_cgroup/test/testmemory_cgroup.py index c6ca6a959..d3444d034 100644 --- a/src/collectors/memory_cgroup/test/testmemory_cgroup.py +++ b/src/collectors/memory_cgroup/test/testmemory_cgroup.py @@ -7,11 +7,7 @@ from test import unittest from test import Mock from test import patch - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from test import StringIO from diamond.collector import Collector from memory_cgroup import MemoryCgroupCollector diff --git a/src/collectors/memory_docker/test/testmemory_docker.py b/src/collectors/memory_docker/test/testmemory_docker.py index 8317ed1cb..fb4f07ea0 100644 --- a/src/collectors/memory_docker/test/testmemory_docker.py +++ b/src/collectors/memory_docker/test/testmemory_docker.py @@ -8,11 +8,7 @@ from test import run_only from test import Mock from test import patch - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from test import StringIO try: from docker import Client diff --git a/src/collectors/mongodb/test/testmongodb.py b/src/collectors/mongodb/test/testmongodb.py index b8b58d3f6..2696ba071 100644 --- a/src/collectors/mongodb/test/testmongodb.py +++ b/src/collectors/mongodb/test/testmongodb.py @@ -19,8 +19,10 @@ except ImportError: import simplejson as json - -########################################################################## +try: + long +except NameError: + long = int def run_only_if_pymongo_is_available(func): diff --git a/src/collectors/mountstats/mountstats.py b/src/collectors/mountstats/mountstats.py index abc9276ff..f66503c60 100755 --- a/src/collectors/mountstats/mountstats.py +++ b/src/collectors/mountstats/mountstats.py @@ -161,12 +161,12 @@ def collect(self): elif tokens[0] == 'events:': for i in range(0, len(self.EVENTS_MAP)): metric_name = "%s.events.%s" % (path, self.EVENTS_MAP[i]) - metric_value = long(tokens[i + 1]) + metric_value = int(tokens[i + 1]) self.publish_counter(metric_name, metric_value) elif tokens[0] == 'bytes:': for i in range(0, len(self.BYTES_MAP)): metric_name = "%s.bytes.%s" % (path, self.BYTES_MAP[i]) - metric_value = long(tokens[i + 1]) + metric_value = int(tokens[i + 1]) self.publish_counter(metric_name, metric_value) elif tokens[0] == 'xprt:': proto = tokens[1] @@ -177,13 +177,13 @@ def collect(self): for i in range(0, len(self.XPRT_MAP[proto])): metric_name = "%s.xprt.%s.%s" % (path, proto, self.XPRT_MAP[proto][i]) - metric_value = long(tokens[i + 2]) + metric_value = int(tokens[i + 2]) self.publish_counter(metric_name, metric_value) elif tokens[0][:-1] in self.RPCS_MAP: rpc = tokens[0][:-1] - ops = long(tokens[1]) - rtt = long(tokens[7]) - exe = long(tokens[8]) + ops = int(tokens[1]) + rtt = int(tokens[7]) + exe = int(tokens[8]) metric_fmt = "%s.rpc.%s.%s" ops_name = metric_fmt % (path, rpc.lower(), 'ops') diff --git a/src/collectors/network/test/testnetwork.py b/src/collectors/network/test/testnetwork.py index ead42d586..c79ff8e2c 100644 --- a/src/collectors/network/test/testnetwork.py +++ b/src/collectors/network/test/testnetwork.py @@ -7,11 +7,7 @@ from test import unittest from test import Mock from test import patch - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from test import StringIO from diamond.collector import Collector from network import NetworkCollector diff --git a/src/collectors/nfs/test/testnfs.py b/src/collectors/nfs/test/testnfs.py index 634067c1b..4eb6f43eb 100644 --- a/src/collectors/nfs/test/testnfs.py +++ b/src/collectors/nfs/test/testnfs.py @@ -7,11 +7,7 @@ from test import unittest from test import Mock from test import patch - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from test import StringIO from diamond.collector import Collector from nfs import NfsCollector diff --git a/src/collectors/nfsd/test/testnfsd.py b/src/collectors/nfsd/test/testnfsd.py index a7f4467c6..e4127335d 100644 --- a/src/collectors/nfsd/test/testnfsd.py +++ b/src/collectors/nfsd/test/testnfsd.py @@ -7,11 +7,7 @@ from test import unittest from test import Mock from test import patch - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from test import StringIO from diamond.collector import Collector from nfsd import NfsdCollector diff --git a/src/collectors/proc/test/testproc.py b/src/collectors/proc/test/testproc.py index 2592f47ae..2a04d53eb 100644 --- a/src/collectors/proc/test/testproc.py +++ b/src/collectors/proc/test/testproc.py @@ -7,11 +7,7 @@ from test import unittest from test import Mock from test import patch - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from test import StringIO from diamond.collector import Collector from proc import ProcessStatCollector diff --git a/src/collectors/slabinfo/test/testslabinfo.py b/src/collectors/slabinfo/test/testslabinfo.py index 58a99b1e1..311d6fabd 100644 --- a/src/collectors/slabinfo/test/testslabinfo.py +++ b/src/collectors/slabinfo/test/testslabinfo.py @@ -7,11 +7,7 @@ from test import unittest from test import Mock from test import patch - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from test import StringIO from diamond.collector import Collector from slabinfo import SlabInfoCollector diff --git a/src/collectors/tcp/test/testtcp.py b/src/collectors/tcp/test/testtcp.py index 658ea0702..fcaae8f48 100644 --- a/src/collectors/tcp/test/testtcp.py +++ b/src/collectors/tcp/test/testtcp.py @@ -7,11 +7,7 @@ from test import unittest from test import Mock from test import patch - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from test import StringIO from tcp import TCPCollector diff --git a/src/collectors/udp/test/testudp.py b/src/collectors/udp/test/testudp.py index ae9dea063..c0f7233af 100644 --- a/src/collectors/udp/test/testudp.py +++ b/src/collectors/udp/test/testudp.py @@ -7,11 +7,7 @@ from test import unittest from test import Mock from test import patch - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from test import StringIO from diamond.collector import Collector from udp import UDPCollector diff --git a/src/collectors/uptime/test/test_uptime.py b/src/collectors/uptime/test/test_uptime.py index 05e1a6353..3ea5593f1 100644 --- a/src/collectors/uptime/test/test_uptime.py +++ b/src/collectors/uptime/test/test_uptime.py @@ -6,11 +6,7 @@ from test import get_collector_config from test import Mock from test import patch - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from test import StringIO from diamond.collector import Collector from uptime import UptimeCollector diff --git a/src/collectors/vmstat/test/testvmstat.py b/src/collectors/vmstat/test/testvmstat.py index 1478dce15..dce562277 100644 --- a/src/collectors/vmstat/test/testvmstat.py +++ b/src/collectors/vmstat/test/testvmstat.py @@ -7,11 +7,7 @@ from test import unittest from test import Mock from test import patch - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from test import StringIO from diamond.collector import Collector from vmstat import VMStatCollector From 3618170e922f112e47554098b6544dc5971af23f Mon Sep 17 00:00:00 2001 From: Hannu Valtonen Date: Mon, 24 Aug 2015 15:22:50 +0300 Subject: [PATCH 14/33] py3k: Change tests mock patching to use a variable from test base file --- src/collectors/cpu/test/testcpu.py | 13 ++++++----- .../cpuacct_cgroup/test/testcpuacct_cgroup.py | 3 ++- .../diskspace/test/testdiskspace.py | 5 ++-- .../diskusage/test/testdiskusage.py | 23 ++++++++++--------- src/collectors/filestat/test/testfilestat.py | 3 ++- .../interrupt/test/testinterrupt.py | 3 ++- src/collectors/interrupt/test/testsoft.py | 7 +++--- src/collectors/ip/test/testip.py | 5 ++-- src/collectors/kvm/test/testkvm.py | 3 ++- src/collectors/loadavg/test/testloadavg.py | 3 ++- src/collectors/memory/test/testmemory.py | 3 ++- .../memory_cgroup/test/testmemory_cgroup.py | 3 ++- .../memory_docker/test/testmemory_docker.py | 5 ++-- .../mountstats/test/testmountstats.py | 3 ++- src/collectors/network/test/testnetwork.py | 3 ++- src/collectors/nfs/test/testnfs.py | 3 ++- src/collectors/nfsd/test/testnfsd.py | 3 ++- .../test/testopenstackswiftrecon.py | 4 ++-- src/collectors/proc/test/testproc.py | 3 ++- src/collectors/slabinfo/test/testslabinfo.py | 3 ++- src/collectors/sockstat/test/testsockstat.py | 5 ++-- src/collectors/tcp/test/testtcp.py | 5 ++-- src/collectors/udp/test/testudp.py | 3 ++- src/collectors/uptime/test/test_uptime.py | 3 ++- src/collectors/vmstat/test/testvmstat.py | 3 ++- test.py | 6 +++++ 26 files changed, 78 insertions(+), 48 deletions(-) diff --git a/src/collectors/cpu/test/testcpu.py b/src/collectors/cpu/test/testcpu.py index 89cfeef74..7ca83ce3d 100644 --- a/src/collectors/cpu/test/testcpu.py +++ b/src/collectors/cpu/test/testcpu.py @@ -1,4 +1,4 @@ -i#!/usr/bin/python +#!/usr/bin/python # coding=utf-8 ########################################################################## @@ -8,6 +8,7 @@ from test import Mock from test import patch from test import StringIO +from test import BUILTIN_OPEN from diamond.collector import Collector from cpu import CPUCollector @@ -28,7 +29,7 @@ def setUp(self): def test_import(self): self.assertTrue(CPUCollector) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch('os.access', Mock(return_value=True)) @patch.object(Collector, 'publish') def test_should_open_proc_stat(self, publish_mock, open_mock): @@ -39,7 +40,7 @@ def test_should_open_proc_stat(self, publish_mock, open_mock): @patch.object(Collector, 'publish') def test_should_work_with_synthetic_data(self, publish_mock): - patch_open = patch('__builtin__.open', Mock(return_value=StringIO( + patch_open = patch(BUILTIN_OPEN, Mock(return_value=StringIO( 'cpu 100 200 300 400 500 0 0 0 0 0'))) patch_open.start() @@ -48,7 +49,7 @@ def test_should_work_with_synthetic_data(self, publish_mock): self.assertPublishedMany(publish_mock, {}) - patch_open = patch('__builtin__.open', Mock(return_value=StringIO( + patch_open = patch(BUILTIN_OPEN, Mock(return_value=StringIO( 'cpu 110 220 330 440 550 0 0 0 0 0'))) patch_open.start() @@ -200,7 +201,7 @@ def input_dict_to_proc_string(self, cpu_id, dict_): @patch.object(Collector, 'publish') def test_should_work_proc_stat(self, publish_mock): - patch_open = patch('__builtin__.open', Mock(return_value=StringIO( + patch_open = patch(BUILTIN_OPEN, Mock(return_value=StringIO( "\n".join([self.input_dict_to_proc_string('', self.input_base), self.input_dict_to_proc_string('0', self.input_base), self.input_dict_to_proc_string('1', self.input_base), @@ -213,7 +214,7 @@ def test_should_work_proc_stat(self, publish_mock): self.assertPublishedMany(publish_mock, {}) - patch_open = patch('__builtin__.open', Mock(return_value=StringIO( + patch_open = patch(BUILTIN_OPEN, Mock(return_value=StringIO( "\n".join([self.input_dict_to_proc_string('', self.input_next), self.input_dict_to_proc_string('0', self.input_next), self.input_dict_to_proc_string('1', self.input_next), diff --git a/src/collectors/cpuacct_cgroup/test/testcpuacct_cgroup.py b/src/collectors/cpuacct_cgroup/test/testcpuacct_cgroup.py index cece2bb7f..56a3fb047 100644 --- a/src/collectors/cpuacct_cgroup/test/testcpuacct_cgroup.py +++ b/src/collectors/cpuacct_cgroup/test/testcpuacct_cgroup.py @@ -7,6 +7,7 @@ from test import unittest from test import patch from test import StringIO +from test import BUILTIN_OPEN from diamond.collector import Collector from cpuacct_cgroup import CpuAcctCgroupCollector @@ -24,7 +25,7 @@ def setUp(self): def test_import(self): self.assertTrue(CpuAcctCgroupCollector) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch.object(Collector, 'publish') def test_should_open_all_cpuacct_stat(self, publish_mock, open_mock): return diff --git a/src/collectors/diskspace/test/testdiskspace.py b/src/collectors/diskspace/test/testdiskspace.py index 8968abc0d..5fbf76537 100644 --- a/src/collectors/diskspace/test/testdiskspace.py +++ b/src/collectors/diskspace/test/testdiskspace.py @@ -8,6 +8,7 @@ from test import run_only from test import Mock from test import patch +from test import BUILTIN_OPEN from diamond.collector import Collector from diskspace import DiskSpaceCollector @@ -34,7 +35,7 @@ def test_import(self): def run_collection(self, statvfs_mock, os_major, os_minor): os_stat_mock = patch('os.stat') os_path_isdir_mock = patch('os.path.isdir', Mock(return_value=False)) - open_mock = patch('__builtin__.open', + open_mock = patch(BUILTIN_OPEN, Mock(return_value=self.getFixture('proc_mounts'))) os_statvfs_mock = patch('os.statvfs', Mock(return_value=statvfs_mock)) @@ -54,7 +55,7 @@ def test_get_file_systems(self): os_stat_mock = patch('os.stat') os_realpath_mock = patch('os.path.realpath') - open_mock = patch('__builtin__.open', + open_mock = patch(BUILTIN_OPEN, Mock(return_value=self.getFixture('proc_mounts'))) stat_mock = os_stat_mock.start() diff --git a/src/collectors/diskusage/test/testdiskusage.py b/src/collectors/diskusage/test/testdiskusage.py index 626b1941d..65a268dc7 100644 --- a/src/collectors/diskusage/test/testdiskusage.py +++ b/src/collectors/diskusage/test/testdiskusage.py @@ -7,6 +7,7 @@ from test import unittest from test import Mock from test import patch +from test import BUILTIN_OPEN from diamond.collector import Collector from diskusage import DiskUsageCollector @@ -35,7 +36,7 @@ def test_import(self): def test_get_disk_statistics(self): patch_open = patch( - '__builtin__.open', + BUILTIN_OPEN, Mock(return_value=self.getFixture('diskstats'))) open_mock = patch_open.start() @@ -56,7 +57,7 @@ def test_get_disk_statistics(self): def test_should_work_with_real_data(self, publish_mock): patch_open = patch( - '__builtin__.open', + BUILTIN_OPEN, Mock( return_value=self.getFixture('proc_diskstats_1'))) patch_time = patch('time.time', Mock(return_value=10)) @@ -70,7 +71,7 @@ def test_should_work_with_real_data(self, publish_mock): self.assertPublishedMany(publish_mock, {}) patch_open = patch( - '__builtin__.open', + BUILTIN_OPEN, Mock( return_value=self.getFixture('proc_diskstats_2'))) patch_time = patch('time.time', Mock(return_value=20)) @@ -92,7 +93,7 @@ def test_should_work_with_real_data(self, publish_mock): @patch.object(Collector, 'publish') def test_verify_supporting_vda_and_xvdb(self, publish_mock): patch_open = patch( - '__builtin__.open', + BUILTIN_OPEN, Mock( return_value=self.getFixture( 'proc_diskstats_1_vda_xvdb'))) @@ -107,7 +108,7 @@ def test_verify_supporting_vda_and_xvdb(self, publish_mock): self.assertPublishedMany(publish_mock, {}) patch_open = patch( - '__builtin__.open', + BUILTIN_OPEN, Mock( return_value=self.getFixture( 'proc_diskstats_2_vda_xvdb'))) @@ -128,7 +129,7 @@ def test_verify_supporting_vda_and_xvdb(self, publish_mock): @patch.object(Collector, 'publish') def test_verify_supporting_md_dm(self, publish_mock): patch_open = patch( - '__builtin__.open', + BUILTIN_OPEN, Mock( return_value=self.getFixture( 'proc_diskstats_1_md_dm'))) @@ -143,7 +144,7 @@ def test_verify_supporting_md_dm(self, publish_mock): self.assertPublishedMany(publish_mock, {}) patch_open = patch( - '__builtin__.open', + BUILTIN_OPEN, Mock( return_value=self.getFixture( 'proc_diskstats_2_md_dm'))) @@ -163,7 +164,7 @@ def test_verify_supporting_md_dm(self, publish_mock): @patch.object(Collector, 'publish') def test_verify_supporting_disk(self, publish_mock): patch_open = patch( - '__builtin__.open', + BUILTIN_OPEN, Mock( return_value=self.getFixture( 'proc_diskstats_1_disk'))) @@ -178,7 +179,7 @@ def test_verify_supporting_disk(self, publish_mock): self.assertPublishedMany(publish_mock, {}) patch_open = patch( - '__builtin__.open', + BUILTIN_OPEN, Mock( return_value=self.getFixture( 'proc_diskstats_2_disk'))) @@ -197,7 +198,7 @@ def test_verify_supporting_disk(self, publish_mock): @patch.object(Collector, 'publish') def test_service_Time(self, publish_mock): patch_open = patch( - '__builtin__.open', + BUILTIN_OPEN, Mock( return_value=self.getFixture( 'proc_diskstats_1_service_time'))) @@ -212,7 +213,7 @@ def test_service_Time(self, publish_mock): self.assertPublishedMany(publish_mock, {}) patch_open = patch( - '__builtin__.open', + BUILTIN_OPEN, Mock( return_value=self.getFixture( 'proc_diskstats_2_service_time'))) diff --git a/src/collectors/filestat/test/testfilestat.py b/src/collectors/filestat/test/testfilestat.py index 7413e35b6..a749b00bc 100644 --- a/src/collectors/filestat/test/testfilestat.py +++ b/src/collectors/filestat/test/testfilestat.py @@ -8,6 +8,7 @@ from test import Mock from test import patch from test import StringIO +from test import BUILTIN_OPEN from diamond.collector import Collector from filestat import FilestatCollector @@ -27,7 +28,7 @@ def setUp(self): def test_import(self): self.assertTrue(FilestatCollector) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch('os.access', Mock(return_value=True)) @patch.object(Collector, 'publish') def test_should_open_proc_sys_fs_file_nr(self, publish_mock, open_mock): diff --git a/src/collectors/interrupt/test/testinterrupt.py b/src/collectors/interrupt/test/testinterrupt.py index 6896214e7..178be7a6c 100644 --- a/src/collectors/interrupt/test/testinterrupt.py +++ b/src/collectors/interrupt/test/testinterrupt.py @@ -8,6 +8,7 @@ from test import Mock from test import patch from test import StringIO +from test import BUILTIN_OPEN from diamond.collector import Collector from interrupt import InterruptCollector @@ -27,7 +28,7 @@ def setUp(self): def test_import(self): self.assertTrue(InterruptCollector) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch('os.access', Mock(return_value=True)) @patch.object(Collector, 'publish') def test_should_open_proc_stat(self, publish_mock, open_mock): diff --git a/src/collectors/interrupt/test/testsoft.py b/src/collectors/interrupt/test/testsoft.py index e36ad93ed..e04108094 100644 --- a/src/collectors/interrupt/test/testsoft.py +++ b/src/collectors/interrupt/test/testsoft.py @@ -8,6 +8,7 @@ from test import Mock from test import patch from test import StringIO +from test import BUILTIN_OPEN from diamond.collector import Collector from soft import SoftInterruptCollector @@ -27,7 +28,7 @@ def setUp(self): def test_import(self): self.assertTrue(SoftInterruptCollector) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch('os.access', Mock(return_value=True)) @patch.object(Collector, 'publish') def test_should_open_proc_stat(self, publish_mock, open_mock): @@ -37,7 +38,7 @@ def test_should_open_proc_stat(self, publish_mock, open_mock): @patch.object(Collector, 'publish') def test_should_work_with_synthetic_data(self, publish_mock): - patch_open = patch('__builtin__.open', Mock(return_value=StringIO( + patch_open = patch(BUILTIN_OPEN, Mock(return_value=StringIO( 'softirq 0 0 0 0 0 0 0 0 0 0 0' ))) @@ -47,7 +48,7 @@ def test_should_work_with_synthetic_data(self, publish_mock): self.assertPublishedMany(publish_mock, {}) - patch_open = patch('__builtin__.open', Mock(return_value=StringIO( + patch_open = patch(BUILTIN_OPEN, Mock(return_value=StringIO( 'softirq 55 1 2 3 4 5 6 7 8 9 10' ))) diff --git a/src/collectors/ip/test/testip.py b/src/collectors/ip/test/testip.py index f3d7fc191..748da6548 100644 --- a/src/collectors/ip/test/testip.py +++ b/src/collectors/ip/test/testip.py @@ -8,6 +8,7 @@ from test import Mock from test import patch from test import StringIO +from test import BUILTIN_OPEN from ip import IPCollector @@ -29,7 +30,7 @@ def test_import(self): self.assertTrue(IPCollector) @patch('os.access', Mock(return_value=True)) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch('diamond.collector.Collector.publish') def test_should_open_proc_net_snmp(self, publish_mock, open_mock): IPCollector.PROC = ['/proc/net/snmp'] @@ -38,7 +39,7 @@ def test_should_open_proc_net_snmp(self, publish_mock, open_mock): open_mock.assert_called_once_with('/proc/net/snmp') @patch('os.access', Mock(return_value=True)) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch('diamond.collector.Collector.publish') def test_should_work_with_synthetic_data(self, publish_mock, open_mock): IPCollector.PROC = ['/proc/net/snmp'] diff --git a/src/collectors/kvm/test/testkvm.py b/src/collectors/kvm/test/testkvm.py index f100333fd..1bbcf8115 100644 --- a/src/collectors/kvm/test/testkvm.py +++ b/src/collectors/kvm/test/testkvm.py @@ -10,6 +10,7 @@ from test import Mock from test import patch from test import StringIO +from test import BUILTIN_OPEN from diamond.collector import Collector from kvm import KVMCollector @@ -33,7 +34,7 @@ def test_import(self): @patch('os.access', Mock(return_value=True)) @patch.object(Collector, 'publish') def test_should_work_with_synthetic_data(self, publish_mock): - patch_open = patch('__builtin__.open', Mock(return_value=StringIO( + patch_open = patch(BUILTIN_OPEN, Mock(return_value=StringIO( '0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0' + '\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n' ))) diff --git a/src/collectors/loadavg/test/testloadavg.py b/src/collectors/loadavg/test/testloadavg.py index 1f6963552..1f5dd4c13 100644 --- a/src/collectors/loadavg/test/testloadavg.py +++ b/src/collectors/loadavg/test/testloadavg.py @@ -9,6 +9,7 @@ from test import Mock from test import patch from test import StringIO +from test import BUILTIN_OPEN from diamond.collector import Collector from loadavg import LoadAverageCollector @@ -28,7 +29,7 @@ def setUp(self): def test_import(self): self.assertTrue(LoadAverageCollector) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch('os.access', Mock(return_value=True)) @patch.object(Collector, 'publish') def test_should_open_proc_loadavg(self, publish_mock, open_mock): diff --git a/src/collectors/memory/test/testmemory.py b/src/collectors/memory/test/testmemory.py index 12a13f1d8..5822519eb 100644 --- a/src/collectors/memory/test/testmemory.py +++ b/src/collectors/memory/test/testmemory.py @@ -8,6 +8,7 @@ from test import Mock from test import patch from test import StringIO +from test import BUILTIN_OPEN from diamond.collector import Collector from memory import MemoryCollector @@ -28,7 +29,7 @@ def setUp(self): def test_import(self): self.assertTrue(MemoryCollector) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch('os.access', Mock(return_value=True)) @patch.object(Collector, 'publish') def test_should_open_proc_meminfo(self, publish_mock, open_mock): diff --git a/src/collectors/memory_cgroup/test/testmemory_cgroup.py b/src/collectors/memory_cgroup/test/testmemory_cgroup.py index d3444d034..8510a0ce0 100644 --- a/src/collectors/memory_cgroup/test/testmemory_cgroup.py +++ b/src/collectors/memory_cgroup/test/testmemory_cgroup.py @@ -8,6 +8,7 @@ from test import Mock from test import patch from test import StringIO +from test import BUILTIN_OPEN from diamond.collector import Collector from memory_cgroup import MemoryCgroupCollector @@ -24,7 +25,7 @@ class TestMemoryCgroupCollector(CollectorTestCase): def test_import(self): self.assertTrue(MemoryCgroupCollector) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch('os.walk', Mock(return_value=iter(fixtures))) @patch.object(Collector, 'publish') def test_should_open_all_memory_stat(self, publish_mock, open_mock): diff --git a/src/collectors/memory_docker/test/testmemory_docker.py b/src/collectors/memory_docker/test/testmemory_docker.py index fb4f07ea0..949f176ea 100644 --- a/src/collectors/memory_docker/test/testmemory_docker.py +++ b/src/collectors/memory_docker/test/testmemory_docker.py @@ -9,6 +9,7 @@ from test import Mock from test import patch from test import StringIO +from test import BUILTIN_OPEN try: from docker import Client @@ -55,7 +56,7 @@ def test_import(self): self.assertTrue(MemoryDockerCollector) @run_only_if_docker_client_is_available - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch.object(Client, 'containers', Mock(return_value=[])) @patch.object(Collector, 'publish') def test_should_open_all_cpuacct_stat(self, publish_mock, open_mock): @@ -67,7 +68,7 @@ def test_should_open_all_cpuacct_stat(self, publish_mock, open_mock): open_mock.assert_any_call(fixtures_path + 'memory.stat') @run_only_if_docker_client_is_available - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch.object(Client, 'containers') @patch.object(Collector, 'publish') def test_should_get_containers(self, publish_mock, containers_mock, diff --git a/src/collectors/mountstats/test/testmountstats.py b/src/collectors/mountstats/test/testmountstats.py index 767e90c9a..89fc446a6 100644 --- a/src/collectors/mountstats/test/testmountstats.py +++ b/src/collectors/mountstats/test/testmountstats.py @@ -7,6 +7,7 @@ from test import MagicMock from test import patch from test import Mock +from test import BUILTIN_OPEN from diamond.collector import Collector from mountstats import MountStatsCollector @@ -25,7 +26,7 @@ def setUp(self): def test_import(self): self.assertTrue(MountStatsCollector) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch('os.access', Mock(return_value=True)) @patch.object(Collector, 'publish') def test_should_open_mountstats(self, publish_mock, open_mock): diff --git a/src/collectors/network/test/testnetwork.py b/src/collectors/network/test/testnetwork.py index c79ff8e2c..ffb125675 100644 --- a/src/collectors/network/test/testnetwork.py +++ b/src/collectors/network/test/testnetwork.py @@ -8,6 +8,7 @@ from test import Mock from test import patch from test import StringIO +from test import BUILTIN_OPEN from diamond.collector import Collector from network import NetworkCollector @@ -29,7 +30,7 @@ def setUp(self): def test_import(self): self.assertTrue(NetworkCollector) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch('os.access', Mock(return_value=True)) @patch.object(Collector, 'publish') def test_should_open_proc_net_dev(self, publish_mock, open_mock): diff --git a/src/collectors/nfs/test/testnfs.py b/src/collectors/nfs/test/testnfs.py index 4eb6f43eb..62e7f2ffa 100644 --- a/src/collectors/nfs/test/testnfs.py +++ b/src/collectors/nfs/test/testnfs.py @@ -8,6 +8,7 @@ from test import Mock from test import patch from test import StringIO +from test import BUILTIN_OPEN from diamond.collector import Collector from nfs import NfsCollector @@ -27,7 +28,7 @@ def setUp(self): def test_import(self): self.assertTrue(NfsCollector) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch('os.access', Mock(return_value=True)) @patch.object(Collector, 'publish') def test_should_open_proc_stat(self, publish_mock, open_mock): diff --git a/src/collectors/nfsd/test/testnfsd.py b/src/collectors/nfsd/test/testnfsd.py index e4127335d..36fd8bc64 100644 --- a/src/collectors/nfsd/test/testnfsd.py +++ b/src/collectors/nfsd/test/testnfsd.py @@ -8,6 +8,7 @@ from test import Mock from test import patch from test import StringIO +from test import BUILTIN_OPEN from diamond.collector import Collector from nfsd import NfsdCollector @@ -27,7 +28,7 @@ def setUp(self): def test_import(self): self.assertTrue(NfsdCollector) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch('os.access', Mock(return_value=True)) @patch.object(Collector, 'publish') def test_should_open_proc_stat(self, publish_mock, open_mock): diff --git a/src/collectors/openstackswiftrecon/test/testopenstackswiftrecon.py b/src/collectors/openstackswiftrecon/test/testopenstackswiftrecon.py index 44970f7bd..3bcd67a8e 100644 --- a/src/collectors/openstackswiftrecon/test/testopenstackswiftrecon.py +++ b/src/collectors/openstackswiftrecon/test/testopenstackswiftrecon.py @@ -5,7 +5,7 @@ from test import CollectorTestCase from test import get_collector_config from test import unittest - +from test import BUILTIN_OPEN from test import patch, Mock from diamond.collector import Collector @@ -29,7 +29,7 @@ def setUp(self, allowed_names=None): def test_import(self): self.assertTrue(OpenstackSwiftReconCollector) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch('os.access', Mock(return_value=False)) @patch.object(Collector, 'publish') def test_recon_no_access(self, publish_mock, open_mock): diff --git a/src/collectors/proc/test/testproc.py b/src/collectors/proc/test/testproc.py index 2a04d53eb..dc91e3bef 100644 --- a/src/collectors/proc/test/testproc.py +++ b/src/collectors/proc/test/testproc.py @@ -8,6 +8,7 @@ from test import Mock from test import patch from test import StringIO +from test import BUILTIN_OPEN from diamond.collector import Collector from proc import ProcessStatCollector @@ -27,7 +28,7 @@ def setUp(self): def test_import(self): self.assertTrue(ProcessStatCollector) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch('os.access', Mock(return_value=True)) @patch.object(Collector, 'publish') def test_should_open_proc_stat(self, publish_mock, open_mock): diff --git a/src/collectors/slabinfo/test/testslabinfo.py b/src/collectors/slabinfo/test/testslabinfo.py index 311d6fabd..558e981cf 100644 --- a/src/collectors/slabinfo/test/testslabinfo.py +++ b/src/collectors/slabinfo/test/testslabinfo.py @@ -8,6 +8,7 @@ from test import Mock from test import patch from test import StringIO +from test import BUILTIN_OPEN from diamond.collector import Collector from slabinfo import SlabInfoCollector @@ -27,7 +28,7 @@ def setUp(self): def test_import(self): self.assertTrue(SlabInfoCollector) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch('os.access', Mock(return_value=True)) @patch.object(Collector, 'publish') def test_should_open_proc_stat(self, publish_mock, open_mock): diff --git a/src/collectors/sockstat/test/testsockstat.py b/src/collectors/sockstat/test/testsockstat.py index 585a35369..132e3e96d 100644 --- a/src/collectors/sockstat/test/testsockstat.py +++ b/src/collectors/sockstat/test/testsockstat.py @@ -8,8 +8,9 @@ from test import Mock from test import patch from test import call -from collections import Iterator +from test import BUILTIN_OPEN +from collections import Iterator from diamond.collector import Collector from sockstat import SockstatCollector @@ -28,7 +29,7 @@ def setUp(self): def test_import(self): self.assertTrue(SockstatCollector) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch('os.access', Mock(return_value=True)) @patch.object(Collector, 'publish') def test_should_open_proc_net_sockstat(self, publish_mock, open_mock): diff --git a/src/collectors/tcp/test/testtcp.py b/src/collectors/tcp/test/testtcp.py index fcaae8f48..948e8b564 100644 --- a/src/collectors/tcp/test/testtcp.py +++ b/src/collectors/tcp/test/testtcp.py @@ -8,6 +8,7 @@ from test import Mock from test import patch from test import StringIO +from test import BUILTIN_OPEN from tcp import TCPCollector @@ -30,7 +31,7 @@ def test_import(self): self.assertTrue(TCPCollector) @patch('os.access', Mock(return_value=True)) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch('diamond.collector.Collector.publish') def test_should_open_proc_net_netstat(self, publish_mock, open_mock): TCPCollector.PROC = ['/proc/net/netstat'] @@ -39,7 +40,7 @@ def test_should_open_proc_net_netstat(self, publish_mock, open_mock): open_mock.assert_called_once_with('/proc/net/netstat') @patch('os.access', Mock(return_value=True)) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch('diamond.collector.Collector.publish') def test_should_work_with_synthetic_data(self, publish_mock, open_mock): TCPCollector.PROC = ['/proc/net/netstat'] diff --git a/src/collectors/udp/test/testudp.py b/src/collectors/udp/test/testudp.py index c0f7233af..f92480c62 100644 --- a/src/collectors/udp/test/testudp.py +++ b/src/collectors/udp/test/testudp.py @@ -8,6 +8,7 @@ from test import Mock from test import patch from test import StringIO +from test import BUILTIN_OPEN from diamond.collector import Collector from udp import UDPCollector @@ -30,7 +31,7 @@ def test_import(self): self.assertTrue(UDPCollector) @patch('os.access', Mock(return_value=True)) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch.object(Collector, 'publish') def test_should_open_proc_net_snmp(self, publish_mock, open_mock): UDPCollector.PROC = ['/proc/net/snmp'] diff --git a/src/collectors/uptime/test/test_uptime.py b/src/collectors/uptime/test/test_uptime.py index 3ea5593f1..4f03f9ba7 100644 --- a/src/collectors/uptime/test/test_uptime.py +++ b/src/collectors/uptime/test/test_uptime.py @@ -7,6 +7,7 @@ from test import Mock from test import patch from test import StringIO +from test import BUILTIN_OPEN from diamond.collector import Collector from uptime import UptimeCollector @@ -29,7 +30,7 @@ def setUp(self, config=None): def test_import(self): self.assertTrue(UptimeCollector) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch('os.path.exists', Mock(return_value=True)) @patch.object(Collector, 'publish') def test_should_open_proc_uptime(self, publish_mock, open_mock): diff --git a/src/collectors/vmstat/test/testvmstat.py b/src/collectors/vmstat/test/testvmstat.py index dce562277..4d6db4cf3 100644 --- a/src/collectors/vmstat/test/testvmstat.py +++ b/src/collectors/vmstat/test/testvmstat.py @@ -2,6 +2,7 @@ # coding=utf-8 ############################################################################### +from test import BUILTIN_OPEN from test import CollectorTestCase from test import get_collector_config from test import unittest @@ -27,7 +28,7 @@ def setUp(self): def test_import(self): self.assertTrue(VMStatCollector) - @patch('__builtin__.open') + @patch(BUILTIN_OPEN) @patch('os.access', Mock(return_value=True)) @patch.object(Collector, 'publish') def test_should_open_proc_vmstat(self, publish_mock, open_mock): diff --git a/test.py b/test.py index aee1e2ef1..cc18faf95 100755 --- a/test.py +++ b/test.py @@ -39,6 +39,12 @@ except ImportError: from unittest.mock import call, MagicMock, Mock, patch +try: # py3k way + import builtins + BUILTIN_OPEN = "builtins.open" +except ImportError: # py2.x way + BUILTIN_OPEN = "__builtin__.open" + sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__)))) sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), 'src'))) From bee85ad4386837c1f981cf3cc2887cc96fdd8333 Mon Sep 17 00:00:00 2001 From: Hannu Valtonen Date: Mon, 24 Aug 2015 17:42:29 +0300 Subject: [PATCH 15/33] py3k: import urllib from a common pycompat file --- src/collectors/aurora/aurora.py | 5 ++--- src/collectors/aurora/test/testaurora.py | 6 +++--- src/collectors/bind/bind.py | 4 ++-- src/collectors/bind/test/testbind.py | 3 ++- src/collectors/celerymon/celerymon.py | 4 ++-- src/collectors/dropwizard/dropwizard.py | 11 +++++------ .../dropwizard/test/testdropwizard.py | 5 +++-- src/collectors/dseopscenter/dseopscenter.py | 7 +++---- .../dseopscenter/test/testdseopscenter.py | 5 +++-- src/collectors/elasticsearch/elasticsearch.py | 13 ++++--------- .../elasticsearch/test/testelasticsearch.py | 11 ++++++----- src/collectors/elb/test/testelb.py | 9 ++++----- src/collectors/endecadgraph/endecadgraph.py | 9 ++++++--- .../endecadgraph/test/testendecadgraph.py | 3 ++- src/collectors/flume/flume.py | 8 +++----- src/collectors/flume/test/testflume.py | 7 ++++--- src/collectors/haproxy/haproxy.py | 11 +++++------ src/collectors/haproxy/test/testhaproxy.py | 5 +++-- src/collectors/http/http.py | 13 +++++-------- src/collectors/http/test/testhttp.py | 3 ++- src/collectors/httpd/httpd.py | 18 +++++++++--------- src/collectors/httpd/test/testhttpd.py | 12 +++++++----- src/collectors/httpjson/httpjson.py | 15 +++++---------- src/collectors/httpjson/test/testhttpjson.py | 3 ++- src/collectors/jolokia/jolokia.py | 14 +++++++------- src/collectors/jolokia/test/testjolokia.py | 9 +++++---- src/collectors/kafkastat/kafkastat.py | 12 ++++-------- src/collectors/kafkastat/test/testkafka.py | 14 ++++++-------- src/collectors/mesos/mesos.py | 6 +++--- src/collectors/mesos/test/testmesos.py | 5 +++-- src/collectors/monit/monit.py | 14 +++++++------- src/collectors/monit/test/testmonit.py | 5 +++-- src/collectors/nginx/nginx.py | 11 ++++------- src/collectors/nginx/test/testnginx.py | 5 +++-- src/collectors/openvpn/openvpn.py | 9 ++------- src/collectors/phpfpm/phpfpm.py | 7 +++---- src/collectors/phpfpm/test/testphpfpm.py | 5 +++-- .../puppetdashboard/puppetdashboard.py | 10 ++-------- .../test/testpuppetdashboard.py | 5 +++-- src/collectors/puppetdb/puppetdb.py | 5 ++--- src/collectors/rabbitmq/rabbitmq.py | 9 ++++----- src/collectors/resqueweb/resqueweb.py | 9 ++------- .../resqueweb/test/testresqueweb.py | 5 +++-- src/collectors/sidekiqweb/sidekiqweb.py | 5 ++--- .../sidekiqweb/test/testsidekiqweb.py | 5 +++-- src/collectors/solr/solr.py | 5 ++--- src/collectors/solr/test/testsolr.py | 7 +++++-- .../websitemonitor/test/testwebsitemonitor.py | 3 ++- .../websitemonitor/websitemonitor.py | 17 +++++------------ src/diamond/handler/httpHandler.py | 14 +++++++++++--- src/diamond/handler/logentries_diamond.py | 11 +++++------ src/diamond/handler/signalfx.py | 19 +++++++------------ src/diamond/handler/test/teststatsdhandler.py | 2 +- test.py | 4 ++-- 54 files changed, 206 insertions(+), 235 deletions(-) diff --git a/src/collectors/aurora/aurora.py b/src/collectors/aurora/aurora.py index 809c6f335..ed01e196a 100644 --- a/src/collectors/aurora/aurora.py +++ b/src/collectors/aurora/aurora.py @@ -1,6 +1,5 @@ import diamond.collector - -import urllib2 +import diamond.pycompat class AuroraCollector(diamond.collector.Collector): @@ -32,7 +31,7 @@ def collect(self): self.config['host'], self.config['port']) - response = urllib2.urlopen(url) + response = diamond.pycompat.urlopen(url) for line in response.readlines(): properties = line.split() diff --git a/src/collectors/aurora/test/testaurora.py b/src/collectors/aurora/test/testaurora.py index b0fdf29c0..c867ef4ef 100644 --- a/src/collectors/aurora/test/testaurora.py +++ b/src/collectors/aurora/test/testaurora.py @@ -9,7 +9,7 @@ from test import patch from diamond.collector import Collector - +from diamond.pycompat import URLOPEN from aurora import AuroraCollector ############################################################################### @@ -31,7 +31,7 @@ def se(url): if url == 'http://localhost:8081/vars': return self.getFixture('metrics') - patch_urlopen = patch('urllib2.urlopen', Mock(side_effect=se)) + patch_urlopen = patch(URLOPEN, Mock(side_effect=se)) patch_urlopen.start() self.collector.collect() @@ -46,7 +46,7 @@ def se(url): @patch.object(Collector, 'publish') def test_should_fail_gracefully(self, publish_mock): - patch_urlopen = patch('urllib2.urlopen', Mock( + patch_urlopen = patch(URLOPEN, Mock( return_value=self.getFixture('metrics_blank'))) patch_urlopen.start() diff --git a/src/collectors/bind/bind.py b/src/collectors/bind/bind.py index 59a0e0857..c390e5312 100644 --- a/src/collectors/bind/bind.py +++ b/src/collectors/bind/bind.py @@ -11,8 +11,8 @@ """ import diamond.collector +import diamond.pycompat import sys -import urllib2 if sys.version_info >= (2, 5): import xml.etree.cElementTree as ElementTree @@ -74,7 +74,7 @@ def clean_counter(self, name, value): def collect(self): try: - req = urllib2.urlopen('http://%s:%d/' % ( + req = diamond.pycompat.urlopen('http://%s:%d/' % ( self.config['host'], int(self.config['port']))) except Exception as e: self.log.error('Couldnt connect to bind: %s', e) diff --git a/src/collectors/bind/test/testbind.py b/src/collectors/bind/test/testbind.py index e2d12e6cd..64a922ec9 100644 --- a/src/collectors/bind/test/testbind.py +++ b/src/collectors/bind/test/testbind.py @@ -9,6 +9,7 @@ from test import patch from diamond.collector import Collector +from diamond.pycompat import URLOPEN from bind import BindCollector ########################################################################## @@ -28,7 +29,7 @@ def test_import(self): @patch.object(Collector, 'publish') def test_should_work_with_real_data(self, publish_mock): - patch_urlopen = patch('urllib2.urlopen', Mock( + patch_urlopen = patch(URLOPEN, Mock( return_value=self.getFixture('bind.xml'))) patch_urlopen.start() diff --git a/src/collectors/celerymon/celerymon.py b/src/collectors/celerymon/celerymon.py index 05ba0030d..d8f325832 100644 --- a/src/collectors/celerymon/celerymon.py +++ b/src/collectors/celerymon/celerymon.py @@ -18,7 +18,7 @@ """ import diamond.collector -import urllib2 +import diamond.pycompat import time try: @@ -68,7 +68,7 @@ def collect(self): celerymon_url = "http://%s:%s/api/task/?since=%i" % ( host, port, self.LastCollectTime) - response = urllib2.urlopen(celerymon_url) + response = diamond.pycompat.urlopen(celerymon_url) body = response.read() celery_data = json.loads(body) diff --git a/src/collectors/dropwizard/dropwizard.py b/src/collectors/dropwizard/dropwizard.py index 2af875287..c07a9a4d4 100644 --- a/src/collectors/dropwizard/dropwizard.py +++ b/src/collectors/dropwizard/dropwizard.py @@ -4,15 +4,14 @@ Collect [dropwizard](http://dropwizard.codahale.com/) stats for the local node """ - -import urllib2 - try: import json except ImportError: import simplejson as json - import diamond.collector +import diamond.pycompat +from diamond.pycompat import HTTPError + class DropwizardCollector(diamond.collector.Collector): @@ -45,8 +44,8 @@ def collect(self): url = 'http://%s:%i/metrics' % ( self.config['host'], int(self.config['port'])) try: - response = urllib2.urlopen(url) - except urllib2.HTTPError as err: + response = diamond.pycompat.urlopen(url) + except HTTPError as err: self.log.error("%s: %s", url, err) return diff --git a/src/collectors/dropwizard/test/testdropwizard.py b/src/collectors/dropwizard/test/testdropwizard.py index d4246bda0..433429ba4 100644 --- a/src/collectors/dropwizard/test/testdropwizard.py +++ b/src/collectors/dropwizard/test/testdropwizard.py @@ -9,6 +9,7 @@ from test import patch from diamond.collector import Collector +from diamond.pycompat import URLOPEN from dropwizard import DropwizardCollector @@ -27,7 +28,7 @@ def test_import(self): @patch.object(Collector, 'publish') def test_should_work_with_real_data(self, publish_mock): - patch_urlopen = patch('urllib2.urlopen', + patch_urlopen = patch(URLOPEN, Mock(return_value=self.getFixture('stats'))) patch_urlopen.start() @@ -69,7 +70,7 @@ def test_should_work_with_real_data(self, publish_mock): @patch.object(Collector, 'publish') def test_should_fail_gracefully(self, publish_mock): patch_urlopen = patch( - 'urllib2.urlopen', + URLOPEN, Mock( return_value=self.getFixture('stats_blank'))) diff --git a/src/collectors/dseopscenter/dseopscenter.py b/src/collectors/dseopscenter/dseopscenter.py index 1217101f9..64190c874 100644 --- a/src/collectors/dseopscenter/dseopscenter.py +++ b/src/collectors/dseopscenter/dseopscenter.py @@ -3,8 +3,6 @@ """ Collect the DataStax OpsCenter metrics """ - -import urllib2 import datetime try: @@ -13,6 +11,7 @@ import simplejson as json import diamond.collector +import diamond.pycompat class DseOpsCenterCollector(diamond.collector.Collector): @@ -119,7 +118,7 @@ def _get_schema(self): int(self.config['port']), self.config['cluster_id']) try: - response = urllib2.urlopen(url) + response = diamond.pycompat.urlopen(url) except Exception as err: self.log.error('%s: %s', url, err) return False @@ -157,7 +156,7 @@ def _get(self, start, end, step=60): self.config['default_tail_opts']) try: - response = urllib2.urlopen(url) + response = diamond.pycompat.urlopen(url) except Exception as err: self.log.error('%s: %s', url, err) return False diff --git a/src/collectors/dseopscenter/test/testdseopscenter.py b/src/collectors/dseopscenter/test/testdseopscenter.py index f654fe749..77a777dec 100644 --- a/src/collectors/dseopscenter/test/testdseopscenter.py +++ b/src/collectors/dseopscenter/test/testdseopscenter.py @@ -8,6 +8,7 @@ from test import patch from diamond.collector import Collector +from diamond.pycompat import URLOPEN from dseopscenter import DseOpsCenterCollector @@ -27,12 +28,12 @@ def test_import(self): @patch.object(Collector, 'publish') def test_should_work_with_real_data(self, publish_mock): - urlopen_mock1 = patch('urllib2.urlopen', Mock( + urlopen_mock1 = patch(URLOPEN, Mock( side_effect=lambda *args: self.getFixture('keyspaces.json'))) urlopen_mock1.start() self.collector._get_schema() urlopen_mock1.stop() - urlopen_mock2 = patch('urllib2.urlopen', Mock( + urlopen_mock2 = patch(URLOPEN, Mock( side_effect=lambda *args: self.getFixture('new-metrics.json'))) urlopen_mock2.start() self.collector.collect() diff --git a/src/collectors/elasticsearch/elasticsearch.py b/src/collectors/elasticsearch/elasticsearch.py index e9ec4d6a9..fa2ec59e6 100755 --- a/src/collectors/elasticsearch/elasticsearch.py +++ b/src/collectors/elasticsearch/elasticsearch.py @@ -6,17 +6,12 @@ Supports multiple instances. When using the 'instances' parameter the instance alias will be appended to the 'path' parameter. - -#### Dependencies - - * urlib2 - """ -import urllib2 import base64 import re from diamond.collector import str_to_bool +import diamond.pycompat try: import json @@ -111,12 +106,12 @@ def _get(self, scheme, host, port, path, assert_key=None): """ url = '%s://%s:%i/%s' % (scheme, host, port, path) try: - request = urllib2.Request(url) + request = diamond.pycompat.Request(url) if self.config['user'] and self.config['password']: base64string = base64.standard_b64encode( '%s:%s' % (self.config['user'], self.config['password'])) request.add_header("Authorization", "Basic %s" % base64string) - response = urllib2.urlopen(request) + response = diamond.pycompat.urlopen(request) except Exception as err: self.log.error("%s: %s" % (url, err)) return False @@ -246,7 +241,7 @@ def collect_instance(self, alias, scheme, host, port): return metrics = {} - node = result['nodes'].keys()[0] + node = list(result['nodes'].keys())[0] data = result['nodes'][node] # diff --git a/src/collectors/elasticsearch/test/testelasticsearch.py b/src/collectors/elasticsearch/test/testelasticsearch.py index 80cf410b0..26e0c332c 100644 --- a/src/collectors/elasticsearch/test/testelasticsearch.py +++ b/src/collectors/elasticsearch/test/testelasticsearch.py @@ -9,6 +9,7 @@ from test import patch from diamond.collector import Collector +from diamond.pycompat import URLOPEN from elasticsearch import ElasticSearchCollector @@ -63,7 +64,7 @@ def test_should_work_with_real_data(self, publish_mock): self.getFixture('cluster_stats'), self.getFixture('indices_stats'), ] - urlopen_mock = patch('urllib2.urlopen', Mock( + urlopen_mock = patch(URLOPEN, Mock( side_effect=lambda *args: returns.pop(0))) self.collector.config['cluster'] = True @@ -199,7 +200,7 @@ def test_should_work_with_real_data_logstash_mode(self, publish_mock): self.getFixture('stats'), self.getFixture('logstash_indices_stats'), ] - urlopen_mock = patch('urllib2.urlopen', Mock( + urlopen_mock = patch(URLOPEN, Mock( side_effect=lambda *args: returns.pop(0))) self.collector.config['logstash_mode'] = True @@ -343,7 +344,7 @@ def test_should_work_with_real_0_90_data(self, publish_mock): self.getFixture('stats0.90'), self.getFixture('indices_stats'), ] - urlopen_mock = patch('urllib2.urlopen', Mock( + urlopen_mock = patch(URLOPEN, Mock( side_effect=lambda *args: returns.pop(0))) urlopen_mock.start() @@ -369,7 +370,7 @@ def test_should_work_with_real_0_90_data(self, publish_mock): @patch.object(Collector, 'publish') def test_should_fail_gracefully(self, publish_mock): - urlopen_mock = patch('urllib2.urlopen', Mock( + urlopen_mock = patch(URLOPEN, Mock( return_value=self.getFixture('stats_blank'))) urlopen_mock.start() @@ -394,7 +395,7 @@ def test_multi_instances_with_real_data(self, publish_mock): self.getFixture('stats2'), self.getFixture('indices_stats2'), ] - urlopen_mock = patch('urllib2.urlopen', Mock( + urlopen_mock = patch(URLOPEN, Mock( side_effect=lambda *args: returns.pop(0))) urlopen_mock.start() diff --git a/src/collectors/elb/test/testelb.py b/src/collectors/elb/test/testelb.py index a88af31bf..aba09a220 100644 --- a/src/collectors/elb/test/testelb.py +++ b/src/collectors/elb/test/testelb.py @@ -2,7 +2,6 @@ # coding=utf-8 import datetime -import mock from test import CollectorTestCase from test import get_collector_config @@ -99,8 +98,8 @@ def test_ignore(self, publish_metric, elb_connect_to_region, collector = ElbCollector(config, handlers=[]) target = ts + datetime.timedelta(minutes=1) - with mock.patch.object(datetime, 'datetime', - mock.Mock(wraps=datetime.datetime)) as patched: + with patch.object(datetime, 'datetime', + Mock(wraps=datetime.datetime)) as patched: patched.utcnow.return_value = target collector.collect() @@ -175,8 +174,8 @@ def test_collect(self, publish_metric, connect_to_region, cloudwatch): collector = ElbCollector(config, handlers=[]) target = ts + datetime.timedelta(minutes=1) - with mock.patch.object(datetime, 'datetime', - mock.Mock(wraps=datetime.datetime)) as patched: + with patch.object(datetime, 'datetime', + Mock(wraps=datetime.datetime)) as patched: patched.utcnow.return_value = target collector.collect() diff --git a/src/collectors/endecadgraph/endecadgraph.py b/src/collectors/endecadgraph/endecadgraph.py index 86a31cc9b..6e8fad8f2 100644 --- a/src/collectors/endecadgraph/endecadgraph.py +++ b/src/collectors/endecadgraph/endecadgraph.py @@ -12,8 +12,11 @@ """ import diamond.collector -import urllib2 -from StringIO import StringIO +import diamond.pycompat +try: + from StringIO import StringIO +except ImportError: + from io import StringIO import re import sys if sys.version_info >= (2, 5): @@ -115,7 +118,7 @@ def walkXML(context, elemList): url = 'http://%s:%d/admin?op=stats' % (self.config['host'], self.config['port']) try: - xml = urllib2.urlopen(url, timeout=self.config['timeout']).read() + xml = diamond.pycompat.urlopen(url, timeout=self.config['timeout']).read() except Exception as e: self.log.error('Could not connect to endeca on %s: %s' % (url, e)) return {} diff --git a/src/collectors/endecadgraph/test/testendecadgraph.py b/src/collectors/endecadgraph/test/testendecadgraph.py index 1d12f688a..0419c333f 100644 --- a/src/collectors/endecadgraph/test/testendecadgraph.py +++ b/src/collectors/endecadgraph/test/testendecadgraph.py @@ -7,6 +7,7 @@ from test import patch from diamond.collector import Collector +from diamond.pycompat import URLOPEN from endecadgraph import EndecaDgraphCollector @@ -20,7 +21,7 @@ def setUp(self): def test_import(self): self.assertTrue(EndecaDgraphCollector) - @patch('urllib2.urlopen') + @patch(URLOPEN) @patch.object(Collector, 'publish') def test_real_data(self, publish_mock, urlopen_mock): urlopen_mock.return_value = self.getFixture('data1.xml') diff --git a/src/collectors/flume/flume.py b/src/collectors/flume/flume.py index 3109803bc..d10c076a9 100644 --- a/src/collectors/flume/flume.py +++ b/src/collectors/flume/flume.py @@ -5,14 +5,12 @@ #### Dependencies - * urllib2 * json or simplejson """ -import urllib2 import diamond.collector - +import diamond.pycompat try: import simplejson as json except ImportError: @@ -79,7 +77,7 @@ def collect(self): ) try: - resp = urllib2.urlopen(url) + resp = diamond.pycompat.urlopen(url) try: j = json.loads(resp.read()) resp.close() @@ -87,7 +85,7 @@ def collect(self): resp.close() self.log.error('Cannot load json data: %s', e) return None - except urllib2.URLError as e: + except diamond.pycompat.URLError as e: self.log.error('Failed to open url: %s', e) return None except Exception as e: diff --git a/src/collectors/flume/test/testflume.py b/src/collectors/flume/test/testflume.py index 5d38a68b9..4a4f14efe 100644 --- a/src/collectors/flume/test/testflume.py +++ b/src/collectors/flume/test/testflume.py @@ -8,6 +8,7 @@ from test import Mock from diamond.collector import Collector +from diamond.pycompat import URLOPEN from flume import FlumeCollector @@ -30,7 +31,7 @@ def test_collect_should_work(self, publish_mock, publish_gauge_mock, publish_counter_mock): - patch_urlopen = patch('urllib2.urlopen', + patch_urlopen = patch(URLOPEN, Mock(return_value=self.getFixture('metrics'))) patch_urlopen.start() @@ -70,7 +71,7 @@ def test_collect_should_work(self, @patch.object(Collector, 'publish') def test_blank_should_fail_gracefully(self, publish_mock): - patch_urlopen = patch('urllib2.urlopen', Mock( + patch_urlopen = patch(URLOPEN, Mock( return_value=self.getFixture('metrics_blank'))) patch_urlopen.start() @@ -82,7 +83,7 @@ def test_blank_should_fail_gracefully(self, publish_mock): @patch.object(Collector, 'publish') def test_invalid_should_fail_gracefully(self, publish_mock): patch_urlopen = patch( - 'urllib2.urlopen', + URLOPEN, Mock(return_value=self.getFixture('metrics_invalid'))) patch_urlopen.start() diff --git a/src/collectors/haproxy/haproxy.py b/src/collectors/haproxy/haproxy.py index f4e1963bf..0d5772aa2 100644 --- a/src/collectors/haproxy/haproxy.py +++ b/src/collectors/haproxy/haproxy.py @@ -6,16 +6,15 @@ #### Dependencies * urlparse - * urllib2 - """ import re -import urllib2 import base64 import csv import socket import diamond.collector +import diamond.pycompat +from diamond.pycompat import Request class HAProxyCollector(diamond.collector.Collector): @@ -64,9 +63,9 @@ def http_get_csv_data(self, section=None): Request stats from HAProxy Server """ metrics = [] - req = urllib2.Request(self._get_config_value(section, 'url')) + req = Request(self._get_config_value(section, 'url')) try: - handle = urllib2.urlopen(req) + handle = diamond.pycompat.urlopen(req) return handle.readlines() except Exception as e: if not hasattr(e, 'code') or e.code != 401: @@ -101,7 +100,7 @@ def http_get_csv_data(self, section=None): authheader = 'Basic %s' % base64string req.add_header("Authorization", authheader) try: - handle = urllib2.urlopen(req) + handle = diamond.pycompat.urlopen(req) metrics = handle.readlines() return metrics except IOError as e: diff --git a/src/collectors/haproxy/test/testhaproxy.py b/src/collectors/haproxy/test/testhaproxy.py index 6fd713ee3..145924e00 100644 --- a/src/collectors/haproxy/test/testhaproxy.py +++ b/src/collectors/haproxy/test/testhaproxy.py @@ -9,6 +9,7 @@ from test import patch from diamond.collector import Collector +from diamond.pycompat import URLOPEN from haproxy import HAProxyCollector ########################################################################## @@ -30,7 +31,7 @@ def test_import(self): def test_should_work_with_real_data(self, publish_mock): self.collector.config['ignore_servers'] = False - patch_urlopen = patch('urllib2.urlopen', + patch_urlopen = patch(URLOPEN, Mock(return_value=self.getFixture('stats.csv'))) patch_urlopen.start() @@ -84,7 +85,7 @@ def recv(obj, bufsize, *args, **kwargs): def test_should_work_with_real_data_and_ignore_servers(self, publish_mock): self.collector.config['ignore_servers'] = True - patch_urlopen = patch('urllib2.urlopen', + patch_urlopen = patch(URLOPEN, Mock(return_value=self.getFixture('stats.csv'))) patch_urlopen.start() diff --git a/src/collectors/http/http.py b/src/collectors/http/http.py index 1f5c73e4a..ae6e37af6 100644 --- a/src/collectors/http/http.py +++ b/src/collectors/http/http.py @@ -3,10 +3,6 @@ """ Collect statistics from a HTTP or HTTPS connexion -#### Dependencies - - * urllib2 - #### Usage Add the collector config as : @@ -27,8 +23,9 @@ http:__www_site_com_admin_page_html """ -import urllib2 import diamond.collector +import diamond.pycompat +from diamond.pycompat import Request import datetime @@ -55,7 +52,7 @@ def get_default_config(self): return default_config def collect(self): - # create urllib2 vars + # create Request vars if self.config['req_vhost'] != "": self.config['headers']['Host'] = self.config['req_vhost'] @@ -63,9 +60,9 @@ def collect(self): for url in self.config['req_url']: self.log.debug("collecting %s", str(url)) req_start = datetime.datetime.now() - req = urllib2.Request(url, headers=self.config['headers']) + req = diamond.pycompat.Request(url, headers=self.config['headers']) try: - handle = urllib2.urlopen(req) + handle = diamond.pycompat.urlopen(req) the_page = handle.read() req_end = datetime.datetime.now() req_time = req_end - req_start diff --git a/src/collectors/http/test/testhttp.py b/src/collectors/http/test/testhttp.py index 74570edf6..e393e8351 100644 --- a/src/collectors/http/test/testhttp.py +++ b/src/collectors/http/test/testhttp.py @@ -9,6 +9,7 @@ from test import patch from diamond.collector import Collector +from diamond.pycompat import URLOPEN from http import HttpCollector @@ -30,7 +31,7 @@ def test_import(self): @patch.object(Collector, 'publish') def test_should_work_with_real_data(self, publish_mock): - patch_urlopen = patch('urllib2.urlopen', Mock( + patch_urlopen = patch(URLOPEN, Mock( return_value=self.getFixture('index'))) patch_urlopen.start() diff --git a/src/collectors/httpd/httpd.py b/src/collectors/httpd/httpd.py index 81463725c..afddb46ab 100644 --- a/src/collectors/httpd/httpd.py +++ b/src/collectors/httpd/httpd.py @@ -6,16 +6,16 @@ #### Dependencies * mod_status - * httplib - * urlparse - """ import re -import httplib -import urlparse -import diamond.collector +try: + from httplib import HTTPConnection +except ImportError: + from http.client import HTTPConnection +import diamond.collector +from diamond.pycompat import urlparse class HttpdCollector(diamond.collector.Collector): @@ -66,7 +66,7 @@ def collect(self): while True: # Parse Url - parts = urlparse.urlparse(url) + parts = urlparse(url) # Parse host and port endpoint = parts[1].split(':') @@ -78,8 +78,8 @@ def collect(self): service_port = 80 # Setup Connection - connection = httplib.HTTPConnection(service_host, - service_port) + connection = HTTPConnection(service_host, + service_port) url = "%s?%s" % (parts[2], parts[4]) diff --git a/src/collectors/httpd/test/testhttpd.py b/src/collectors/httpd/test/testhttpd.py index 866b0a2ed..233db6bac 100644 --- a/src/collectors/httpd/test/testhttpd.py +++ b/src/collectors/httpd/test/testhttpd.py @@ -10,13 +10,15 @@ from diamond.collector import Collector from httpd import HttpdCollector -import httplib +try: + from httplib import HTTPConnection, HTTPResponse +except ImportError: + from http.client import HTTPConnection, HTTPResponse ########################################################################## -class TestHTTPResponse(httplib.HTTPResponse): - +class TestHTTPResponse(HTTPResponse): def __init__(self): pass @@ -39,8 +41,8 @@ def setUp(self, config=None): self.HTTPResponse = TestHTTPResponse() - httplib.HTTPConnection.request = Mock(return_value=True) - httplib.HTTPConnection.getresponse = Mock( + HTTPConnection.request = Mock(return_value=True) + HTTPConnection.getresponse = Mock( return_value=self.HTTPResponse) def test_import(self): diff --git a/src/collectors/httpjson/httpjson.py b/src/collectors/httpjson/httpjson.py index 304f0beee..32685a2d8 100644 --- a/src/collectors/httpjson/httpjson.py +++ b/src/collectors/httpjson/httpjson.py @@ -2,17 +2,12 @@ """ Simple collector which get JSON and parse it into flat metrics - -#### Dependencies - - * urllib2 - """ -import urllib2 import json import diamond.collector - +import diamond.pycompat +from diamond.pycompat import Request, URLError class HTTPJSONCollector(diamond.collector.Collector): @@ -51,12 +46,12 @@ def _json_to_flat_metrics(self, prefix, data): def collect(self): url = self.config['url'] - req = urllib2.Request(url, headers=self.config['headers']) + req = Request(url, headers=self.config['headers']) req.add_header('Content-type', 'application/json') try: - resp = urllib2.urlopen(req) - except urllib2.URLError as e: + resp = diamond.pycompat.urlopen(req) + except URLError as e: self.log.error("Can't open url %s. %s", url, e) else: diff --git a/src/collectors/httpjson/test/testhttpjson.py b/src/collectors/httpjson/test/testhttpjson.py index acc1a6cff..0a90435c3 100644 --- a/src/collectors/httpjson/test/testhttpjson.py +++ b/src/collectors/httpjson/test/testhttpjson.py @@ -6,6 +6,7 @@ from test import Mock from test import patch from diamond.collector import Collector +from diamond.pycompat import URLOPEN from httpjson import HTTPJSONCollector ########################################################################## @@ -22,7 +23,7 @@ def test_import(self): @patch.object(Collector, 'publish') def test_should_work_with_real_data(self, publish_mock): - urlopen_mock = patch('urllib2.urlopen', + urlopen_mock = patch(URLOPEN, Mock(return_value=self.getFixture('stats.json'))) urlopen_mock.start() diff --git a/src/collectors/jolokia/jolokia.py b/src/collectors/jolokia/jolokia.py index c242bc1ee..fda15a80f 100644 --- a/src/collectors/jolokia/jolokia.py +++ b/src/collectors/jolokia/jolokia.py @@ -51,10 +51,10 @@ import diamond.collector import base64 from contextlib import closing +import diamond.pycompat import json import re -import urllib -import urllib2 +from diamond.pycompat import HTTPError, long, quote class JolokiaCollector(diamond.collector.Collector): @@ -230,10 +230,10 @@ def _list_request(self): # need some time to process the downloaded metrics, so that's why # timeout is lower than the interval. timeout = max(2, float(self.config['interval']) * 2 / 3) - with closing(urllib2.urlopen(self._create_request(url), + with closing(diamond.pycompat.urlopen(self._create_request(url), timeout=timeout)) as response: return self._read_json(response) - except (urllib2.HTTPError, ValueError) as e: + except (HTTPError, ValueError) as e: self.log.error('Unable to read JSON response: %s', str(e)) return {} @@ -253,10 +253,10 @@ def _read_request(self, domain): # need some time to process the downloaded metrics, so that's why # timeout is lower than the interval. timeout = max(2, float(self.config['interval']) * 2 / 3) - with closing(urllib2.urlopen(self._create_request(url), + with closing(diamond.pycompat.urlopen(self._create_request(url), timeout=timeout)) as response: return self._read_json(response) - except (urllib2.HTTPError, ValueError): + except (HTTPError, ValueError): self.log.error('Unable to read JSON response.') return {} @@ -268,7 +268,7 @@ def _escape_domain(self, domain): domain = re.sub('!', '!!', domain) domain = re.sub('/', '!/', domain) domain = re.sub('"', '!"', domain) - domain = urllib.quote(domain) + domain = quote(domain) return domain def _create_request(self, url): diff --git a/src/collectors/jolokia/test/testjolokia.py b/src/collectors/jolokia/test/testjolokia.py index 63d65b13f..7449f17f5 100644 --- a/src/collectors/jolokia/test/testjolokia.py +++ b/src/collectors/jolokia/test/testjolokia.py @@ -10,6 +10,7 @@ import re from diamond.collector import Collector +from diamond.pycompat import URLOPEN from jolokia import JolokiaCollector @@ -33,7 +34,7 @@ def se(url, timeout=0): return self.getFixture('listing') else: return self.getFixture('stats') - patch_urlopen = patch('urllib2.urlopen', Mock(side_effect=se)) + patch_urlopen = patch(URLOPEN, Mock(side_effect=se)) patch_urlopen.start() self.collector.collect() @@ -52,7 +53,7 @@ def se(url, timeout=0): return self.getFixture('listing') else: return self.getFixture('stats') - patch_urlopen = patch('urllib2.urlopen', Mock(side_effect=se)) + patch_urlopen = patch(URLOPEN, Mock(side_effect=se)) patch_urlopen.start() rewrite = [ @@ -74,7 +75,7 @@ def test_should_work_with_real_data_and_basic_auth(self, publish_mock): @patch.object(Collector, 'publish') def test_should_fail_gracefully(self, publish_mock): - patch_urlopen = patch('urllib2.urlopen', Mock( + patch_urlopen = patch(URLOPEN, Mock( return_value=self.getFixture('stats_blank'))) patch_urlopen.start() @@ -93,7 +94,7 @@ def se(url, timeout=0): return self.getFixture('stats_error') else: return self.getFixture('stats') - patch_urlopen = patch('urllib2.urlopen', Mock(side_effect=se)) + patch_urlopen = patch(URLOPEN, Mock(side_effect=se)) patch_urlopen.start() self.collector.collect() diff --git a/src/collectors/kafkastat/kafkastat.py b/src/collectors/kafkastat/kafkastat.py index 9d28c46f2..77c1f29d1 100644 --- a/src/collectors/kafkastat/kafkastat.py +++ b/src/collectors/kafkastat/kafkastat.py @@ -5,13 +5,8 @@ #### Dependencies - * urllib2 * xml.etree """ -import urllib2 - -from urllib import urlencode - try: from xml.etree import ElementTree except ImportError: @@ -23,7 +18,8 @@ ETParseError = Exception import diamond.collector - +import diamond.pycompat +from diamond.pycompat import long, urlencode, URLError class KafkaCollector(diamond.collector.Collector): ATTRIBUTE_TYPES = { @@ -67,8 +63,8 @@ def _get(self, path, query_args=None): path, urlencode(qargs)) try: - response = urllib2.urlopen(url) - except urllib2.URLError as err: + response = diamond.pycompat.urlopen(url) + except URLError as err: self.log.error("%s: %s", url, err) return None diff --git a/src/collectors/kafkastat/test/testkafka.py b/src/collectors/kafkastat/test/testkafka.py index 6ff62ded8..55293d274 100644 --- a/src/collectors/kafkastat/test/testkafka.py +++ b/src/collectors/kafkastat/test/testkafka.py @@ -1,9 +1,6 @@ #!/usr/bin/python # coding=utf-8 ############################################################################### -import urllib2 -from urlparse import urlparse, parse_qs - try: from xml.etree import ElementTree except ImportError: @@ -16,6 +13,7 @@ from test import patch from diamond.collector import Collector +from diamond.pycompat import URLError, URLOPEN from kafkastat import KafkaCollector ########## @@ -48,7 +46,7 @@ def test_import(self): self.assertTrue(KafkaCollector) @run_only_if_ElementTree_is_available - @patch('urllib2.urlopen') + @patch(URLOPEN) def test_get(self, urlopen_mock): urlopen_mock.return_value = self.getFixture('empty.xml') @@ -58,16 +56,16 @@ def test_get(self, urlopen_mock): self.assertEqual(result_string, '') @run_only_if_ElementTree_is_available - @patch('urllib2.urlopen') + @patch(URLOPEN) def test_get_httperror(self, urlopen_mock): - urlopen_mock.side_effect = urllib2.URLError('BOOM') + urlopen_mock.side_effect = URLError('BOOM') result = self.collector._get('/path') self.assertFalse(result) @run_only_if_ElementTree_is_available - @patch('urllib2.urlopen') + @patch(URLOPEN) def test_get_bad_xml(self, urlopen_mock): urlopen_mock.return_value = self.getFixture('bad.xml') @@ -173,7 +171,7 @@ def getKafkaFixture(self, url): return '' @run_only_if_ElementTree_is_available - @patch('urllib2.urlopen') + @patch(URLOPEN) @patch.object(Collector, 'publish') def test(self, publish_mock, urlopen_mock): urlopen_mock.side_effect = self.getKafkaFixture diff --git a/src/collectors/mesos/mesos.py b/src/collectors/mesos/mesos.py index 8c1e23c1b..c5ca5866a 100644 --- a/src/collectors/mesos/mesos.py +++ b/src/collectors/mesos/mesos.py @@ -19,9 +19,9 @@ import copy import diamond.collector +import diamond.pycompat +from diamond.pycompat import HTTPError import json -import urllib2 -from urlparse import urlparse import diamond.collector @@ -199,7 +199,7 @@ def _get(self, path): """ url = self._get_url(path) try: - response = urllib2.urlopen(url) + response = diamond.pycompat.urlopen(url) except Exception as err: self.log.error("%s: %s", url, err) return False diff --git a/src/collectors/mesos/test/testmesos.py b/src/collectors/mesos/test/testmesos.py index 89398615f..92c8dcf15 100644 --- a/src/collectors/mesos/test/testmesos.py +++ b/src/collectors/mesos/test/testmesos.py @@ -9,6 +9,7 @@ from test import patch from diamond.collector import Collector +from diamond.pycompat import URLOPEN from mesos import MesosCollector @@ -31,7 +32,7 @@ def test_import(self): @patch.object(Collector, 'publish') def test_should_work_for_master_with_real_data(self, publish_mock): returns = self.getFixture('master_metrics_snapshot.json') - urlopen_mock = patch('urllib2.urlopen', Mock( + urlopen_mock = patch('diamond.pycompat.urlopen', Mock( side_effect=lambda *args: returns)) urlopen_mock.start() @@ -114,7 +115,7 @@ def test_should_compute_cpus_utilisation(self, publish_mock): @patch.object(Collector, 'publish') def test_should_fail_gracefully(self, publish_mock): - patch_urlopen = patch('urllib2.urlopen', Mock( + patch_urlopen = patch(URLOPEN, Mock( return_value=self.getFixture('metrics_blank'))) patch_urlopen.start() diff --git a/src/collectors/monit/monit.py b/src/collectors/monit/monit.py index abd3e6d92..59a5c5a34 100644 --- a/src/collectors/monit/monit.py +++ b/src/collectors/monit/monit.py @@ -8,8 +8,6 @@ * monit serving up /_status """ - -import urllib2 import base64 from xml.dom.minidom import parseString @@ -17,6 +15,8 @@ import diamond.collector from diamond.collector import str_to_bool +import diamond.pycompat +from diamond.pycompat import HTTPError, Request class MonitCollector(diamond.collector.Collector): @@ -47,15 +47,15 @@ def collect(self): url = 'http://%s:%i/_status?format=xml' % (self.config['host'], int(self.config['port'])) try: - request = urllib2.Request(url) + request = Request(url) # # shouldn't need to check this - base64string = base64.encodestring('%s:%s' % ( - self.config['user'], self.config['passwd'])).replace('\n', '') + original_string = '%s:%s' % (self.config['user'], self.config['passwd']) + base64string = base64.encodestring(original_string.encode("utf8")).replace(b'\n', b'') request.add_header("Authorization", "Basic %s" % base64string) - response = urllib2.urlopen(request) - except urllib2.HTTPError as err: + response = diamond.pycompat.urlopen(request) + except HTTPError as err: self.log.error("%s: %s", err, url) return diff --git a/src/collectors/monit/test/testmonit.py b/src/collectors/monit/test/testmonit.py index 480bcb624..d5641ab4f 100644 --- a/src/collectors/monit/test/testmonit.py +++ b/src/collectors/monit/test/testmonit.py @@ -9,6 +9,7 @@ from test import patch from diamond.collector import Collector +from diamond.pycompat import URLOPEN from monit import MonitCollector @@ -28,7 +29,7 @@ def test_import(self): @patch.object(Collector, 'publish') def test_should_work_with_real_data(self, publish_mock): - patch_urlopen = patch('urllib2.urlopen', Mock( + patch_urlopen = patch(URLOPEN, Mock( return_value=self.getFixture('status.xml'))) patch_urlopen.start() @@ -66,7 +67,7 @@ def test_should_work_with_real_data(self, publish_mock): @patch.object(Collector, 'publish') def test_should_fail_gracefully(self, publish_mock): patch_urlopen = patch( - 'urllib2.urlopen', + URLOPEN, Mock( return_value=self.getFixture( 'status_blank.xml'))) diff --git a/src/collectors/nginx/nginx.py b/src/collectors/nginx/nginx.py index 8272fab07..058e454c6 100644 --- a/src/collectors/nginx/nginx.py +++ b/src/collectors/nginx/nginx.py @@ -1,11 +1,10 @@ -# coding=utf-8 + # coding=utf-8 """ Collect statistics from Nginx and Nginx+ #### Dependencies - * urllib2 * json #### Usage @@ -55,12 +54,11 @@ """ - -import urllib2 import re import diamond.collector import json +import diamond.pycompat class NginxCollector(diamond.collector.Collector): @@ -217,9 +215,9 @@ def collect(self): int(self.config['req_port']), self.config['req_path']) - req = urllib2.Request(url=url, headers=headers) + req = diamond.pycompat.Request(url=url, headers=headers) try: - handle = urllib2.urlopen(req) + handle = diamond.pycompat.urlopen(req) # Test for json payload; indicates nginx+ if handle.info().gettype() == 'application/json': @@ -228,7 +226,6 @@ def collect(self): # Plain payload; indicates open source nginx else: self.collect_nginx(handle) - except IOError as e: self.log.error("Unable to open %s" % url) except Exception as e: diff --git a/src/collectors/nginx/test/testnginx.py b/src/collectors/nginx/test/testnginx.py index 3e2ab360c..ff41147fe 100644 --- a/src/collectors/nginx/test/testnginx.py +++ b/src/collectors/nginx/test/testnginx.py @@ -9,6 +9,7 @@ from test import patch from diamond.collector import Collector +from diamond.pycompat import URLOPEN from nginx import NginxCollector @@ -38,7 +39,7 @@ def test_should_work_with_real_data(self, publish_counter_mock, } ) - patch_urlopen = patch('urllib2.urlopen', Mock( + patch_urlopen = patch(URLOPEN, Mock( return_value=mockResponse)) patch_urlopen.start() @@ -179,7 +180,7 @@ def test_should_fail_gracefully(self, publish_mock): } ) - patch_urlopen = patch('urllib2.urlopen', Mock( + patch_urlopen = patch(URLOPEN, Mock( return_value=mockResponse)) patch_urlopen.start() diff --git a/src/collectors/openvpn/openvpn.py b/src/collectors/openvpn/openvpn.py index fe99b9f17..5a4338158 100644 --- a/src/collectors/openvpn/openvpn.py +++ b/src/collectors/openvpn/openvpn.py @@ -27,18 +27,13 @@ You can also specify multiple and mixed instances:: instances = file:///var/log/openvpn/openvpn.log, tcp://10.0.0.1:1195?admins - -#### Dependencies - - * urlparse - """ import socket import diamond.collector import os.path -import urlparse import time +from diamond.pycompat import urlparse class OpenVPNCollector(diamond.collector.Collector): @@ -67,7 +62,7 @@ def parse_url(self, uri): """ Convert urlparse from a python 2.4 layout to a python 2.7 layout """ - parsed = urlparse.urlparse(uri) + parsed = urlparse(uri) if 'scheme' not in parsed: class Object(object): pass diff --git a/src/collectors/phpfpm/phpfpm.py b/src/collectors/phpfpm/phpfpm.py index 4c2763c8e..9d283645b 100644 --- a/src/collectors/phpfpm/phpfpm.py +++ b/src/collectors/phpfpm/phpfpm.py @@ -26,8 +26,7 @@ #### Dependencies - * urllib2 - * json (or simeplejson) + * json (or simplejson) """ @@ -36,8 +35,8 @@ except ImportError: import simplejson as json -import urllib2 import diamond.collector +import diamond.pycompat class PhpFpmCollector(diamond.collector.Collector): @@ -72,7 +71,7 @@ def collect(self): self.config['uri'] = self.config['uri'][1:] try: - response = urllib2.urlopen("http://%s:%s/%s?json" % ( + response = diamond.pycompat.urlopen("http://%s:%s/%s?json" % ( self.config['host'], int(self.config['port']), self.config['uri'])) except Exception as e: diff --git a/src/collectors/phpfpm/test/testphpfpm.py b/src/collectors/phpfpm/test/testphpfpm.py index c6868106c..b07158b6a 100644 --- a/src/collectors/phpfpm/test/testphpfpm.py +++ b/src/collectors/phpfpm/test/testphpfpm.py @@ -9,6 +9,7 @@ from test import patch from diamond.collector import Collector +from diamond.pycompat import URLOPEN from phpfpm import PhpFpmCollector ########################################################################## @@ -28,7 +29,7 @@ def test_import(self): @patch.object(Collector, 'publish') def test_should_work_with_real_data(self, publish_mock): - patch_urlopen = patch('urllib2.urlopen', Mock( + patch_urlopen = patch(URLOPEN, Mock( return_value=self.getFixture('stats'))) patch_urlopen.start() @@ -52,7 +53,7 @@ def test_should_work_with_real_data(self, publish_mock): @patch.object(Collector, 'publish') def test_should_fail_gracefully(self, publish_mock): - patch_urlopen = patch('urllib2.urlopen', Mock( + patch_urlopen = patch(URLOPEN, Mock( return_value=self.getFixture('stats_blank'))) patch_urlopen.start() diff --git a/src/collectors/puppetdashboard/puppetdashboard.py b/src/collectors/puppetdashboard/puppetdashboard.py index b07e38159..11f06064c 100644 --- a/src/collectors/puppetdashboard/puppetdashboard.py +++ b/src/collectors/puppetdashboard/puppetdashboard.py @@ -2,17 +2,11 @@ """ Collect metrics from Puppet Dashboard - -#### Dependencies - - * urllib2 - """ -import urllib2 import re import diamond.collector - +import diamond.pycompat class PuppetDashboardCollector(diamond.collector.Collector): @@ -40,7 +34,7 @@ def get_default_config(self): def collect(self): try: - response = urllib2.urlopen("http://%s:%s/" % ( + response = diamond.pycompat.urlopen("http://%s:%s/" % ( self.config['host'], int(self.config['port']))) except Exception as e: self.log.error('Couldnt connect to puppet-dashboard: %s', e) diff --git a/src/collectors/puppetdashboard/test/testpuppetdashboard.py b/src/collectors/puppetdashboard/test/testpuppetdashboard.py index 8b080e594..54a7f3f90 100644 --- a/src/collectors/puppetdashboard/test/testpuppetdashboard.py +++ b/src/collectors/puppetdashboard/test/testpuppetdashboard.py @@ -9,6 +9,7 @@ from test import patch from diamond.collector import Collector +from diamond.pycompat import URLOPEN from puppetdashboard import PuppetDashboardCollector ########################################################################## @@ -28,7 +29,7 @@ def test_import(self): @patch.object(Collector, 'publish') def test_should_work_with_real_data(self, publish_mock): - patch_urlopen = patch('urllib2.urlopen', Mock( + patch_urlopen = patch(URLOPEN, Mock( return_value=self.getFixture('index.html'))) patch_urlopen.start() @@ -50,7 +51,7 @@ def test_should_work_with_real_data(self, publish_mock): @patch.object(Collector, 'publish') def test_should_fail_gracefully(self, publish_mock): - patch_urlopen = patch('urllib2.urlopen', Mock( + patch_urlopen = patch(URLOPEN, Mock( return_value=self.getFixture('index.blank'))) patch_urlopen.start() diff --git a/src/collectors/puppetdb/puppetdb.py b/src/collectors/puppetdb/puppetdb.py index 34320bc1b..eeb7894d5 100644 --- a/src/collectors/puppetdb/puppetdb.py +++ b/src/collectors/puppetdb/puppetdb.py @@ -5,14 +5,13 @@ #### Dependencies - * urllib2 * json """ -import urllib2 import diamond.collector from diamond.convertor import time as time_convertor +import diamond.pycompat try: import json @@ -90,7 +89,7 @@ def fetch_metrics(self, url): try: url = "http://%s:%s/%s" % ( self.config['host'], int(self.config['port']), url) - response = urllib2.urlopen(url) + response = diamond.pycompat.urlopen(url) except Exception as e: self.log.error('Couldn\'t connect to puppetdb: %s -> %s', url, e) return {} diff --git a/src/collectors/rabbitmq/rabbitmq.py b/src/collectors/rabbitmq/rabbitmq.py index c9e94ae8b..d671ae1f2 100644 --- a/src/collectors/rabbitmq/rabbitmq.py +++ b/src/collectors/rabbitmq/rabbitmq.py @@ -22,10 +22,9 @@ """ import diamond.collector +import diamond.pycompat +from diamond.pycompat import quote, Request, urljoin import re -from urlparse import urljoin -from urllib import quote -import urllib2 from base64 import b64encode try: @@ -47,9 +46,9 @@ def __init__(self, log, host, user, password, timeout=5, scheme="http"): def do_call(self, path): url = urljoin(self.base_url, path) - req = urllib2.Request(url) + req = Request(url) req.add_header('Authorization', self._authorization) - return json.load(urllib2.urlopen(req, timeout=self.timeout)) + return json.load(diamond.pycompat.urlopen(req, timeout=self.timeout)) def get_all_vhosts(self): return self.do_call('vhosts') diff --git a/src/collectors/resqueweb/resqueweb.py b/src/collectors/resqueweb/resqueweb.py index 1b7606e20..683a2f8c0 100644 --- a/src/collectors/resqueweb/resqueweb.py +++ b/src/collectors/resqueweb/resqueweb.py @@ -2,15 +2,10 @@ """ Collects data for Resque Web - -#### Dependencies - - * urllib2 - """ -import urllib2 import diamond.collector +import diamond.pycompat class ResqueWebCollector(diamond.collector.Collector): @@ -35,7 +30,7 @@ def get_default_config(self): def collect(self): try: - response = urllib2.urlopen("http://%s:%s/stats.txt" % ( + response = diamond.pycompat.urlopen("http://%s:%s/stats.txt" % ( self.config['host'], int(self.config['port']))) except Exception as e: self.log.error('Couldnt connect to resque-web: %s', e) diff --git a/src/collectors/resqueweb/test/testresqueweb.py b/src/collectors/resqueweb/test/testresqueweb.py index 7baa91046..7b37eb584 100644 --- a/src/collectors/resqueweb/test/testresqueweb.py +++ b/src/collectors/resqueweb/test/testresqueweb.py @@ -9,6 +9,7 @@ from test import patch from diamond.collector import Collector +from diamond.pycompat import URLOPEN from resqueweb import ResqueWebCollector ########################################################################## @@ -28,7 +29,7 @@ def test_import(self): @patch.object(Collector, 'publish') def test_should_work_with_real_data(self, publish_mock): - patch_urlopen = patch('urllib2.urlopen', Mock( + patch_urlopen = patch(URLOPEN, Mock( return_value=self.getFixture('stats.txt'))) patch_urlopen.start() @@ -54,7 +55,7 @@ def test_should_work_with_real_data(self, publish_mock): @patch.object(Collector, 'publish') def test_should_fail_gracefully(self, publish_mock): - patch_urlopen = patch('urllib2.urlopen', Mock( + patch_urlopen = patch(URLOPEN, Mock( return_value=self.getFixture('stats_blank.txt'))) patch_urlopen.start() diff --git a/src/collectors/sidekiqweb/sidekiqweb.py b/src/collectors/sidekiqweb/sidekiqweb.py index 52ef9be1b..e0ed630ed 100644 --- a/src/collectors/sidekiqweb/sidekiqweb.py +++ b/src/collectors/sidekiqweb/sidekiqweb.py @@ -5,7 +5,6 @@ #### Dependencies - * urllib2 * json (or simeplejson) """ @@ -15,8 +14,8 @@ except ImportError: import simplejson as json -import urllib2 import diamond.collector +import diamond.pycompat class SidekiqWebCollector(diamond.collector.Collector): @@ -42,7 +41,7 @@ def get_default_config(self): def collect(self): try: - response = urllib2.urlopen("http://%s:%s/dashboard/stats" % ( + response = diamond.pycompat.urlopen("http://%s:%s/dashboard/stats" % ( self.config['host'], int(self.config['port']))) except Exception as e: self.log.error('Couldnt connect to sidekiq-web: %s', e) diff --git a/src/collectors/sidekiqweb/test/testsidekiqweb.py b/src/collectors/sidekiqweb/test/testsidekiqweb.py index 4aff95486..f540f28df 100644 --- a/src/collectors/sidekiqweb/test/testsidekiqweb.py +++ b/src/collectors/sidekiqweb/test/testsidekiqweb.py @@ -9,6 +9,7 @@ from test import patch from diamond.collector import Collector +from diamond.pycompat import URLOPEN from sidekiqweb import SidekiqWebCollector ########################################################################## @@ -28,7 +29,7 @@ def test_import(self): @patch.object(Collector, 'publish') def test_should_work_with_real_data(self, publish_mock): - patch_urlopen = patch('urllib2.urlopen', Mock( + patch_urlopen = patch(URLOPEN, Mock( return_value=self.getFixture('stats'))) patch_urlopen.start() @@ -53,7 +54,7 @@ def test_should_work_with_real_data(self, publish_mock): @patch.object(Collector, 'publish') def test_should_fail_gracefully(self, publish_mock): - patch_urlopen = patch('urllib2.urlopen', Mock( + patch_urlopen = patch(URLOPEN, Mock( return_value=self.getFixture('stats_blank'))) patch_urlopen.start() diff --git a/src/collectors/solr/solr.py b/src/collectors/solr/solr.py index 1c247fee6..cf127e35b 100644 --- a/src/collectors/solr/solr.py +++ b/src/collectors/solr/solr.py @@ -6,13 +6,11 @@ #### Dependencies * posixpath - * urllib2 * json """ import posixpath -import urllib2 try: import json @@ -20,6 +18,7 @@ import simplejson as json import diamond.collector +import diamond.pycompat class SolrCollector(diamond.collector.Collector): @@ -75,7 +74,7 @@ def _get(self, path): url = 'http://%s:%i/%s' % ( self.config['host'], int(self.config['port']), path) try: - response = urllib2.urlopen(url) + response = diamond.pycompat.urlopen(url) except Exception as err: self.log.error("%s: %s", url, err) return False diff --git a/src/collectors/solr/test/testsolr.py b/src/collectors/solr/test/testsolr.py index 0f31f01a0..03e8de206 100644 --- a/src/collectors/solr/test/testsolr.py +++ b/src/collectors/solr/test/testsolr.py @@ -9,7 +9,7 @@ from test import patch from diamond.collector import Collector - +from diamond.pycompat import URLOPEN from solr import SolrCollector ########################################################################## @@ -31,7 +31,8 @@ def test_should_work_with_real_data(self, publish_mock, urlopen_mock): self.getFixture('ping'), self.getFixture('stats'), self.getFixture('system')] - urlopen_mock.side_effect = lambda *args: returns.pop(0) + urlopen_mock = patch(URLOPEN, Mock( + side_effect=lambda *args: returns.pop(0))) self.collector.collect() @@ -140,6 +141,8 @@ def test_should_work_with_real_data(self, publish_mock, urlopen_mock): @patch.object(Collector, 'publish') def test_should_fail_gracefully(self, publish_mock, urlopen_mock): urlopen_mock.return_value = self.getFixture('stats_blank') + urlopen_mock = patch(URLOPEN, Mock( + return_value=self.getFixture('stats_blank'))) self.collector.collect() diff --git a/src/collectors/websitemonitor/test/testwebsitemonitor.py b/src/collectors/websitemonitor/test/testwebsitemonitor.py index 0dfdf1cfb..b3db9efb1 100644 --- a/src/collectors/websitemonitor/test/testwebsitemonitor.py +++ b/src/collectors/websitemonitor/test/testwebsitemonitor.py @@ -7,6 +7,7 @@ from test import patch from diamond.collector import Collector +from diamond.pycompat import URLOPEN from websitemonitor import WebsiteMonitorCollector @@ -38,7 +39,7 @@ def setUp(self, config=None): self.collector = WebsiteMonitorCollector(config, None) - self.patcher = patch('urllib2.urlopen') + self.patcher = patch(URLOPEN) self.urlopen_mock = self.patcher.start() def test_import(self): diff --git a/src/collectors/websitemonitor/websitemonitor.py b/src/collectors/websitemonitor/websitemonitor.py index af0723469..f002f87da 100644 --- a/src/collectors/websitemonitor/websitemonitor.py +++ b/src/collectors/websitemonitor/websitemonitor.py @@ -1,17 +1,12 @@ # coding=utf-8 """ Gather HTTP Response code and Duration of HTTP request - -#### Dependencies - * urllib2 - """ - -import urllib2 import time from datetime import datetime import diamond.collector - +import diamond.pycompat +from diamond.pycompat import URLError class WebsiteMonitorCollector(diamond.collector.Collector): """ @@ -38,8 +33,6 @@ def get_default_config(self): return default_config def collect(self): - req = urllib2.Request('%s' % (self.config['URL'])) - try: # time in seconds since epoch as a floating number start_time = time.time() @@ -48,7 +41,7 @@ def collect(self): ).strftime('%B %d, %Y %H:%M:%S') self.log.debug('Start time: %s' % (st)) - resp = urllib2.urlopen(req) + resp = diamond.pycompat.urlopen(self.config['URL']) # time in seconds since epoch as a floating number end_time = time.time() # human-readable end time e.eg. November 25, 2013 18:15:56 @@ -59,8 +52,8 @@ def collect(self): # Publish metrics self.publish('response_time.%s' % (resp.code), rt, metric_type='COUNTER') - # urllib2 will puke on non HTTP 200/OK URLs - except urllib2.URLError as e: + # urllib will puke on non HTTP 200/OK URLs + except URLError as e: if e.code != 200: # time in seconds since epoch as a floating number end_time = time.time() diff --git a/src/diamond/handler/httpHandler.py b/src/diamond/handler/httpHandler.py index f3bfefbe5..e591bdbad 100755 --- a/src/diamond/handler/httpHandler.py +++ b/src/diamond/handler/httpHandler.py @@ -3,10 +3,18 @@ """ Send metrics to a http endpoint via POST + +#### Configuration +Enable this handler + + * handlers = diamond.handler.httpHandler.HttpPostHandler + + * url = http://www.example.com/endpoint + """ from Handler import Handler -import urllib2 +from . pycompat import Request, urlopen class HttpPostHandler(Handler): @@ -56,6 +64,6 @@ def flush(self): self.post() def post(self): - req = urllib2.Request(self.url, "\n".join(self.metrics)) - urllib2.urlopen(req) + req = Request(self.url, "\n".join(self.metrics)) + urlopen(req) self.metrics = [] diff --git a/src/diamond/handler/logentries_diamond.py b/src/diamond/handler/logentries_diamond.py index 59cd81593..a33e564b6 100644 --- a/src/diamond/handler/logentries_diamond.py +++ b/src/diamond/handler/logentries_diamond.py @@ -7,10 +7,9 @@ from Handler import Handler import logging -import urllib2 import json from collections import deque - +from diamond.pycompat import URLError, urlopen, Request class LogentriesDiamondHandler(Handler): """ @@ -75,9 +74,9 @@ def _send(self): metric = self.queue.popleft() topic, value, timestamp = str(metric).split() msg = json.dumps({"event": {topic: value}}) - req = urllib2.Request("https://js.logentries.com/v1/logs/" + - self.log_token, msg) + req = Request("https://js.logentries.com/v1/logs/" + + self.log_token, msg) try: - urllib2.urlopen(req) - except urllib2.URLError as e: + urlopen(req) + except URLError as e: logging.error("Can't send log message to Logentries %s", e) diff --git a/src/diamond/handler/signalfx.py b/src/diamond/handler/signalfx.py index d1d7eb787..4d2d634ec 100644 --- a/src/diamond/handler/signalfx.py +++ b/src/diamond/handler/signalfx.py @@ -3,11 +3,6 @@ """ Send metrics to signalfx -#### Dependencies - - * urllib2 - - #### Configuration Enable this handler @@ -20,10 +15,10 @@ from Handler import Handler from diamond.util import get_diamond_version +from diamond.pycompat import URLError, urlopen, Request import json import logging import time -import urllib2 class SignalfxHandler(Handler): @@ -127,13 +122,13 @@ def _send(self): self.metrics = [] postBody = json.dumps(postDictionary) logging.debug("Body is %s", postBody) - req = urllib2.Request(self.url, postBody, - {"Content-type": "application/json", - "X-SF-TOKEN": self.auth_token, - "User-Agent": self.user_agent()}) + req = Request(self.url, postBody, + {"Content-type": "application/json", + "X-SF-TOKEN": self.auth_token, + "User-Agent": self.user_agent()}) self.resetBatchTimeout() try: - urllib2.urlopen(req) - except urllib2.URLError: + urlopen(req) + except URLError: logging.exception("Unable to post signalfx metrics") return diff --git a/src/diamond/handler/test/teststatsdhandler.py b/src/diamond/handler/test/teststatsdhandler.py index 701188f21..859c1c307 100644 --- a/src/diamond/handler/test/teststatsdhandler.py +++ b/src/diamond/handler/test/teststatsdhandler.py @@ -5,7 +5,7 @@ from test import unittest from test import run_only from test import patch -from mock import ANY +from test import ANY import configobj diff --git a/test.py b/test.py index cc18faf95..f3f735f45 100755 --- a/test.py +++ b/test.py @@ -35,9 +35,9 @@ setproctitle = None try: - from mock import call, MagicMock, Mock, patch + from mock import ANY, call, MagicMock, Mock, mock_open, patch except ImportError: - from unittest.mock import call, MagicMock, Mock, patch + from unittest.mock import ANY, call, MagicMock, Mock, mock_open, patch try: # py3k way import builtins From 00a73407f57eb4624361494c917f022d8f2d65e6 Mon Sep 17 00:00:00 2001 From: Hannu Valtonen Date: Mon, 24 Aug 2015 17:44:32 +0300 Subject: [PATCH 16/33] py3k: Fix cpickle import to have a fallback --- src/collectors/elb/elb.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/collectors/elb/elb.py b/src/collectors/elb/elb.py index 61bb0a7cf..e4bac6820 100644 --- a/src/collectors/elb/elb.py +++ b/src/collectors/elb/elb.py @@ -39,7 +39,11 @@ """ import calendar -import cPickle +try: + import cPickle as pickle +except ImportError: + import pickle + import datetime import functools import re @@ -76,7 +80,7 @@ def __init__(self, func): def __call__(self, *args, **kwargs): # If the function args cannot be used as a cache hash key, fail fast - key = cPickle.dumps((args, kwargs)) + key = pickle.dumps((args, kwargs)) try: return self.cache[key] except KeyError: From 23a7971e57de5e5edd45e47dc7f3a39a6283ba75 Mon Sep 17 00:00:00 2001 From: Hannu Valtonen Date: Mon, 24 Aug 2015 17:50:43 +0300 Subject: [PATCH 17/33] py3k: import Queue related functionality from pycompat --- src/collectors/jcollectd/jcollectd.py | 8 ++++---- src/diamond/handler/rrdtool.py | 7 +++---- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/collectors/jcollectd/jcollectd.py b/src/collectors/jcollectd/jcollectd.py index b651c25ac..958cd5aa9 100644 --- a/src/collectors/jcollectd/jcollectd.py +++ b/src/collectors/jcollectd/jcollectd.py @@ -25,10 +25,10 @@ import threading import re -import Queue import diamond.collector import diamond.metric +from diamond.pycompat import Empty, Full, Queue import collectd_network @@ -63,7 +63,7 @@ def collect(self): try: dp = q.get(False) metric = self.make_metric(dp) - except Queue.Empty: + except Empty: break self.publish_metric(metric) @@ -118,7 +118,7 @@ def __init__(self, host, port, log, poll_interval=0.4): self.log = log self.poll_interval = poll_interval - self.queue = Queue.Queue() + self.queue = Queue() def run(self): self.log.info('ListenerThread started on {0}:{1}(udp)'.format( @@ -147,7 +147,7 @@ def send_to_collector(self, items): try: metric = self.transform(item) self.queue.put(metric) - except Queue.Full: + except Full: self.log.error('Queue to collector is FULL') except Exception as e: self.log.error('B00M! type={0}, exception={1}'.format(type(e), diff --git a/src/diamond/handler/rrdtool.py b/src/diamond/handler/rrdtool.py index 032b18166..02cecc60d 100644 --- a/src/diamond/handler/rrdtool.py +++ b/src/diamond/handler/rrdtool.py @@ -7,8 +7,7 @@ import os import re import subprocess -import Queue - +from diamond.pycompat import Queue, Empty from Handler import Handler # @@ -160,7 +159,7 @@ def process(self, metric): def _queue(self, filename, timestamp, value): if filename not in self._queues: - queue = Queue.Queue() + queue = Queue() self._queues[filename] = queue else: queue = self._queues[filename] @@ -199,7 +198,7 @@ def _flush_queue(self, filename): if timestamp not in updates: updates[timestamp] = [] updates[timestamp].append(value) - except Queue.Empty: + except Empty: break # Save the last update time. From b4186e6e51059ff2c8c6aa1eca5390405becee1f Mon Sep 17 00:00:00 2001 From: Hannu Valtonen Date: Mon, 24 Aug 2015 21:33:05 +0300 Subject: [PATCH 18/33] py3k: Move http client to a file with a different name to unmask the import on py3k The name needs to be different from the package name so the test can import the correct name. --- src/collectors/http/{http.py => http_collector.py} | 0 src/collectors/http/test/testhttp.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename src/collectors/http/{http.py => http_collector.py} (100%) diff --git a/src/collectors/http/http.py b/src/collectors/http/http_collector.py similarity index 100% rename from src/collectors/http/http.py rename to src/collectors/http/http_collector.py diff --git a/src/collectors/http/test/testhttp.py b/src/collectors/http/test/testhttp.py index e393e8351..4af4b97ab 100644 --- a/src/collectors/http/test/testhttp.py +++ b/src/collectors/http/test/testhttp.py @@ -11,7 +11,7 @@ from diamond.collector import Collector from diamond.pycompat import URLOPEN -from http import HttpCollector +from http_collector import HttpCollector ########################################################################## From b01d6e30b5bd86bcc858806a43a289f17d694ecd Mon Sep 17 00:00:00 2001 From: Hannu Valtonen Date: Tue, 25 Aug 2015 01:18:27 +0300 Subject: [PATCH 19/33] py3k: Change xmlrpclib imports to work with both py3k and 2.x --- src/collectors/supervisord/supervisord.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/collectors/supervisord/supervisord.py b/src/collectors/supervisord/supervisord.py index c1e56dfae..e48385ebe 100644 --- a/src/collectors/supervisord/supervisord.py +++ b/src/collectors/supervisord/supervisord.py @@ -27,7 +27,10 @@ """ -import xmlrpclib +try: + from xmlrpclib import Server, ServerProxy +except ImportError: + from xmlrpc.client import Server, ServerProxy try: import supervisor.xmlrpc @@ -66,13 +69,13 @@ def getAllProcessInfo(self): 'Attempting to connect to XML-RPC server "%s"', uri) if protocol == 'unix': - server = xmlrpclib.ServerProxy( + server = ServerProxy( 'http://127.0.0.1', supervisor.xmlrpc.SupervisorTransport(None, None, uri) ).supervisor elif protocol == 'http': - server = xmlrpclib.Server(uri).supervisor + server = Server(uri).supervisor else: self.log.debug( From fba635ec631ca04cfd3d4fc8b74d195f99f73f38 Mon Sep 17 00:00:00 2001 From: Hannu Valtonen Date: Tue, 25 Aug 2015 01:18:39 +0300 Subject: [PATCH 20/33] py3k: Since py3k no longer has a long type, mock int as its replacement This is done so we can keep the existing long() typecasts in place, though it's unclear if they're still needed. This commit just replaces the existing logic without taking a stance on that. --- src/collectors/interrupt/interrupt.py | 1 + src/collectors/interrupt/soft.py | 1 + src/collectors/ip/ip.py | 1 + src/collectors/kafkastat/test/testkafka.py | 2 +- src/collectors/network/network.py | 1 + src/collectors/nfs/nfs.py | 1 + src/collectors/nfsd/nfsd.py | 1 + src/collectors/openvpn/openvpn.py | 2 +- src/collectors/proc/proc.py | 2 +- src/collectors/tcp/tcp.py | 1 + src/collectors/udp/udp.py | 1 + src/collectors/xen_collector/test/testxen.py | 10 +++++----- src/diamond/metric.py | 1 + 13 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/collectors/interrupt/interrupt.py b/src/collectors/interrupt/interrupt.py index ccf52f8b0..cf1ae609d 100644 --- a/src/collectors/interrupt/interrupt.py +++ b/src/collectors/interrupt/interrupt.py @@ -13,6 +13,7 @@ import platform import os import diamond.collector +from diamond.pycompat import long # Detect the architecture of the system # and set the counters for MAX_VALUES diff --git a/src/collectors/interrupt/soft.py b/src/collectors/interrupt/soft.py index 31626fc88..1cfe8e1b6 100644 --- a/src/collectors/interrupt/soft.py +++ b/src/collectors/interrupt/soft.py @@ -13,6 +13,7 @@ import platform import os import diamond.collector +from diamond.pycompat import long # Detect the architecture of the system # and set the counters for MAX_VALUES diff --git a/src/collectors/ip/ip.py b/src/collectors/ip/ip.py index 3e70228d7..70ccf77ce 100644 --- a/src/collectors/ip/ip.py +++ b/src/collectors/ip/ip.py @@ -24,6 +24,7 @@ """ import diamond.collector +from diamond.pycompat import long import os diff --git a/src/collectors/kafkastat/test/testkafka.py b/src/collectors/kafkastat/test/testkafka.py index 55293d274..d78b31292 100644 --- a/src/collectors/kafkastat/test/testkafka.py +++ b/src/collectors/kafkastat/test/testkafka.py @@ -13,7 +13,7 @@ from test import patch from diamond.collector import Collector -from diamond.pycompat import URLError, URLOPEN +from diamond.pycompat import long, URLError, URLOPEN from kafkastat import KafkaCollector ########## diff --git a/src/collectors/network/network.py b/src/collectors/network/network.py index 366f5e070..f2ef034b8 100644 --- a/src/collectors/network/network.py +++ b/src/collectors/network/network.py @@ -13,6 +13,7 @@ import diamond.collector from diamond.collector import str_to_bool import diamond.convertor +from diamond.pycompat import long import os import re diff --git a/src/collectors/nfs/nfs.py b/src/collectors/nfs/nfs.py index ac8f12ad2..49948aa05 100644 --- a/src/collectors/nfs/nfs.py +++ b/src/collectors/nfs/nfs.py @@ -11,6 +11,7 @@ import diamond.collector import os +from diamond.pycompat import long class NfsCollector(diamond.collector.Collector): diff --git a/src/collectors/nfsd/nfsd.py b/src/collectors/nfsd/nfsd.py index 63d4b26f3..7382fa021 100644 --- a/src/collectors/nfsd/nfsd.py +++ b/src/collectors/nfsd/nfsd.py @@ -10,6 +10,7 @@ """ import diamond.collector +from diamond.pycompat import long import os diff --git a/src/collectors/openvpn/openvpn.py b/src/collectors/openvpn/openvpn.py index 5a4338158..1b0ee92a3 100644 --- a/src/collectors/openvpn/openvpn.py +++ b/src/collectors/openvpn/openvpn.py @@ -33,7 +33,7 @@ import diamond.collector import os.path import time -from diamond.pycompat import urlparse +from diamond.pycompat import long, urlparse class OpenVPNCollector(diamond.collector.Collector): diff --git a/src/collectors/proc/proc.py b/src/collectors/proc/proc.py index 4ee9444ba..d224efda6 100644 --- a/src/collectors/proc/proc.py +++ b/src/collectors/proc/proc.py @@ -13,7 +13,7 @@ import platform import os import diamond.collector - +from diamond.pycompat import long # Detect the architecture of the system # and set the counters for MAX_VALUES # appropriately. Otherwise, rolling over diff --git a/src/collectors/tcp/tcp.py b/src/collectors/tcp/tcp.py index 18b664c4b..c7a86a9c1 100644 --- a/src/collectors/tcp/tcp.py +++ b/src/collectors/tcp/tcp.py @@ -177,6 +177,7 @@ """ import diamond.collector +from diamond.pycompat import long import os diff --git a/src/collectors/udp/udp.py b/src/collectors/udp/udp.py index d9872fc15..6b73b075d 100644 --- a/src/collectors/udp/udp.py +++ b/src/collectors/udp/udp.py @@ -10,6 +10,7 @@ """ import diamond.collector +from diamond.pycompat import long import os diff --git a/src/collectors/xen_collector/test/testxen.py b/src/collectors/xen_collector/test/testxen.py index 4e5ce98f9..65ce5bfe1 100644 --- a/src/collectors/xen_collector/test/testxen.py +++ b/src/collectors/xen_collector/test/testxen.py @@ -48,15 +48,15 @@ def __init__(self, id): def info(self): if self.id == 0: - return [1, 49420888L, 49420888L, 8, 911232000000000L] + return [1, 49420888, 49420888, 8, 911232000000000] if self.id == 1: - return [1, 2097152L, 2097152L, 2, 310676150000000L] + return [1, 2097152, 2097152, 2, 310676150000000] if self.id == 2: - return [1, 2097152L, 2097152L, 2, 100375300000000L] + return [1, 2097152, 2097152, 2, 100375300000000] if self.id == 3: - return [1, 10485760L, 10485760L, 2, 335312040000000L] + return [1, 10485760, 10485760, 2, 335312040000000] if self.id == 4: - return [1, 10485760L, 10485760L, 2, 351313480000000L] + return [1, 10485760, 10485760, 2, 351313480000000] libvirt_m = Mock() libvirt_m.getInfo.return_value = ['x86_64', 48262, 8, 1200, 2, 1, 4, 1] diff --git a/src/diamond/metric.py b/src/diamond/metric.py index 8570b7cca..91b361387 100644 --- a/src/diamond/metric.py +++ b/src/diamond/metric.py @@ -4,6 +4,7 @@ import re import logging from . error import DiamondException +from diamond.pycompat import long class Metric(object): From cbd98e3a20c50ec481479e5c27cf3a0aa8dff9dc Mon Sep 17 00:00:00 2001 From: Hannu Valtonen Date: Mon, 24 Aug 2015 18:30:56 +0300 Subject: [PATCH 21/33] py3k: subprocess returns bytes not strings on py3k, convert to unicode We always assume a utf8 encoding which is the case almost everywhere these days, and even in the places where it's not, the command line tools called actually mostly return just ASCII test which converts correctly anyway. --- src/collectors/amavis/amavis.py | 2 ++ src/collectors/ceph/ceph.py | 2 ++ src/collectors/cephstats/cephstats.py | 2 ++ src/collectors/chronyd/chronyd.py | 2 ++ src/collectors/disktemp/disktemp.py | 2 ++ src/collectors/ipvs/ipvs.py | 10 ++++++++-- src/collectors/jbossapi/jbossapi.py | 2 ++ src/collectors/kafkastat/test/testkafka.py | 2 +- src/collectors/mountstats/mountstats.py | 5 ++++- src/collectors/nagios/nagios.py | 5 ++++- src/collectors/ntpd/ntpd.py | 4 +++- src/collectors/numa/numa.py | 4 ++-- src/collectors/ossec/ossec.py | 2 ++ src/collectors/passenger_stats/passenger_stats.py | 8 ++++++-- src/collectors/postqueue/postqueue.py | 7 +++++-- src/collectors/powerdns/powerdns.py | 2 ++ src/collectors/sidekiqweb/sidekiqweb.py | 2 +- src/collectors/smart/smart.py | 5 ++++- src/collectors/ups/ups.py | 2 ++ src/collectors/userscripts/userscripts.py | 8 +++++--- src/collectors/varnish/varnish.py | 2 ++ src/collectors/zookeeper/zookeeper.py | 6 +++--- src/diamond/collector.py | 14 +++++++++++--- test.py | 2 +- 24 files changed, 78 insertions(+), 24 deletions(-) diff --git a/src/collectors/amavis/amavis.py b/src/collectors/amavis/amavis.py index 8a245a328..aa69b9284 100644 --- a/src/collectors/amavis/amavis.py +++ b/src/collectors/amavis/amavis.py @@ -74,6 +74,8 @@ def collect(self): cmdline = [self.config['amavisd_exe'], '-c', '1'] agent = subprocess.Popen(cmdline, stdout=subprocess.PIPE) agent_out = agent.communicate()[0] + if isinstance(agent_out, bytes): + agent_out = agent_out.decode("utf8") lines = agent_out.strip().split(os.linesep) for line in lines: for rex in self.matchers: diff --git a/src/collectors/ceph/ceph.py b/src/collectors/ceph/ceph.py index 6dff5cd9f..127808583 100644 --- a/src/collectors/ceph/ceph.py +++ b/src/collectors/ceph/ceph.py @@ -108,6 +108,8 @@ def _get_stats_from_socket(self, name): 'perf', 'dump', ]) + if isinstance(json_blob, bytes): + json_blob = json_blob.decode("utf8") except subprocess.CalledProcessError as err: self.log.info('Could not get stats from %s: %s', name, err) diff --git a/src/collectors/cephstats/cephstats.py b/src/collectors/cephstats/cephstats.py index 2d31dfd60..bb1c82684 100644 --- a/src/collectors/cephstats/cephstats.py +++ b/src/collectors/cephstats/cephstats.py @@ -68,6 +68,8 @@ def _get_stats(self): """ try: output = subprocess.check_output(['ceph', '-s']) + if isinstance(output, bytes): + output = output.decode("utf8") except subprocess.CalledProcessError as err: self.log.info( 'Could not get stats: %s' % err) diff --git a/src/collectors/chronyd/chronyd.py b/src/collectors/chronyd/chronyd.py index cb4e4a086..fb7c7154c 100644 --- a/src/collectors/chronyd/chronyd.py +++ b/src/collectors/chronyd/chronyd.py @@ -66,6 +66,8 @@ def get_output(self): def collect(self): output = self.get_output() + if isinstance(output, bytes): + output = output.decode("utf8") for line in output.strip().split("\n"): m = LINE_PATTERN.search(line) diff --git a/src/collectors/disktemp/disktemp.py b/src/collectors/disktemp/disktemp.py index ff900bda0..f29d68918 100644 --- a/src/collectors/disktemp/disktemp.py +++ b/src/collectors/disktemp/disktemp.py @@ -90,6 +90,8 @@ def collect(self): metrics = {} for device, p in instances.items(): output = p.communicate()[0].strip() + if isinstance(output, bytes): + output = output.decode("utf8") try: metrics[device + ".Temperature"] = float(output) diff --git a/src/collectors/ipvs/ipvs.py b/src/collectors/ipvs/ipvs.py index fd4961faf..924118ad0 100644 --- a/src/collectors/ipvs/ipvs.py +++ b/src/collectors/ipvs/ipvs.py @@ -75,7 +75,10 @@ def collect(self): lambda a: a != '--exact', self.statcommand) p = subprocess.Popen(self.statcommand, - stdout=subprocess.PIPE).communicate()[0][:-1] + stdout=subprocess.PIPE).communicate()[0] + if isinstance(p, bytes): + p = p.decode("utf8") + p = p[:-1] columns = { 'conns': 2, @@ -117,7 +120,10 @@ def collect(self): self.publish(metric_name, metric_value) p = subprocess.Popen(self.concommand, - stdout=subprocess.PIPE).communicate()[0][:-1] + stdout=subprocess.PIPE).communicate()[0] + if isinstance(p, bytes): + p = p.decode("utf8") + p = p[:-1] columns = { 'active': 4, diff --git a/src/collectors/jbossapi/jbossapi.py b/src/collectors/jbossapi/jbossapi.py index f3646ae0b..89c22e596 100644 --- a/src/collectors/jbossapi/jbossapi.py +++ b/src/collectors/jbossapi/jbossapi.py @@ -356,6 +356,8 @@ def get_data(self, op_type, current_host, current_port, current_proto, attributes = subprocess.Popen(the_cmd, shell=True, stdout=subprocess.PIPE ).communicate()[0] + if isinstance(attributes, bytes): + attributes = attributes.decode("utf8") output = json.loads(attributes) except Exception as e: self.log.error("JbossApiCollector: There was an exception %s", e) diff --git a/src/collectors/kafkastat/test/testkafka.py b/src/collectors/kafkastat/test/testkafka.py index d78b31292..5c05dbea2 100644 --- a/src/collectors/kafkastat/test/testkafka.py +++ b/src/collectors/kafkastat/test/testkafka.py @@ -53,7 +53,7 @@ def test_get(self, urlopen_mock): result = self.collector._get('/path') result_string = ElementTree.tostring(result) - self.assertEqual(result_string, '') + self.assertEqual(result_string, b'') @run_only_if_ElementTree_is_available @patch(URLOPEN) diff --git a/src/collectors/mountstats/mountstats.py b/src/collectors/mountstats/mountstats.py index f66503c60..802e9c6f1 100755 --- a/src/collectors/mountstats/mountstats.py +++ b/src/collectors/mountstats/mountstats.py @@ -120,7 +120,10 @@ def collect(self): command = [self.config['sudo_cmd'], '/bin/cat', self.MOUNTSTATS] p = subprocess.Popen(command, - stdout=subprocess.PIPE).communicate()[0][:-1] + stdout=subprocess.PIPE).communicate()[0] + if isinstance(p, bytes): + p = p.decode("utf8") + p = p[:-1] lines = p.split("\n") else: diff --git a/src/collectors/nagios/nagios.py b/src/collectors/nagios/nagios.py index 61dcdc9eb..90371bba8 100644 --- a/src/collectors/nagios/nagios.py +++ b/src/collectors/nagios/nagios.py @@ -82,7 +82,10 @@ def collect(self): command.insert(0, self.config['sudo_cmd']) p = subprocess.Popen(command, - stdout=subprocess.PIPE).communicate()[0][:-1] + stdout=subprocess.PIPE).communicate()[0] + if isinstance(p, bytes): + p = p.decode("utf8") + p = p[:-1] for i, v in enumerate(p.split("\n")): metric_name = self.config['vars'][i] diff --git a/src/collectors/ntpd/ntpd.py b/src/collectors/ntpd/ntpd.py index bd66b4944..0d86a2ec1 100644 --- a/src/collectors/ntpd/ntpd.py +++ b/src/collectors/ntpd/ntpd.py @@ -46,8 +46,10 @@ def run_command(self, command): if str_to_bool(self.config['use_sudo']): command.insert(0, self.config['sudo_cmd']) - return subprocess.Popen(command, + stdout = subprocess.Popen(command, stdout=subprocess.PIPE).communicate()[0] + if isinstance(stdout, bytes): + stdout = stdout.decode("utf8") except OSError: self.log.exception("Unable to run %s", command) return "" diff --git a/src/collectors/numa/numa.py b/src/collectors/numa/numa.py index 0ffeeadba..9e58d441b 100644 --- a/src/collectors/numa/numa.py +++ b/src/collectors/numa/numa.py @@ -31,9 +31,9 @@ def get_default_config(self): def collect(self): p = Popen([self.config['bin'], '--hardware'], stdout=PIPE, stderr=PIPE) - output, errors = p.communicate() - + if isinstance(output, bytes): + output = output.decode("utf8") lines = output.split('\n') for line in lines: try: diff --git a/src/collectors/ossec/ossec.py b/src/collectors/ossec/ossec.py index 9c9da1cef..09d078260 100644 --- a/src/collectors/ossec/ossec.py +++ b/src/collectors/ossec/ossec.py @@ -54,6 +54,8 @@ def collect(self): try: p = subprocess.Popen(command, stdout=subprocess.PIPE) res = p.communicate()[0] + if isinstance(res, bytes): + res = res.decode("utf8") except Exception as e: self.log.error('Unable to exec cmd: %s, because %s' % (' '.join(command), str(e))) diff --git a/src/collectors/passenger_stats/passenger_stats.py b/src/collectors/passenger_stats/passenger_stats.py index 82ba557ed..52d0a4114 100644 --- a/src/collectors/passenger_stats/passenger_stats.py +++ b/src/collectors/passenger_stats/passenger_stats.py @@ -77,7 +77,9 @@ def get_passenger_memory_stats(self): try: proc1 = subprocess.Popen(command, stdout=subprocess.PIPE) - (std_out, std_err) = proc1.communicate() + std_out = proc1.communicate()[0] + if isinstance(std_out, bytes): + std_out = std_out.decode("utf8") except OSError: return {} @@ -135,7 +137,9 @@ def get_passenger_cpu_usage(self, dict_stats): proc1 = subprocess.Popen( ["top", "-b", "-n", "2"], stdout=subprocess.PIPE) - (std_out, std_err) = proc1.communicate() + std_out = proc1.communicate()[0] + if isinstance(std_out, bytes): + std_out = std_out.decode("utf8") except OSError: return (-1) diff --git a/src/collectors/postqueue/postqueue.py b/src/collectors/postqueue/postqueue.py index ab1a8fc78..f133185bb 100644 --- a/src/collectors/postqueue/postqueue.py +++ b/src/collectors/postqueue/postqueue.py @@ -45,8 +45,11 @@ def get_postqueue_output(self): if str_to_bool(self.config['use_sudo']): command.insert(0, self.config['sudo_cmd']) - return subprocess.Popen(command, - stdout=subprocess.PIPE).communicate()[0] + stdout = subprocess.Popen(command, + stdout=subprocess.PIPE).communicate()[0] + if isinstance(stdout, bytes): + stdout = stdout.decode("utf8") + return stdout except OSError: return "" diff --git a/src/collectors/powerdns/powerdns.py b/src/collectors/powerdns/powerdns.py index af7991a1f..a10cb639a 100644 --- a/src/collectors/powerdns/powerdns.py +++ b/src/collectors/powerdns/powerdns.py @@ -60,6 +60,8 @@ def collect(self): data = subprocess.Popen(command, stdout=subprocess.PIPE).communicate()[0] + if isinstance(data, bytes): + data = data.decode("utf8") for metric in data.split(','): if not metric.strip(): diff --git a/src/collectors/sidekiqweb/sidekiqweb.py b/src/collectors/sidekiqweb/sidekiqweb.py index e0ed630ed..b0ca41ca9 100644 --- a/src/collectors/sidekiqweb/sidekiqweb.py +++ b/src/collectors/sidekiqweb/sidekiqweb.py @@ -56,7 +56,7 @@ def collect(self): for k in j: for item, value in j[k].items(): - if isinstance(value, (str, unicode)) and 'M' in value: + if isinstance(value, (bytes, diamond.pycompat.unicode)) and 'M' in value: value = float(value.replace('M', '')) for unit in self.config['byte_unit']: unit_value = diamond.convertor.binary.convert( diff --git a/src/collectors/smart/smart.py b/src/collectors/smart/smart.py index 6df3a28f7..b3624ca03 100644 --- a/src/collectors/smart/smart.py +++ b/src/collectors/smart/smart.py @@ -60,7 +60,10 @@ def collect(self): attributes = subprocess.Popen( command, stdout=subprocess.PIPE - ).communicate()[0].strip().splitlines() + ).communicate()[0] + if isinstance(attributes, bytes): + attributes = attributes.decode("utf8") + attributes = attributes.strip().splitlines() metrics = {} diff --git a/src/collectors/ups/ups.py b/src/collectors/ups/ups.py index 9aa8e0024..eda46a370 100644 --- a/src/collectors/ups/ups.py +++ b/src/collectors/ups/ups.py @@ -54,6 +54,8 @@ def collect(self): p = subprocess.Popen(command, stdout=subprocess.PIPE).communicate()[0] + if isinstance(p, bytes): + p = p.decode("utf8") for ln in p.strip().splitlines(): datapoint = ln.split(": ") diff --git a/src/collectors/userscripts/userscripts.py b/src/collectors/userscripts/userscripts.py index 4e2174d02..c1ce5c688 100644 --- a/src/collectors/userscripts/userscripts.py +++ b/src/collectors/userscripts/userscripts.py @@ -70,7 +70,7 @@ def collect(self): proc = subprocess.Popen([absolutescriptpath], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - (out, err) = proc.communicate() + out, err = proc.communicate() except subprocess.CalledProcessError as e: self.log.error("%s error launching: %s; skipping" % (absolutescriptpath, e)) @@ -82,10 +82,12 @@ def collect(self): self.log.info("%s return no output" % absolutescriptpath) continue if err: - self.log.error("%s returned error output (stderr): %s" % + self.log.error("%s returned error output (stderr): %r" % (absolutescriptpath, err)) + if isinstance(out, bytes): + out = out.decode("utf8") # Use filter to remove empty lines of output - for line in filter(None, out.split('\n')): + for line in [_f for _f in out.split('\n') if _f]: # Ignore invalid lines try: name, value = line.split() diff --git a/src/collectors/varnish/varnish.py b/src/collectors/varnish/varnish.py index 1baa40031..827992a12 100644 --- a/src/collectors/varnish/varnish.py +++ b/src/collectors/varnish/varnish.py @@ -176,6 +176,8 @@ def poll(self): output = subprocess.Popen(command, stdout=subprocess.PIPE).communicate()[0] + if isinstance(output, bytes): + output = output.decode("utf8") except OSError: output = "" diff --git a/src/collectors/zookeeper/zookeeper.py b/src/collectors/zookeeper/zookeeper.py index 393e504d3..d7e8f5937 100644 --- a/src/collectors/zookeeper/zookeeper.py +++ b/src/collectors/zookeeper/zookeeper.py @@ -64,7 +64,7 @@ def get_default_config(self): return config def get_raw_stats(self, host, port): - data = '' + data = b'' # connect try: if port is None: @@ -74,13 +74,13 @@ def get_raw_stats(self, host, port): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect((host, int(port))) # request stats - sock.send('mntr\n') + sock.send(b'mntr\n') # something big enough to get whatever is sent back data = sock.recv(4096) except socket.error: self.log.exception('Failed to get stats from %s:%s', host, port) - return data + return data.decode("utf8") def get_stats(self, host, port): # stuff that's always ignored, aren't 'stats' diff --git a/src/diamond/collector.py b/src/diamond/collector.py index 5c25743c3..432d222cd 100644 --- a/src/diamond/collector.py +++ b/src/diamond/collector.py @@ -52,7 +52,10 @@ def get_hostname(config, method=None): proc = subprocess.Popen(config['hostname'], shell=True, stdout=subprocess.PIPE) - hostname = proc.communicate()[0].strip() + hostname = proc.communicate()[0] + if isinstance(hostname, bytes): + hostname = hostname.decode("utf8") + hostname = hostname.strip() if proc.returncode != 0: raise subprocess.CalledProcessError(proc.returncode, config['hostname']) @@ -569,8 +572,13 @@ def run_command(self, args): if str_to_bool(self.config['use_sudo']): command.insert(0, self.config['sudo_cmd']) - return subprocess.Popen(command, - stdout=subprocess.PIPE).communicate() + stdout, stderr = subprocess.Popen(command, + stdout=subprocess.PIPE).communicate() + if isinstance(stdout, bytes): + stdout = stdout.decode("utf8") + if isinstance(stderr, bytes): + stderr = stderr.decode("utf8") + return stdout, stderr except OSError: self.log.exception("Unable to run %s", command) return None diff --git a/test.py b/test.py index f3f735f45..65a3dbc88 100755 --- a/test.py +++ b/test.py @@ -138,7 +138,7 @@ def getFixtures(self): return fixtures def getPickledResults(self, results_name): - with open(self.getFixturePath(results_name), 'r') as f: + with open(self.getFixturePath(results_name), 'rb') as f: return pickle.load(f) def setPickledResults(self, results_name, data): From 9ec08f1beaab039da92b353f8ab9b42334272cd6 Mon Sep 17 00:00:00 2001 From: Hannu Valtonen Date: Tue, 25 Aug 2015 11:24:10 +0300 Subject: [PATCH 22/33] py3k: Fix test class to work with different type of __next__/next methods --- src/collectors/sockstat/test/testsockstat.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/collectors/sockstat/test/testsockstat.py b/src/collectors/sockstat/test/testsockstat.py index 132e3e96d..d3fbf3ce1 100644 --- a/src/collectors/sockstat/test/testsockstat.py +++ b/src/collectors/sockstat/test/testsockstat.py @@ -13,6 +13,7 @@ from collections import Iterator from diamond.collector import Collector from sockstat import SockstatCollector +import sys ########################################################################## @@ -33,15 +34,23 @@ def test_import(self): @patch('os.access', Mock(return_value=True)) @patch.object(Collector, 'publish') def test_should_open_proc_net_sockstat(self, publish_mock, open_mock): - class Klass(Iterator): - + class KlassPy2(Iterator): def close(self): pass def next(self): - raise StopIteration + raise StopIteration() + + class KlassPy3(Iterator): + def close(self): + pass - open_mock.return_value = Klass() + def __next__(self): + raise StopIteration() + if sys.version_info[0] == 2: # py2.6x does not support the namedtuple .major + open_mock.return_value = KlassPy2() + else: + open_mock.return_value = KlassPy3() self.collector.collect() calls = [call('/proc/net/sockstat'), call('/proc/net/sockstat6')] open_mock.assert_has_calls(calls) From 01e3b8193943f990f6599649a9c3dbca62e6acd7 Mon Sep 17 00:00:00 2001 From: Hannu Valtonen Date: Wed, 26 Aug 2015 14:24:55 +0300 Subject: [PATCH 23/33] py3k: Makefile/setup.py changes to support building both py2.x and py3k Also add __pycache__ files of py3k into the clean Makefile target. --- Makefile | 22 +++++++++++++++------- setup.py | 8 ++------ 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index a40037f20..38f7c0b52 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,7 @@ VERSION :=$(shell bash version.sh ) RELEASE :=$(shell ls -1 dist/*.noarch.rpm 2>/dev/null | wc -l ) HASH :=$(shell git rev-parse HEAD ) DISTRO=precise +PYTHON?=python3 all: @echo "make run - Run Diamond from this directory" @@ -40,19 +41,19 @@ docs: version make test sdist: version - ./setup.py sdist --prune + $(PYTHON) setup.py sdist bdist: version - ./setup.py bdist --prune + $(PYTHON) setup.py bdist bdist_wheel: version - USE_SETUPTOOLS=1 ./setup.py bdist_wheel + $(PYTHON) setup.py bdist_wheel install: version - ./setup.py install --root $(DESTDIR) + $(PYTHON) setup.py install --root $(DESTDIR) develop: version - USE_SETUPTOOLS=1 ./setup.py develop + $(PYTHON) setup.py develop rpm: buildrpm @@ -62,6 +63,12 @@ buildrpm: sdist --build-requires='python, python-configobj, python-setuptools' \ --requires='python, python-configobj, python-setuptools' +buildrpmpy3: sdist + $(PYTHON) setup.py bdist_rpm \ + --release=`ls dist/*.noarch.rpm | wc -l` \ + --build-requires='python3, python3-configobj' \ + --requires='python3, python3-configobj' + deb: builddeb sdeb: buildsourcedeb @@ -69,7 +76,7 @@ sdeb: buildsourcedeb builddeb: version dch --newversion $(VERSION) --distribution unstable --force-distribution -b "Last Commit: $(shell git log -1 --pretty=format:'(%ai) %H %cn <%ce>')" dch --release "new upstream" - ./setup.py sdist --prune + ./setup.py sdist mkdir -p build tar -C build -zxf dist/$(PROJECT)-$(VERSION).tar.gz (cd build/$(PROJECT)-$(VERSION) && debuild -us -uc -v$(VERSION)) @@ -78,7 +85,7 @@ builddeb: version buildsourcedeb: version dch --newversion $(VERSION)~$(DISTRO) --distribution $(DISTRO) --force-distribution -b "Last Commit: $(shell git log -1 --pretty=format:'(%ai) %H %cn <%ce>')" dch --release "new upstream" - ./setup.py sdist --prune + ./setup.py sdist mkdir -p build tar -C build -zxf dist/$(PROJECT)-$(VERSION).tar.gz (cd build/$(PROJECT)-$(VERSION) && debuild -S -sa -v$(VERSION)) @@ -96,6 +103,7 @@ clean: ./setup.py clean rm -rf dist build MANIFEST .tox *.log find . -name '*.pyc' -delete + find -name __pycache__ -delete version: ./version.sh > version.txt diff --git a/setup.py b/setup.py index cace4c5d1..5093d74e9 100755 --- a/setup.py +++ b/setup.py @@ -17,12 +17,8 @@ def running_under_virtualenv(): return False -if os.environ.get('USE_SETUPTOOLS'): - from setuptools import setup - setup_kwargs = dict(zip_safe=0) -else: - from distutils.core import setup - setup_kwargs = dict() +from setuptools import setup +setup_kwargs = dict(zip_safe=0) if os.name == 'nt': pgm_files = os.environ["ProgramFiles"] From d35960bb19af7d7651abc6d8c54b5ad5f81d7daa Mon Sep 17 00:00:00 2001 From: Hannu Valtonen Date: Wed, 26 Aug 2015 14:55:20 +0300 Subject: [PATCH 24/33] Makefile: get distribution name by asking python itself --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 38f7c0b52..e94ae958d 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ PROJECT=diamond VERSION :=$(shell bash version.sh ) RELEASE :=$(shell ls -1 dist/*.noarch.rpm 2>/dev/null | wc -l ) HASH :=$(shell git rev-parse HEAD ) -DISTRO=precise +DISTRO:=$(shell python -c "import platform;print(platform.linux_distribution()[0])") PYTHON?=python3 all: From 3d1f1438f6ac24f4db8a9aee209661039228119d Mon Sep 17 00:00:00 2001 From: Hannu Valtonen Date: Wed, 26 Aug 2015 15:52:52 +0300 Subject: [PATCH 25/33] pep8: Fix pep8 issues created during py3k work --- src/collectors/dropwizard/dropwizard.py | 1 - src/collectors/endecadgraph/endecadgraph.py | 5 ++++- src/collectors/httpd/httpd.py | 1 + src/collectors/httpjson/httpjson.py | 1 + src/collectors/kafkastat/kafkastat.py | 1 + src/collectors/monit/monit.py | 8 +++++--- src/collectors/nginx/nginx.py | 1 + src/collectors/ntpd/ntpd.py | 2 +- src/collectors/puppetdashboard/puppetdashboard.py | 1 + src/collectors/sidekiqweb/sidekiqweb.py | 8 +++++--- src/collectors/sockstat/test/testsockstat.py | 3 ++- src/collectors/websitemonitor/websitemonitor.py | 1 + src/diamond/collector.py | 4 ++-- src/diamond/handler/logentries_diamond.py | 1 + test.py | 2 +- 15 files changed, 27 insertions(+), 13 deletions(-) diff --git a/src/collectors/dropwizard/dropwizard.py b/src/collectors/dropwizard/dropwizard.py index c07a9a4d4..1020393e5 100644 --- a/src/collectors/dropwizard/dropwizard.py +++ b/src/collectors/dropwizard/dropwizard.py @@ -13,7 +13,6 @@ from diamond.pycompat import HTTPError - class DropwizardCollector(diamond.collector.Collector): def get_default_config_help(self): diff --git a/src/collectors/endecadgraph/endecadgraph.py b/src/collectors/endecadgraph/endecadgraph.py index 6e8fad8f2..4da6a12a3 100644 --- a/src/collectors/endecadgraph/endecadgraph.py +++ b/src/collectors/endecadgraph/endecadgraph.py @@ -118,7 +118,10 @@ def walkXML(context, elemList): url = 'http://%s:%d/admin?op=stats' % (self.config['host'], self.config['port']) try: - xml = diamond.pycompat.urlopen(url, timeout=self.config['timeout']).read() + xml = diamond.pycompat.urlopen( + url, + timeout=self.config['timeout']).read( + ) except Exception as e: self.log.error('Could not connect to endeca on %s: %s' % (url, e)) return {} diff --git a/src/collectors/httpd/httpd.py b/src/collectors/httpd/httpd.py index afddb46ab..9dd41da1c 100644 --- a/src/collectors/httpd/httpd.py +++ b/src/collectors/httpd/httpd.py @@ -17,6 +17,7 @@ import diamond.collector from diamond.pycompat import urlparse + class HttpdCollector(diamond.collector.Collector): def process_config(self): diff --git a/src/collectors/httpjson/httpjson.py b/src/collectors/httpjson/httpjson.py index 32685a2d8..406f81c3f 100644 --- a/src/collectors/httpjson/httpjson.py +++ b/src/collectors/httpjson/httpjson.py @@ -9,6 +9,7 @@ import diamond.pycompat from diamond.pycompat import Request, URLError + class HTTPJSONCollector(diamond.collector.Collector): def get_default_config_help(self): diff --git a/src/collectors/kafkastat/kafkastat.py b/src/collectors/kafkastat/kafkastat.py index 77c1f29d1..b08a810f6 100644 --- a/src/collectors/kafkastat/kafkastat.py +++ b/src/collectors/kafkastat/kafkastat.py @@ -21,6 +21,7 @@ import diamond.pycompat from diamond.pycompat import long, urlencode, URLError + class KafkaCollector(diamond.collector.Collector): ATTRIBUTE_TYPES = { 'double': float, diff --git a/src/collectors/monit/monit.py b/src/collectors/monit/monit.py index 59a5c5a34..cb7657c44 100644 --- a/src/collectors/monit/monit.py +++ b/src/collectors/monit/monit.py @@ -18,6 +18,7 @@ import diamond.pycompat from diamond.pycompat import HTTPError, Request + class MonitCollector(diamond.collector.Collector): def get_default_config_help(self): @@ -49,10 +50,11 @@ def collect(self): try: request = Request(url) - # # shouldn't need to check this - original_string = '%s:%s' % (self.config['user'], self.config['passwd']) - base64string = base64.encodestring(original_string.encode("utf8")).replace(b'\n', b'') + original_string = '%s:%s' % (self.config['user'], + self.config['passwd']) + base64string = base64.encodestring(original_string.encode("utf8")) + base64string = base64string.replace(b'\n', b'') request.add_header("Authorization", "Basic %s" % base64string) response = diamond.pycompat.urlopen(request) except HTTPError as err: diff --git a/src/collectors/nginx/nginx.py b/src/collectors/nginx/nginx.py index 058e454c6..faba5f430 100644 --- a/src/collectors/nginx/nginx.py +++ b/src/collectors/nginx/nginx.py @@ -60,6 +60,7 @@ import diamond.pycompat + class NginxCollector(diamond.collector.Collector): def get_default_config_help(self): diff --git a/src/collectors/ntpd/ntpd.py b/src/collectors/ntpd/ntpd.py index 0d86a2ec1..c120e562a 100644 --- a/src/collectors/ntpd/ntpd.py +++ b/src/collectors/ntpd/ntpd.py @@ -47,7 +47,7 @@ def run_command(self, command): command.insert(0, self.config['sudo_cmd']) stdout = subprocess.Popen(command, - stdout=subprocess.PIPE).communicate()[0] + stdout=subprocess.PIPE).communicate()[0] if isinstance(stdout, bytes): stdout = stdout.decode("utf8") except OSError: diff --git a/src/collectors/puppetdashboard/puppetdashboard.py b/src/collectors/puppetdashboard/puppetdashboard.py index 11f06064c..2ed44c5e0 100644 --- a/src/collectors/puppetdashboard/puppetdashboard.py +++ b/src/collectors/puppetdashboard/puppetdashboard.py @@ -8,6 +8,7 @@ import diamond.collector import diamond.pycompat + class PuppetDashboardCollector(diamond.collector.Collector): def get_default_config_help(self): diff --git a/src/collectors/sidekiqweb/sidekiqweb.py b/src/collectors/sidekiqweb/sidekiqweb.py index b0ca41ca9..86d5fb6e4 100644 --- a/src/collectors/sidekiqweb/sidekiqweb.py +++ b/src/collectors/sidekiqweb/sidekiqweb.py @@ -41,8 +41,9 @@ def get_default_config(self): def collect(self): try: - response = diamond.pycompat.urlopen("http://%s:%s/dashboard/stats" % ( - self.config['host'], int(self.config['port']))) + url = "http://%s:%s/dashboard/stats" % ( + self.config['host'], int(self.config['port'])) + response = diamond.pycompat.urlopen(url) except Exception as e: self.log.error('Couldnt connect to sidekiq-web: %s', e) return {} @@ -56,7 +57,8 @@ def collect(self): for k in j: for item, value in j[k].items(): - if isinstance(value, (bytes, diamond.pycompat.unicode)) and 'M' in value: + if isinstance(value, (bytes, diamond.pycompat.unicode)) \ + and 'M' in value: value = float(value.replace('M', '')) for unit in self.config['byte_unit']: unit_value = diamond.convertor.binary.convert( diff --git a/src/collectors/sockstat/test/testsockstat.py b/src/collectors/sockstat/test/testsockstat.py index d3fbf3ce1..0753fe724 100644 --- a/src/collectors/sockstat/test/testsockstat.py +++ b/src/collectors/sockstat/test/testsockstat.py @@ -47,7 +47,8 @@ def close(self): def __next__(self): raise StopIteration() - if sys.version_info[0] == 2: # py2.6x does not support the namedtuple .major + # py2.6x does not support the namedtuple .major + if sys.version_info[0] == 2: open_mock.return_value = KlassPy2() else: open_mock.return_value = KlassPy3() diff --git a/src/collectors/websitemonitor/websitemonitor.py b/src/collectors/websitemonitor/websitemonitor.py index f002f87da..cd3700baa 100644 --- a/src/collectors/websitemonitor/websitemonitor.py +++ b/src/collectors/websitemonitor/websitemonitor.py @@ -8,6 +8,7 @@ import diamond.pycompat from diamond.pycompat import URLError + class WebsiteMonitorCollector(diamond.collector.Collector): """ Gather HTTP response code and Duration of HTTP request diff --git a/src/diamond/collector.py b/src/diamond/collector.py index 432d222cd..2856f7169 100644 --- a/src/diamond/collector.py +++ b/src/diamond/collector.py @@ -572,8 +572,8 @@ def run_command(self, args): if str_to_bool(self.config['use_sudo']): command.insert(0, self.config['sudo_cmd']) - stdout, stderr = subprocess.Popen(command, - stdout=subprocess.PIPE).communicate() + p = subprocess.Popen(command, stdout=subprocess.PIPE) + stdout, stderr = p.communicate() if isinstance(stdout, bytes): stdout = stdout.decode("utf8") if isinstance(stderr, bytes): diff --git a/src/diamond/handler/logentries_diamond.py b/src/diamond/handler/logentries_diamond.py index a33e564b6..11be9f9d7 100644 --- a/src/diamond/handler/logentries_diamond.py +++ b/src/diamond/handler/logentries_diamond.py @@ -11,6 +11,7 @@ from collections import deque from diamond.pycompat import URLError, urlopen, Request + class LogentriesDiamondHandler(Handler): """ Implements the abstract Handler class diff --git a/test.py b/test.py index 65a3dbc88..c5e477bdc 100755 --- a/test.py +++ b/test.py @@ -42,7 +42,7 @@ try: # py3k way import builtins BUILTIN_OPEN = "builtins.open" -except ImportError: # py2.x way +except ImportError: # py2.x way BUILTIN_OPEN = "__builtin__.open" sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__)))) From 55c7d5638bc4698fbc3687477c4679fb13bd3743 Mon Sep 17 00:00:00 2001 From: Hannu Valtonen Date: Wed, 26 Aug 2015 18:25:09 +0300 Subject: [PATCH 26/33] py3k: Replace file() calls with open() --- bin/diamond | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/diamond b/bin/diamond index d29959341..1a2a81494 100755 --- a/bin/diamond +++ b/bin/diamond @@ -147,7 +147,7 @@ def main(): # Read existing pid file try: - pf = file(options.pidfile, 'r') + pf = open(options.pidfile, 'r') pid = int(pf.read().strip()) pf.close() except (IOError, ValueError): @@ -186,7 +186,7 @@ def main(): # Write pid file pid = str(os.getpid()) try: - pf = file(options.pidfile, 'w+') + pf = open(options.pidfile, 'w+') except IOError as e: print("Failed to write PID file: %s" % (e), file=sys.stderr) @@ -271,7 +271,7 @@ def main(): # Write pid file pid = str(os.getpid()) try: - pf = file(options.pidfile, 'w+') + pf = open(options.pidfile, 'w+') except IOError as e: log.error("Failed to write child PID file: %s" % (e)) sys.exit(1) From 35510ed870648511a15ca9a9e6267238d60635e7 Mon Sep 17 00:00:00 2001 From: Hannu Valtonen Date: Wed, 26 Aug 2015 18:34:13 +0300 Subject: [PATCH 27/33] py3k: Change imports to be relative imports for Handlers --- src/diamond/handler/archive.py | 2 +- src/diamond/handler/cloudwatch.py | 2 +- src/diamond/handler/datadog.py | 2 +- src/diamond/handler/g_metric.py | 2 +- src/diamond/handler/graphite.py | 2 +- src/diamond/handler/graphitepickle.py | 2 +- src/diamond/handler/hostedgraphite.py | 4 ++-- src/diamond/handler/httpHandler.py | 2 +- src/diamond/handler/influxdbHandler.py | 2 +- src/diamond/handler/libratohandler.py | 2 +- src/diamond/handler/logentries_diamond.py | 2 +- src/diamond/handler/mqtt.py | 2 +- src/diamond/handler/multigraphite.py | 4 ++-- src/diamond/handler/multigraphitepickle.py | 4 ++-- src/diamond/handler/mysql.py | 2 +- src/diamond/handler/null.py | 2 +- src/diamond/handler/queue.py | 2 +- src/diamond/handler/rabbitmq_pubsub.py | 2 +- src/diamond/handler/rabbitmq_topic.py | 2 +- src/diamond/handler/riemann.py | 2 +- src/diamond/handler/rrdtool.py | 2 +- src/diamond/handler/sentry.py | 2 +- src/diamond/handler/signalfx.py | 2 +- src/diamond/handler/stats_d.py | 2 +- src/diamond/handler/statsite.py | 2 +- src/diamond/handler/tsdb.py | 2 +- src/diamond/handler/zmq_pubsub.py | 2 +- 27 files changed, 30 insertions(+), 30 deletions(-) diff --git a/src/diamond/handler/archive.py b/src/diamond/handler/archive.py index 0157cb6d3..f41f8c107 100644 --- a/src/diamond/handler/archive.py +++ b/src/diamond/handler/archive.py @@ -5,7 +5,7 @@ every night and remove after 7 days. """ -from Handler import Handler +from . Handler import Handler import logging import logging.handlers diff --git a/src/diamond/handler/cloudwatch.py b/src/diamond/handler/cloudwatch.py index ebdf0d862..9a9897200 100644 --- a/src/diamond/handler/cloudwatch.py +++ b/src/diamond/handler/cloudwatch.py @@ -41,7 +41,7 @@ import sys import datetime -from Handler import Handler +from . Handler import Handler from configobj import Section try: diff --git a/src/diamond/handler/datadog.py b/src/diamond/handler/datadog.py index d25be756c..35e740dce 100644 --- a/src/diamond/handler/datadog.py +++ b/src/diamond/handler/datadog.py @@ -10,7 +10,7 @@ """ -from Handler import Handler +from . Handler import Handler import logging from collections import deque diff --git a/src/diamond/handler/g_metric.py b/src/diamond/handler/g_metric.py index 9d41804af..4d7ae23d9 100644 --- a/src/diamond/handler/g_metric.py +++ b/src/diamond/handler/g_metric.py @@ -5,7 +5,7 @@ [Ganglia Monitoring System](http://ganglia.sourceforge.net/) """ -from Handler import Handler +from . Handler import Handler import logging try: import gmetric diff --git a/src/diamond/handler/graphite.py b/src/diamond/handler/graphite.py index 87d0cae1e..1c5d39507 100644 --- a/src/diamond/handler/graphite.py +++ b/src/diamond/handler/graphite.py @@ -16,7 +16,7 @@ """ -from Handler import Handler +from . Handler import Handler import socket import time diff --git a/src/diamond/handler/graphitepickle.py b/src/diamond/handler/graphitepickle.py index edda9c5e8..888f0562b 100644 --- a/src/diamond/handler/graphitepickle.py +++ b/src/diamond/handler/graphitepickle.py @@ -18,7 +18,7 @@ import struct -from graphite import GraphiteHandler +from . graphite import GraphiteHandler try: import cPickle as pickle diff --git a/src/diamond/handler/hostedgraphite.py b/src/diamond/handler/hostedgraphite.py index 01289d6b1..287d889af 100644 --- a/src/diamond/handler/hostedgraphite.py +++ b/src/diamond/handler/hostedgraphite.py @@ -16,8 +16,8 @@ """ -from Handler import Handler -from graphite import GraphiteHandler +from . Handler import Handler +from . graphite import GraphiteHandler class HostedGraphiteHandler(Handler): diff --git a/src/diamond/handler/httpHandler.py b/src/diamond/handler/httpHandler.py index e591bdbad..c8b1c926e 100755 --- a/src/diamond/handler/httpHandler.py +++ b/src/diamond/handler/httpHandler.py @@ -13,7 +13,7 @@ """ -from Handler import Handler +from . Handler import Handler from . pycompat import Request, urlopen diff --git a/src/diamond/handler/influxdbHandler.py b/src/diamond/handler/influxdbHandler.py index 7f3af4b3c..0757177d9 100644 --- a/src/diamond/handler/influxdbHandler.py +++ b/src/diamond/handler/influxdbHandler.py @@ -29,7 +29,7 @@ """ import time -from Handler import Handler +from . Handler import Handler try: from influxdb.client import InfluxDBClient diff --git a/src/diamond/handler/libratohandler.py b/src/diamond/handler/libratohandler.py index f4a565cc0..18cfced0a 100644 --- a/src/diamond/handler/libratohandler.py +++ b/src/diamond/handler/libratohandler.py @@ -12,7 +12,7 @@ """ -from Handler import Handler +from . Handler import Handler import logging import time import re diff --git a/src/diamond/handler/logentries_diamond.py b/src/diamond/handler/logentries_diamond.py index 11be9f9d7..473b856da 100644 --- a/src/diamond/handler/logentries_diamond.py +++ b/src/diamond/handler/logentries_diamond.py @@ -5,7 +5,7 @@ based on data in real time. """ -from Handler import Handler +from . Handler import Handler import logging import json from collections import deque diff --git a/src/diamond/handler/mqtt.py b/src/diamond/handler/mqtt.py index fc60ffa7a..1d6558ec5 100644 --- a/src/diamond/handler/mqtt.py +++ b/src/diamond/handler/mqtt.py @@ -59,7 +59,7 @@ """ -from Handler import Handler +from . Handler import Handler from diamond.collector import get_hostname import os HAVE_SSL = True diff --git a/src/diamond/handler/multigraphite.py b/src/diamond/handler/multigraphite.py index 4d8f8e55f..3755673e5 100644 --- a/src/diamond/handler/multigraphite.py +++ b/src/diamond/handler/multigraphite.py @@ -6,8 +6,8 @@ Specify them as a list of hosts divided by comma. """ -from Handler import Handler -from graphite import GraphiteHandler +from . Handler import Handler +from . graphite import GraphiteHandler from copy import deepcopy diff --git a/src/diamond/handler/multigraphitepickle.py b/src/diamond/handler/multigraphitepickle.py index 3c49d4320..4f1bfc24f 100644 --- a/src/diamond/handler/multigraphitepickle.py +++ b/src/diamond/handler/multigraphitepickle.py @@ -6,8 +6,8 @@ servers. Specify them as a list of hosts divided by comma. """ -from Handler import Handler -from graphitepickle import GraphitePickleHandler +from . Handler import Handler +from . graphitepickle import GraphitePickleHandler from copy import deepcopy diff --git a/src/diamond/handler/mysql.py b/src/diamond/handler/mysql.py index ee4e188ab..bf0c6cea7 100644 --- a/src/diamond/handler/mysql.py +++ b/src/diamond/handler/mysql.py @@ -4,7 +4,7 @@ Insert the collected values into a mysql table """ -from Handler import Handler +from . Handler import Handler import MySQLdb diff --git a/src/diamond/handler/null.py b/src/diamond/handler/null.py index c593dd852..411aca0ed 100644 --- a/src/diamond/handler/null.py +++ b/src/diamond/handler/null.py @@ -4,7 +4,7 @@ Output the collected values to the debug log channel. """ -from Handler import Handler +from . Handler import Handler class NullHandler(Handler): diff --git a/src/diamond/handler/queue.py b/src/diamond/handler/queue.py index 8ebf44107..9d1a9c786 100644 --- a/src/diamond/handler/queue.py +++ b/src/diamond/handler/queue.py @@ -5,7 +5,7 @@ do not try to use it as a normal handler """ -from Handler import Handler +from . Handler import Handler import Queue diff --git a/src/diamond/handler/rabbitmq_pubsub.py b/src/diamond/handler/rabbitmq_pubsub.py index 974824f59..229907b34 100644 --- a/src/diamond/handler/rabbitmq_pubsub.py +++ b/src/diamond/handler/rabbitmq_pubsub.py @@ -4,7 +4,7 @@ Output the collected values to RabitMQ pub/sub channel """ -from Handler import Handler +from . Handler import Handler import time try: diff --git a/src/diamond/handler/rabbitmq_topic.py b/src/diamond/handler/rabbitmq_topic.py index 664a2a442..2909c3417 100644 --- a/src/diamond/handler/rabbitmq_topic.py +++ b/src/diamond/handler/rabbitmq_topic.py @@ -6,7 +6,7 @@ the metric path """ -from Handler import Handler +from . Handler import Handler try: import pika diff --git a/src/diamond/handler/riemann.py b/src/diamond/handler/riemann.py index 02943dc7a..fa07e3356 100644 --- a/src/diamond/handler/riemann.py +++ b/src/diamond/handler/riemann.py @@ -18,7 +18,7 @@ """ -from Handler import Handler +from . Handler import Handler import logging try: diff --git a/src/diamond/handler/rrdtool.py b/src/diamond/handler/rrdtool.py index 02cecc60d..a51ccedf7 100644 --- a/src/diamond/handler/rrdtool.py +++ b/src/diamond/handler/rrdtool.py @@ -8,7 +8,7 @@ import re import subprocess from diamond.pycompat import Queue, Empty -from Handler import Handler +from . Handler import Handler # # Constants for RRD file creation. diff --git a/src/diamond/handler/sentry.py b/src/diamond/handler/sentry.py index 821d74643..e39686db2 100644 --- a/src/diamond/handler/sentry.py +++ b/src/diamond/handler/sentry.py @@ -31,7 +31,7 @@ import logging import re -from Handler import Handler +from . Handler import Handler from diamond.collector import get_hostname from configobj import Section diff --git a/src/diamond/handler/signalfx.py b/src/diamond/handler/signalfx.py index 4d2d634ec..51246c97c 100644 --- a/src/diamond/handler/signalfx.py +++ b/src/diamond/handler/signalfx.py @@ -13,7 +13,7 @@ posting """ -from Handler import Handler +from . Handler import Handler from diamond.util import get_diamond_version from diamond.pycompat import URLError, urlopen, Request import json diff --git a/src/diamond/handler/stats_d.py b/src/diamond/handler/stats_d.py index 83f7a2816..3e60465e8 100644 --- a/src/diamond/handler/stats_d.py +++ b/src/diamond/handler/stats_d.py @@ -27,7 +27,7 @@ """ -from Handler import Handler +from . Handler import Handler import logging try: import statsd diff --git a/src/diamond/handler/statsite.py b/src/diamond/handler/statsite.py index 60f18381f..2be0493a2 100644 --- a/src/diamond/handler/statsite.py +++ b/src/diamond/handler/statsite.py @@ -52,7 +52,7 @@ """ -from Handler import Handler +from . Handler import Handler import socket diff --git a/src/diamond/handler/tsdb.py b/src/diamond/handler/tsdb.py index c67471733..d688e28cf 100644 --- a/src/diamond/handler/tsdb.py +++ b/src/diamond/handler/tsdb.py @@ -50,7 +50,7 @@ ' """ -from Handler import Handler +from . Handler import Handler from diamond.metric import Metric import socket diff --git a/src/diamond/handler/zmq_pubsub.py b/src/diamond/handler/zmq_pubsub.py index 30bd0cad9..92f76ecaa 100644 --- a/src/diamond/handler/zmq_pubsub.py +++ b/src/diamond/handler/zmq_pubsub.py @@ -4,7 +4,7 @@ Output the collected values to a Zer0MQ pub/sub channel """ -from Handler import Handler +from . Handler import Handler try: import zmq From 40ee7478701869fd13827731c63b6ccad30ecf6b Mon Sep 17 00:00:00 2001 From: Grant Ridder Date: Fri, 10 Mar 2017 23:48:24 -0800 Subject: [PATCH 28/33] Add python 3 to Travis testing --- .travis.requirements3.txt | 19 +++++++++++++++++++ .travis.yml | 6 +++++- 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 .travis.requirements3.txt diff --git a/.travis.requirements3.txt b/.travis.requirements3.txt new file mode 100644 index 000000000..ce105c411 --- /dev/null +++ b/.travis.requirements3.txt @@ -0,0 +1,19 @@ +#MySQL-python +PyYAML +#beanstalkc +riemann-client +boto +configobj +docker-py +kitchen +mock +psutil +pymongo<3.0 +statsd +#pyutmp +redis +simplejson +setproctitle +psycopg2 +#PySensors +pysnmp diff --git a/.travis.yml b/.travis.yml index fddae9303..3704ec481 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,9 @@ python: # - "2.5" - "2.6" - "2.7" + - "3.4" + - "3.5" + - "3.6" cache: directories: @@ -14,7 +17,8 @@ cache: # command to install dependencies, e.g. pip install -r requirements.txt install: - pip install Cython - - pip install -r .travis.requirements.txt + - if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then pip install -r .travis.requirements.txt; fi + - if [[ $TRAVIS_PYTHON_VERSION == 3* ]]; then pip install -r .travis.requirements3.txt; fi - pip install pep8==1.5.7 - pip install coveralls - if [[ $TRAVIS_PYTHON_VERSION == '2.6' ]]; then pip install unittest2; fi From dfbe06ed3dcf66a863f7acf43c51ad40544e738d Mon Sep 17 00:00:00 2001 From: Grant Ridder Date: Fri, 10 Mar 2017 23:55:07 -0800 Subject: [PATCH 29/33] Install python34 in Vagrant centos7-test host --- Vagrantfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Vagrantfile b/Vagrantfile index aec84d004..643ed8b52 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -81,12 +81,17 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| c.vm.provision "shell", inline: "sudo rpm -ivh http://dl.fedoraproject.org/pub/epel/7/x86_64/e/epel-release-7-9.noarch.rpm" c.vm.provision "shell", inline: "sudo yum install -y git rpm-build python-configobj python-test python-mock tree vim-enhanced MySQL-python htop gcc" + # Install python 3 + c.vm.provision "shell", inline: "sudo yum install -y python34 python34-devel" + c.vm.provision "shell", inline: "curl -O https://bootstrap.pypa.io/get-pip.py && sudo /usr/bin/python3.4 get-pip.py" + # Install python libraries needed by specific collectors c.vm.provision "shell", inline: "sudo yum install -y postgresql-devel" # req for psycopg2 c.vm.provision "shell", inline: "sudo yum install -y Cython" # req for pyutmp c.vm.provision "shell", inline: "sudo yum install -y lm_sensors-devel lm_sensors python-devel" # req for pyutmp c.vm.provision "shell", inline: "sudo yum install -y python-pip" c.vm.provision "shell", inline: "sudo pip install -r /vagrant/.travis.requirements.txt" + c.vm.provision "shell", inline: "sudo pip3 install -r /vagrant/.travis.requirements3.txt" # Setup Diamond to run as a service c.vm.provision "shell", inline: "sudo yum install -y python-setuptools" @@ -104,6 +109,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| # Build Diamond docs and run tests c.vm.provision "shell", inline: "sudo pip install pep8==1.5.7" + c.vm.provision "shell", inline: "sudo pip3 install pep8==1.5.7" c.vm.provision "shell", inline: "echo 'Build docs...' && python /vagrant/build_doc.py" c.vm.provision "shell", inline: "echo 'Running tests...' && python /vagrant/test.py" c.vm.provision "shell", inline: "echo 'Running pep8...' && pep8 --config=/vagrant/.pep8 /vagrant/src /vagrant/bin/diamond /vagrant/bin/diamond-setup /vagrant/build_doc.py /vagrant/setup.py /vagrant/test.py" From f0fadc500309561a45e5d25351b166e64daea5d3 Mon Sep 17 00:00:00 2001 From: Grant Ridder Date: Fri, 10 Mar 2017 23:57:05 -0800 Subject: [PATCH 30/33] More compatability changes Update several compat changes that were missed or added since origional commits - StringIO - urllib - print function --- src/collectors/etcdstat/etcdstat.py | 6 +++--- .../eventstoreprojections/eventstoreprojections.py | 11 +++++------ src/collectors/fluentd/fluentd.py | 2 +- src/collectors/jolokia/jolokia.py | 2 +- src/collectors/mesos_cgroup/mesos_cgroup.py | 6 +++--- src/collectors/mesos_cgroup/test/testmesos_cgroup.py | 6 +++--- src/collectors/netstat/test/testnetstat.py | 2 +- src/collectors/xfs/test/testxfs.py | 5 +---- src/diamond/handler/tsdb.py | 2 +- 9 files changed, 19 insertions(+), 23 deletions(-) diff --git a/src/collectors/etcdstat/etcdstat.py b/src/collectors/etcdstat/etcdstat.py index c376937cd..02bf88be2 100644 --- a/src/collectors/etcdstat/etcdstat.py +++ b/src/collectors/etcdstat/etcdstat.py @@ -13,8 +13,8 @@ """ import diamond.collector +import diamond.pycompat import json -import urllib2 METRICS_KEYS = ['sendPkgRate', 'recvPkgRate', @@ -100,8 +100,8 @@ def get_metrics(self, category): url = "%s://%s:%s/v2/stats/%s" % (protocol, self.config['host'], self.config['port'], category) - return json.load(urllib2.urlopen(url, **opts)) - except (urllib2.HTTPError, ValueError), err: + return json.load(diamond.pycompat.urlopen(url, **opts)) + except (HTTPError, ValueError) as err: self.log.error('Unable to read JSON response: %s' % err) return {} diff --git a/src/collectors/eventstoreprojections/eventstoreprojections.py b/src/collectors/eventstoreprojections/eventstoreprojections.py index 7c77a01e7..7a58e5fe3 100644 --- a/src/collectors/eventstoreprojections/eventstoreprojections.py +++ b/src/collectors/eventstoreprojections/eventstoreprojections.py @@ -17,10 +17,9 @@ """ -import urllib2 import json import diamond.collector - +import diamond.pycompat class EventstoreProjectionsCollector(diamond.collector.Collector): @@ -64,7 +63,7 @@ def _json_to_flat_metrics(self, prefix, data): for k, v in self._json_to_flat_metrics( "%s.%s" % (prefix, key), value): yield k, v - elif isinstance(value, basestring): + elif isinstance(value, str): if value == "Running": value = 1 yield ("%s.%s" % (prefix, key), value) @@ -91,12 +90,12 @@ def collect(self): self.config['route'] ) - req = urllib2.Request(eventstore_host, headers=self.config['headers']) + req = diamond.pycompat.Request(eventstore_host, headers=self.config['headers']) req.add_header('Content-type', 'application/json') try: - resp = urllib2.urlopen(req) - except urllib2.URLError as e: + resp = diamond.pycompat.urlopen(req) + except URLError as e: self.log.error("Can't open url %s. %s", eventstore_host, e) else: content = resp.read() diff --git a/src/collectors/fluentd/fluentd.py b/src/collectors/fluentd/fluentd.py index 567a86888..0121138a6 100644 --- a/src/collectors/fluentd/fluentd.py +++ b/src/collectors/fluentd/fluentd.py @@ -20,7 +20,7 @@ """ import diamond.collector -import urllib2 +import diamond.pycompat import json diff --git a/src/collectors/jolokia/jolokia.py b/src/collectors/jolokia/jolokia.py index fda15a80f..d84cc8b68 100644 --- a/src/collectors/jolokia/jolokia.py +++ b/src/collectors/jolokia/jolokia.py @@ -139,7 +139,7 @@ def __init__(self, *args, **kwargs): self.domains = [] if 'domains' in self.config: - if isinstance(self.config['domains'], basestring): + if isinstance(self.config['domains'], str): for domain in self.config['domains'].split('|'): self.domains.append(domain.strip()) elif isinstance(self.config['domains'], list): diff --git a/src/collectors/mesos_cgroup/mesos_cgroup.py b/src/collectors/mesos_cgroup/mesos_cgroup.py index 325671de3..61b3e9a22 100644 --- a/src/collectors/mesos_cgroup/mesos_cgroup.py +++ b/src/collectors/mesos_cgroup/mesos_cgroup.py @@ -23,8 +23,8 @@ """ import diamond.collector +import diamond.pycompat import json -import urllib2 import os @@ -125,8 +125,8 @@ def get_mesos_state(self): self.config['port'], self.config['mesos_state_path']) - return json.load(urllib2.urlopen(url)) - except (urllib2.HTTPError, ValueError), err: + return json.load(diamond.pycompat.urlopen(url)) + except (HTTPError, ValueError) as err: self.log.error('Unable to read JSON response: %s' % err) return {} diff --git a/src/collectors/mesos_cgroup/test/testmesos_cgroup.py b/src/collectors/mesos_cgroup/test/testmesos_cgroup.py index c2236b237..06cd8ee4b 100644 --- a/src/collectors/mesos_cgroup/test/testmesos_cgroup.py +++ b/src/collectors/mesos_cgroup/test/testmesos_cgroup.py @@ -35,7 +35,7 @@ def urlopen_se(url): if url == 'http://localhost:5051/state.json': return self.getFixture('state.json') else: - print url + print(url) raise NotImplementedError() def listdir_se(directory): @@ -48,7 +48,7 @@ def listdir_se(directory): if directory in cgroup_directories: return ["b0d5971e-915c-414b-aa25-0da46e64ff4e"] else: - print directory + print(directory) raise NotImplementedError() def isdir_se(directory): @@ -61,7 +61,7 @@ def isdir_se(directory): if directory in task_directories: return True else: - print directory + print(directory) raise NotImplementedError() def open_se(path, mode='r', create=True): diff --git a/src/collectors/netstat/test/testnetstat.py b/src/collectors/netstat/test/testnetstat.py index f4b22d4e1..261503a77 100644 --- a/src/collectors/netstat/test/testnetstat.py +++ b/src/collectors/netstat/test/testnetstat.py @@ -32,7 +32,7 @@ def test(self, publish_mock): self.setDocExample(collector=self.collector.__class__.__name__, metrics=metrics, defaultpath=self.collector.config['path']) - print publish_mock + print(publish_mock) self.assertPublishedMany(publish_mock, metrics) ################################################################################ diff --git a/src/collectors/xfs/test/testxfs.py b/src/collectors/xfs/test/testxfs.py index de708c1f2..4311cd9a9 100644 --- a/src/collectors/xfs/test/testxfs.py +++ b/src/collectors/xfs/test/testxfs.py @@ -8,10 +8,7 @@ from test import Mock from test import patch -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from test import StringIO from diamond.collector import Collector from xfs import XFSCollector diff --git a/src/diamond/handler/tsdb.py b/src/diamond/handler/tsdb.py index d688e28cf..aa1180502 100644 --- a/src/diamond/handler/tsdb.py +++ b/src/diamond/handler/tsdb.py @@ -77,7 +77,7 @@ def __init__(self, config=None): self.timeout = int(self.config['timeout']) self.metric_format = str(self.config['format']) self.tags = "" - if isinstance(self.config['tags'], basestring): + if isinstance(self.config['tags'], str): self.tags = self.config['tags'] elif isinstance(self.config['tags'], list): for tag in self.config['tags']: From 1bdf771fa08bc6603ba324a5fd283ac7242c50a6 Mon Sep 17 00:00:00 2001 From: Grant Ridder Date: Sat, 11 Mar 2017 00:14:20 -0800 Subject: [PATCH 31/33] Switch from izip to zip --- src/collectors/nvidia_gpu/nvidia_gpu.py | 3 +-- src/collectors/sidekiq/sidekiq.py | 4 +--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/collectors/nvidia_gpu/nvidia_gpu.py b/src/collectors/nvidia_gpu/nvidia_gpu.py index 2f1cfb96b..5371ae99d 100644 --- a/src/collectors/nvidia_gpu/nvidia_gpu.py +++ b/src/collectors/nvidia_gpu/nvidia_gpu.py @@ -11,7 +11,6 @@ * nvidia-ml-py (Optional) """ -from itertools import izip try: import pynvml USE_PYTHON_BINDING = True @@ -71,7 +70,7 @@ def collect_via_nvidia_smi(self, stats_config): stats = result.strip().split(',') assert len(stats) == len(stats_config) index = stats[0] - for stat_name, metric in izip(stats_config[1:], stats[1:]): + for stat_name, metric in zip(stats_config[1:], stats[1:]): metric_name = 'gpu_{index}.{stat_name}'.format( index=str(index), stat_name=stat_name diff --git a/src/collectors/sidekiq/sidekiq.py b/src/collectors/sidekiq/sidekiq.py index 8c35fad2e..9e686d7de 100644 --- a/src/collectors/sidekiq/sidekiq.py +++ b/src/collectors/sidekiq/sidekiq.py @@ -8,8 +8,6 @@ * redis """ -from itertools import izip - try: import redis from redis.sentinel import Sentinel @@ -89,7 +87,7 @@ def get_redis_client(self): else: sentinel_ports = [None for _ in xrange(len(ports))] - for port, sentinel_port in izip(ports, sentinel_ports): + for port, sentinel_port in zip(ports, sentinel_ports): for db in xrange(0, int(databases)): master = self.get_master( host, port, sentinel_port, sentinel_name From be0e557a25dfa6ee45e11b1a107b7dcb556b49b3 Mon Sep 17 00:00:00 2001 From: Grant Ridder Date: Sat, 11 Mar 2017 00:23:02 -0800 Subject: [PATCH 32/33] Update urllib compat more --- src/collectors/elasticsearch/test/testelasticsearch.py | 6 +++--- .../tests/testeventstoreprojections.py | 2 +- src/collectors/fluentd/fluentd.py | 2 +- src/collectors/jolokia/jolokia.py | 4 ++-- src/collectors/jolokia/test/testjolokia.py | 4 ++-- src/collectors/mesos/test/testmesos.py | 4 ++-- src/collectors/mesos_cgroup/test/testmesos_cgroup.py | 3 ++- src/collectors/nginx/test/testnginx.py | 2 +- src/collectors/solr/test/testsolr.py | 4 ++-- 9 files changed, 16 insertions(+), 15 deletions(-) diff --git a/src/collectors/elasticsearch/test/testelasticsearch.py b/src/collectors/elasticsearch/test/testelasticsearch.py index 26e0c332c..3bb111fd0 100644 --- a/src/collectors/elasticsearch/test/testelasticsearch.py +++ b/src/collectors/elasticsearch/test/testelasticsearch.py @@ -132,7 +132,7 @@ def test_should_work_with_real_data_v2(self, publish_mock): self.getFixture('cluster_stats_v2'), self.getFixture('indices_stats'), ] - urlopen_mock = patch('urllib2.urlopen', Mock( + urlopen_mock = patch(URLOPEN, Mock( side_effect=lambda *args: returns.pop(0))) self.collector.config['cluster'] = True @@ -272,7 +272,7 @@ def test_should_work_with_real_data_logstash_hourlymode(self, publish_mock): self.getFixture('stats'), self.getFixture('logstash_hourly_indices_stats'), ] - urlopen_mock = patch('urllib2.urlopen', Mock( + urlopen_mock = patch(URLOPEN, Mock( side_effect=lambda *args: returns.pop(0))) self.collector.config['logstash_mode'] = True @@ -430,7 +430,7 @@ def test_should_work_with_real_1_7_data(self, publish_mock): self.getFixture('stats1.7'), self.getFixture('indices_stats'), ] - urlopen_mock = patch('urllib2.urlopen', Mock( + urlopen_mock = patch(URLOPEN, Mock( side_effect=lambda *args: returns.pop(0))) urlopen_mock.start() diff --git a/src/collectors/eventstoreprojections/tests/testeventstoreprojections.py b/src/collectors/eventstoreprojections/tests/testeventstoreprojections.py index 33d89be0f..93b5ea95c 100644 --- a/src/collectors/eventstoreprojections/tests/testeventstoreprojections.py +++ b/src/collectors/eventstoreprojections/tests/testeventstoreprojections.py @@ -24,7 +24,7 @@ def setUp(self): def test_import(self): self.assertTrue(EventstoreProjectionsCollector) - @patch('urllib2.urlopen') + @patch(URLOPEN) @patch.object(Collector, 'publish') def test_should_work_with_real_data(self, publish_mock, urlopen_mock): returns = [self.getFixture('projections')] diff --git a/src/collectors/fluentd/fluentd.py b/src/collectors/fluentd/fluentd.py index 0121138a6..ea566754c 100644 --- a/src/collectors/fluentd/fluentd.py +++ b/src/collectors/fluentd/fluentd.py @@ -51,7 +51,7 @@ def collect(self): params = (self.config['host'], self.config['port'], self.API_PATH) url = "http://%s:%s/%s" % params - res = urllib2.urlopen(url) + res = diamond.pycompat.urlopen(url) data = json.load(res) result = self.parse_api_output(data) diff --git a/src/collectors/jolokia/jolokia.py b/src/collectors/jolokia/jolokia.py index d84cc8b68..72e9a9461 100644 --- a/src/collectors/jolokia/jolokia.py +++ b/src/collectors/jolokia/jolokia.py @@ -239,7 +239,7 @@ def _list_request(self): def _read_request(self, domain): try: - url_path = '/?%s' % urllib.urlencode({ + url_path = '/?%s' % diamond.pycompat.urlencode({ 'maxCollectionSize': '0', 'ignoreErrors': 'true', 'canonicalNaming': @@ -272,7 +272,7 @@ def _escape_domain(self, domain): return domain def _create_request(self, url): - req = urllib2.Request(url) + req = diamond.pycompat.Request(url) username = self.config["username"] password = self.config["password"] if username is not None and password is not None: diff --git a/src/collectors/jolokia/test/testjolokia.py b/src/collectors/jolokia/test/testjolokia.py index 7449f17f5..4ffa2f862 100644 --- a/src/collectors/jolokia/test/testjolokia.py +++ b/src/collectors/jolokia/test/testjolokia.py @@ -127,7 +127,7 @@ def test_canonical_names_setting_not_set(self): logger_mock.error.assert_not_called() @patch('jolokia.JolokiaCollector._create_request') - @patch('urllib2.urlopen') + @patch(URLOPEN) def test_should_handle_canonical_names_setting_True(self, urlopen_mock, create_request_mock): config = get_collector_config('JolokiaCollector', {}) @@ -145,7 +145,7 @@ def test_should_handle_canonical_names_setting_True(self, urlopen_mock, self.assertIs(collector.config['use_canonical_names'], True) @patch('jolokia.JolokiaCollector._create_request') - @patch('urllib2.urlopen') + @patch(URLOPEN) def test_should_handle_canonical_names_setting_False(self, urlopen_mock, create_request_mock): config = get_collector_config('JolokiaCollector', {}) diff --git a/src/collectors/mesos/test/testmesos.py b/src/collectors/mesos/test/testmesos.py index 92c8dcf15..bac193925 100644 --- a/src/collectors/mesos/test/testmesos.py +++ b/src/collectors/mesos/test/testmesos.py @@ -62,7 +62,7 @@ def test_should_work_for_slave_with_real_data(self, publish_mock): self.getFixture('slave_monitor_statistics.json') ] - urlopen_mock = patch('urllib2.urlopen', Mock( + urlopen_mock = patch(URLOPEN, Mock( side_effect=lambda *args: returns.pop(0))) urlopen_mock.start() @@ -147,7 +147,7 @@ def fixture_cpu_utilisation(self, publish_mock): self.getFixture('slave_metrics_state.json'), self.getFixture('slave_monitor_statistics_cpus_utilisation.json'), ] - urlopen_mock = patch('urllib2.urlopen', Mock( + urlopen_mock = patch(URLOPEN, Mock( side_effect=lambda *args: returns.pop(0))) urlopen_mock.start() self.collector.collect() diff --git a/src/collectors/mesos_cgroup/test/testmesos_cgroup.py b/src/collectors/mesos_cgroup/test/testmesos_cgroup.py index 06cd8ee4b..6fc2d527b 100644 --- a/src/collectors/mesos_cgroup/test/testmesos_cgroup.py +++ b/src/collectors/mesos_cgroup/test/testmesos_cgroup.py @@ -10,6 +10,7 @@ from test import Mock from diamond.collector import Collector +from diamond.pycompat import URLOPEN from mesos_cgroup import MesosCGroupCollector @@ -91,7 +92,7 @@ def open_se(path, mode='r', create=True): patch_open.start() return o - patch_urlopen = patch('urllib2.urlopen', Mock(side_effect=urlopen_se)) + patch_urlopen = patch(URLOPEN, Mock(side_effect=urlopen_se)) patch_listdir = patch('os.listdir', Mock(side_effect=listdir_se)) patch_isdir = patch('os.path.isdir', Mock(side_effect=isdir_se)) patch_open = patch('__builtin__.open', MagicMock(spec=file, diff --git a/src/collectors/nginx/test/testnginx.py b/src/collectors/nginx/test/testnginx.py index ff41147fe..f01404a69 100644 --- a/src/collectors/nginx/test/testnginx.py +++ b/src/collectors/nginx/test/testnginx.py @@ -79,7 +79,7 @@ def test_plus_should_work_with_real_data(self, publish_counter_mock, } ) - patch_urlopen = patch('urllib2.urlopen', Mock( + patch_urlopen = patch(URLOPEN, Mock( return_value=mockResponse)) patch_urlopen.start() diff --git a/src/collectors/solr/test/testsolr.py b/src/collectors/solr/test/testsolr.py index 03e8de206..a7af2cf29 100644 --- a/src/collectors/solr/test/testsolr.py +++ b/src/collectors/solr/test/testsolr.py @@ -24,7 +24,7 @@ def setUp(self): def test_import(self): self.assertTrue(SolrCollector) - @patch('urllib2.urlopen') + @patch(URLOPEN) @patch.object(Collector, 'publish') def test_should_work_with_real_data(self, publish_mock, urlopen_mock): returns = [self.getFixture('cores'), @@ -137,7 +137,7 @@ def test_should_work_with_real_data(self, publish_mock, urlopen_mock): call('http://localhost:8983/solr/admin/system?stats=true&wt=json') ]) - @patch('urllib2.urlopen') + @patch(URLOPEN) @patch.object(Collector, 'publish') def test_should_fail_gracefully(self, publish_mock, urlopen_mock): urlopen_mock.return_value = self.getFixture('stats_blank') From 73207d04e0739a4ce92bc201b36681c42d9fa7e7 Mon Sep 17 00:00:00 2001 From: Grant Ridder Date: Sat, 11 Mar 2017 00:25:37 -0800 Subject: [PATCH 33/33] Fix several pep8 errors --- src/collectors/eventstoreprojections/eventstoreprojections.py | 4 +++- src/collectors/jolokia/jolokia.py | 4 ++-- src/collectors/nginx/nginx.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/collectors/eventstoreprojections/eventstoreprojections.py b/src/collectors/eventstoreprojections/eventstoreprojections.py index 7a58e5fe3..38cc5debb 100644 --- a/src/collectors/eventstoreprojections/eventstoreprojections.py +++ b/src/collectors/eventstoreprojections/eventstoreprojections.py @@ -21,6 +21,7 @@ import diamond.collector import diamond.pycompat + class EventstoreProjectionsCollector(diamond.collector.Collector): def get_default_config_help(self): @@ -90,7 +91,8 @@ def collect(self): self.config['route'] ) - req = diamond.pycompat.Request(eventstore_host, headers=self.config['headers']) + req = diamond.pycompat.Request(eventstore_host, + headers=self.config['headers']) req.add_header('Content-type', 'application/json') try: diff --git a/src/collectors/jolokia/jolokia.py b/src/collectors/jolokia/jolokia.py index 72e9a9461..899d0073e 100644 --- a/src/collectors/jolokia/jolokia.py +++ b/src/collectors/jolokia/jolokia.py @@ -231,7 +231,7 @@ def _list_request(self): # timeout is lower than the interval. timeout = max(2, float(self.config['interval']) * 2 / 3) with closing(diamond.pycompat.urlopen(self._create_request(url), - timeout=timeout)) as response: + timeout=timeout)) as response: return self._read_json(response) except (HTTPError, ValueError) as e: self.log.error('Unable to read JSON response: %s', str(e)) @@ -254,7 +254,7 @@ def _read_request(self, domain): # timeout is lower than the interval. timeout = max(2, float(self.config['interval']) * 2 / 3) with closing(diamond.pycompat.urlopen(self._create_request(url), - timeout=timeout)) as response: + timeout=timeout)) as response: return self._read_json(response) except (HTTPError, ValueError): self.log.error('Unable to read JSON response.') diff --git a/src/collectors/nginx/nginx.py b/src/collectors/nginx/nginx.py index faba5f430..9e1bef749 100644 --- a/src/collectors/nginx/nginx.py +++ b/src/collectors/nginx/nginx.py @@ -1,4 +1,4 @@ - # coding=utf-8 +# coding=utf-8 """ Collect statistics from Nginx and Nginx+