From 5c1a47c9366401c2bddecbb3425eec46400ae860 Mon Sep 17 00:00:00 2001 From: Ilya Mashchenko Date: Thu, 13 Sep 2018 16:23:24 +0900 Subject: rethinkdb python plugin (#3955) * rethinkdb python module init version * python readme: add rethinkdb --- conf.d/Makefile.am | 1 + conf.d/python.d.conf | 1 + conf.d/python.d/rethinkdbs.conf | 78 +++++++++++++ python.d/Makefile.am | 1 + python.d/README.md | 35 ++++++ python.d/rethinkdbs.chart.py | 235 ++++++++++++++++++++++++++++++++++++++++ web/dashboard_info.js | 6 + 7 files changed, 357 insertions(+) create mode 100644 conf.d/python.d/rethinkdbs.conf create mode 100644 python.d/rethinkdbs.chart.py diff --git a/conf.d/Makefile.am b/conf.d/Makefile.am index e1bc551122..5ef5ebd7c8 100644 --- a/conf.d/Makefile.am +++ b/conf.d/Makefile.am @@ -71,6 +71,7 @@ dist_pythonconfig_DATA = \ python.d/puppet.conf \ python.d/rabbitmq.conf \ python.d/redis.conf \ + python.d/rethinkdbs.conf \ python.d/retroshare.conf \ python.d/samba.conf \ python.d/sensors.conf \ diff --git a/conf.d/python.d.conf b/conf.d/python.d.conf index dac7fc7852..f26c068271 100644 --- a/conf.d/python.d.conf +++ b/conf.d/python.d.conf @@ -73,6 +73,7 @@ nginx_log: no # puppet: yes # rabbitmq: yes # redis: yes +# rethinkdbs: yes # retroshare: yes # sensors: yes # spigotmc: yes diff --git a/conf.d/python.d/rethinkdbs.conf b/conf.d/python.d/rethinkdbs.conf new file mode 100644 index 0000000000..73544fc2e9 --- /dev/null +++ b/conf.d/python.d/rethinkdbs.conf @@ -0,0 +1,78 @@ +# netdata python.d.plugin configuration for rethinkdb +# +# This file is in YaML format. Generally the format is: +# +# name: value +# +# There are 2 sections: +# - global variables +# - one or more JOBS +# +# JOBS allow you to collect values from multiple sources. +# Each source will have its own set of charts. +# +# JOB parameters have to be indented (using spaces only, example below). + +# ---------------------------------------------------------------------- +# Global Variables +# These variables set the defaults for all JOBs, however each JOB +# may define its own, overriding the defaults. + +# update_every sets the default data collection frequency. +# If unset, the python.d.plugin default is used. +# update_every: 1 + +# priority controls the order of charts at the netdata dashboard. +# Lower numbers move the charts towards the top of the page. +# If unset, the default for python.d.plugin is used. +# priority: 60000 + +# retries sets the number of retries to be made in case of failures. +# If unset, the default for python.d.plugin is used. +# Attempts to restore the service are made once every update_every +# and only if the module has collected values in the past. +# retries: 60 + +# autodetection_retry sets the job re-check interval in seconds. +# The job is not deleted if check fails. +# Attempts to start the job are made once every autodetection_retry. +# This feature is disabled by default. +# autodetection_retry: 0 + +# ---------------------------------------------------------------------- +# JOBS (data collection sources) +# +# The default JOBS share the same *name*. JOBS with the same name +# are mutually exclusive. Only one of them will be allowed running at +# any time. This allows autodetection to try several alternatives and +# pick the one that works. +# +# Any number of jobs is supported. +# +# All python.d.plugin JOBS (for all its modules) support a set of +# predefined parameters. These are: +# +# job_name: +# name: myname # the JOB's name as it will appear at the +# # dashboard (by default is the job_name) +# # JOBs sharing a name are mutually exclusive +# update_every: 1 # the JOB's data collection frequency +# priority: 60000 # the JOB's order on the dashboard +# retries: 60 # the JOB's number of restoration attempts +# autodetection_retry: 0 # the JOB's re-check interval in seconds +# +# Additionally to the above, rethinkdb also supports the following: +# +# host: IP or HOSTNAME # default is 'localhost' +# port: PORT # default is 28015 +# user: USERNAME # default is 'admin' +# password: PASSWORD # not set by default +# timeout: TIMEOUT # default is 2 + +# ---------------------------------------------------------------------- +# AUTO-DETECTION JOBS +# only one of them will run (they have the same name) + +local: + name: 'local' + host: 'localhost' diff --git a/python.d/Makefile.am b/python.d/Makefile.am index c04bd5a1af..018500ff94 100644 --- a/python.d/Makefile.am +++ b/python.d/Makefile.am @@ -57,6 +57,7 @@ dist_python_DATA = \ puppet.chart.py \ rabbitmq.chart.py \ redis.chart.py \ + rethinkdbs.chart.py \ retroshare.chart.py \ samba.chart.py \ sensors.chart.py \ diff --git a/python.d/README.md b/python.d/README.md index 90e538aa67..62026d5420 100644 --- a/python.d/README.md +++ b/python.d/README.md @@ -2227,6 +2227,41 @@ When no configuration file is found, module tries to connect to TCP/IP socket: ` --- +# rethinkdb + +Module monitor rethinkdb health metrics. + +Following charts are drawn: + +1. **Connected Servers** + * connected + * missing + +2. **Active Clients** + * active + +3. **Queries** per second + * queries + +4. **Documents** per second + * documents + +### configuration + +```yaml + +localhost: + name : 'local' + host : '127.0.0.1' + port : 28015 + user : "user" + password : "pass" +``` + +When no configuration file is found, module tries to connect to `127.0.0.1:28015`. + +--- + # samba Performance metrics of Samba file sharing. diff --git a/python.d/rethinkdbs.chart.py b/python.d/rethinkdbs.chart.py new file mode 100644 index 0000000000..a5f825ae1b --- /dev/null +++ b/python.d/rethinkdbs.chart.py @@ -0,0 +1,235 @@ +# -*- coding: utf-8 -*- +# Description: rethinkdb netdata python.d module +# Author: Ilya Mashchenko (l2isbad) +# SPDX-License-Identifier: GPL-3.0+ + +try: + import rethinkdb as rdb + HAS_RETHINKDB = True +except ImportError: + HAS_RETHINKDB = False + +from bases.FrameworkServices.SimpleService import SimpleService + +ORDER = [ + 'cluster_connected_servers', + 'cluster_clients_active', + 'cluster_queries', + 'cluster_documents', +] + + +def cluster_charts(): + return { + 'cluster_connected_servers': { + 'options': [None, 'Connected Servers', 'servers', 'cluster', 'rethinkdb.cluster_connected_servers', + 'stacked'], + 'lines': [ + ['cluster_servers_connected', 'connected'], + ['cluster_servers_missing', 'missing'], + ] + }, + 'cluster_clients_active': { + 'options': [None, 'Active Clients', 'clients', 'cluster', 'rethinkdb.cluster_clients_active', + 'line'], + 'lines': [ + ['cluster_clients_active', 'active'], + ] + }, + 'cluster_queries': { + 'options': [None, 'Queries', 'queries/s', 'cluster', 'rethinkdb.cluster_queries', 'line'], + 'lines': [ + ['cluster_queries_per_sec', 'queries'], + ] + }, + 'cluster_documents': { + 'options': [None, 'Documents', 'documents/s', 'cluster', 'rethinkdb.cluster_documents', 'line'], + 'lines': [ + ['cluster_read_docs_per_sec', 'reads'], + ['cluster_written_docs_per_sec', 'writes'], + ] + }, + } + + +def server_charts(n): + o = [ + '{0}_client_connections'.format(n), + '{0}_clients_active'.format(n), + '{0}_queries'.format(n), + '{0}_documents'.format(n), + ] + f = 'server {0}'.format(n) + + c = { + o[0]: { + 'options': [None, 'Client Connections', 'connections', f, 'rethinkdb.client_connections', 'line'], + 'lines': [ + ['{0}_client_connections'.format(n), 'connections'], + ] + }, + o[1]: { + 'options': [None, 'Active Clients', 'clients', f, 'rethinkdb.clients_active', 'line'], + 'lines': [ + ['{0}_clients_active'.format(n), 'active'], + ] + }, + o[2]: { + 'options': [None, 'Queries', 'queries/s', f, 'rethinkdb.queries', 'line'], + 'lines': [ + ['{0}_queries_total'.format(n), 'queries', 'incremental'], + ] + }, + o[3]: { + 'options': [None, 'Documents', 'documents/s', f, 'rethinkdb.documents', 'line'], + 'lines': [ + ['{0}_read_docs_total'.format(n), 'reads', 'incremental'], + ['{0}_written_docs_total'.format(n), 'writes', 'incremental'], + ] + }, + } + + return o, c + + +class Cluster: + def __init__(self, raw): + self.raw = raw + + def data(self): + qe = self.raw['query_engine'] + + return { + 'cluster_clients_active': qe['clients_active'], + 'cluster_queries_per_sec': qe['queries_per_sec'], + 'cluster_read_docs_per_sec': qe['read_docs_per_sec'], + 'cluster_written_docs_per_sec': qe['written_docs_per_sec'], + 'cluster_servers_connected': 0, + 'cluster_servers_missing': 0, + } + + +class Server: + def __init__(self, raw): + self.name = raw['server'] + self.raw = raw + + def error(self): + return self.raw.get('error') + + def data(self): + qe = self.raw['query_engine'] + + d = { + 'client_connections': qe['client_connections'], + 'clients_active': qe['clients_active'], + 'queries_total': qe['queries_total'], + 'read_docs_total': qe['read_docs_total'], + 'written_docs_total': qe['written_docs_total'], + } + + return dict(('{0}_{1}'.format(self.name, k), d[k]) for k in d) + + +class Service(SimpleService): + def __init__(self, configuration=None, name=None): + SimpleService.__init__(self, configuration=configuration, name=name) + self.order = list(ORDER) + self.definitions = cluster_charts() + + self.host = self.configuration.get('host', '127.0.0.1') + self.port = self.configuration.get('port', 28015) + self.user = self.configuration.get('user', 'admin') + self.password = self.configuration.get('password') + self.timeout = self.configuration.get('timeout', 2) + + self.conn = None + self.alive = True + + def check(self): + if not HAS_RETHINKDB: + self.error('"rethinkdb" module is needed to use rethinkdbs.py') + return False + + if not self.connect(): + return None + + stats = self.get_stats() + + if not stats: + return None + + for v in stats[1:]: + if get_id(v) == 'server': + o, c = server_charts(v['server']) + self.order.extend(o) + self.definitions.update(c) + + return True + + def get_data(self): + if not self.is_alive(): + return None + + stats = self.get_stats() + + if not stats: + return None + + data = dict() + + # cluster + data.update(Cluster(stats[0]).data()) + + # servers + for v in stats[1:]: + if get_id(v) != 'server': + continue + + s = Server(v) + + if s.error(): + data['cluster_servers_missing'] += 1 + else: + data['cluster_servers_connected'] += 1 + data.update(s.data()) + + return data + + def get_stats(self): + try: + return list(rdb.db('rethinkdb').table('stats').run(self.conn).items) + except rdb.errors.ReqlError: + self.alive = False + return None + + def connect(self): + try: + self.conn = rdb.connect( + host=self.host, + port=self.port, + user=self.user, + password=self.password, + timeout=self.timeout, + ) + self.alive = True + return True + except rdb.errors.ReqlError as error: + self.error('Connection to {0}:{1} failed: {2}'.format(self.host, self.port, error)) + return False + + def reconnect(self): + # The connection is already closed after rdb.errors.ReqlError, + # so we do not need to call conn.close() + if self.connect(): + return True + return False + + def is_alive(self): + if not self.alive: + return self.reconnect() + return True + + +def get_id(v): + return v['id'][0] diff --git a/web/dashboard_info.js b/web/dashboard_info.js index 325acf0da6..e5f66aff59 100644 --- a/web/dashboard_info.js +++ b/web/dashboard_info.js @@ -220,6 +220,12 @@ netdataDashboard.menu = { info: 'Performance metrics for redis. Redis (REmote DIctionary Server) is a software project that implements data structure servers. It is open-source, networked, in-memory, and stores keys with optional durability.' }, + 'rethinkdbs': { + title: 'RethinkDB', + icon: '', + info: 'Performance metrics for rethinkdb. RethinkDB is the first open-source scalable database built for realtime applications' + }, + 'retroshare': { title: 'RetroShare', icon: '', -- cgit v1.2.3