From 43bc627b1dc2e667ad2e6e8a0c2ff1e0b512dcdf Mon Sep 17 00:00:00 2001 From: Ilya Mashchenko Date: Fri, 31 Jan 2020 16:19:15 +0300 Subject: /collectors/python.d/varnish: collect smf metrics (#7926) * /collectors/python.d/varnish: collect per smf g_bytes g_space * /collectors/python.d/varnish: update README.md --- collectors/python.d.plugin/varnish/README.md | 91 +++++---------- .../python.d.plugin/varnish/varnish.chart.py | 128 +++++++++++++++------ 2 files changed, 122 insertions(+), 97 deletions(-) (limited to 'collectors') diff --git a/collectors/python.d.plugin/varnish/README.md b/collectors/python.d.plugin/varnish/README.md index 4de883d319..f254e9a06d 100644 --- a/collectors/python.d.plugin/varnish/README.md +++ b/collectors/python.d.plugin/varnish/README.md @@ -1,80 +1,41 @@ # varnish -Module uses the `varnishstat` command to provide varnish cache statistics. +This module will monitor [`Varnish Cache`](https://varnish-cache.org/) performance metrics. -It produces: +It uses the `varnishstat` command to provide Varnish Cache statistics. -1. **Connections Statistics** in connections/s +## Requirements - - accepted - - dropped +- `netdata` user must be a member of the `varnish` group -2. **Client Requests** in requests/s +## Charts - - received +This module produces the following charts: -3. **All History Hit Rate Ratio** in percent +- Connections Statistics in `connections/s` +- Client Requests in `requests/s` +- All History Hit Rate Ratio in `percent` +- Current Poll Hit Rate Ratio in `percent` +- Expired Objects in `expired/s` +- Least Recently Used Nuked Objects in `nuked/s` +- Number Of Threads In All Pools in `pools` +- Threads Statistics in `threads/s` +- Current Queue Length in `requests` +- Backend Connections Statistics in `connections/s` +- Requests To The Backend in `requests/s` +- ESI Statistics in `problems/s` +- Memory Usage in `MiB` +- Uptime in `seconds` - - hit - - miss - - hitpass +For every backend (VBE): -4. **Current Poll Hit Rate Ratio** in percent +- Backend Response Statistics in `kilobits/s` - - hit - - miss - - hitpass +For every disk (SMF): -5. **Expired Objects** in expired/s +- Disk Usage in `KiB` - - objects - -6. **Least Recently Used Nuked Objects** in nuked/s - - - objects - -7. **Number Of Threads In All Pools** in threads - - - threads - -8. **Threads Statistics** in threads/s - - - created - - failed - - limited - -9. **Current Queue Length** in requests - - - in queue - -10. **Backend Connections Statistics** in connections/s - - - successful - - unhealthy - - reused - - closed - - resycled - - failed - -11. **Requests To The Backend** in requests/s - - - received - -12. **ESI Statistics** in problems/s - - - errors - - warnings - -13. **Memory Usage** in MB - - - free - - allocated - -14. **Uptime** in seconds - - - uptime - -## configuration +## Configuration Only one parameter is supported: @@ -82,7 +43,7 @@ Only one parameter is supported: instance_name: 'name' ``` -The name of the varnishd instance to get logs from. If not specified, the host name is used. +The name of the `varnishd` instance to get logs from. If not specified, the host name is used. --- diff --git a/collectors/python.d.plugin/varnish/varnish.chart.py b/collectors/python.d.plugin/varnish/varnish.chart.py index b0770d1c03..15cb6c1016 100644 --- a/collectors/python.d.plugin/varnish/varnish.chart.py +++ b/collectors/python.d.plugin/varnish/varnish.chart.py @@ -135,6 +135,44 @@ CHARTS = { } } + +def backend_charts_template(name): + order = [ + '{0}_response_statistics'.format(name), + ] + + charts = { + order[0]: { + 'options': [None, 'Backend "{0}"'.format(name), 'kilobits/s', 'backend response statistics', + 'varnish.backend', 'area'], + 'lines': [ + ['{0}_beresp_hdrbytes'.format(name), 'header', 'incremental', 8, 1000], + ['{0}_beresp_bodybytes'.format(name), 'body', 'incremental', -8, 1000] + ] + }, + } + + return order, charts + + +def disk_charts_template(name): + order = [ + 'disk_{0}_usage'.format(name), + ] + + charts = { + order[0]: { + 'options': [None, 'Disk "{0}" Usage'.format(name), 'KiB', 'disk usage', 'varnish.disk_usage', 'stacked'], + 'lines': [ + ['{0}.g_space'.format(name), 'free', 'absolute', 1, 1 << 10], + ['{0}.g_bytes'.format(name), 'allocated', 'absolute', 1, 1 << 10] + ] + }, + } + + return order, charts + + VARNISHSTAT = 'varnishstat' re_version = re.compile(r'varnish-(?P\d+)\.(?P\d+)\.(?P\d+)') @@ -188,6 +226,8 @@ class Service(ExecutableService): self.instance_name = configuration.get('instance_name') self.parser = Parser() self.command = None + self.collected_vbe = set() + self.collected_smf = set() def create_command(self): varnishstat = find_binary(VARNISHSTAT) @@ -206,10 +246,7 @@ class Service(ExecutableService): ver = parse_varnish_version(reply) if not ver: self.error("failed to parse reply from '{0}', used regex :'{1}', reply : {2}".format( - ' '.join(command), - re_version.pattern, - reply, - )) + ' '.join(command), re_version.pattern, reply)) return False if self.instance_name: @@ -241,9 +278,6 @@ class Service(ExecutableService): self.error('cant parse the output...') return False - if self.parser.re_backend: - backends = [b[0] for b in self.parser.backend_stats(reply)[::2]] - self.create_backends_charts(backends) return True def get_data(self): @@ -260,11 +294,11 @@ class Service(ExecutableService): if not server_stats: return None - if self.parser.re_backend: - backend_stats = self.parser.backend_stats(raw) - data.update(dict(('_'.join([name, param]), value) for name, param, value in backend_stats)) + stats = dict((param, value) for _, param, value in server_stats) + data.update(stats) - data.update(dict((param, value) for _, param, value in server_stats)) + self.get_vbe_backends(data, raw) + self.get_smf_disks(server_stats) # varnish 5 uses default.g_bytes and default.g_space data['memory_allocated'] = data.get('s0.g_bytes') or data.get('default.g_bytes') @@ -272,27 +306,57 @@ class Service(ExecutableService): return data - def create_backends_charts(self, backends): - for backend in backends: - chart_name = ''.join([backend, '_response_statistics']) - title = 'Backend "{0}"'.format(backend.capitalize()) - hdr_bytes = ''.join([backend, '_beresp_hdrbytes']) - body_bytes = ''.join([backend, '_beresp_bodybytes']) - - chart = { - chart_name: - { - 'options': [None, title, 'kilobits/s', 'backend response statistics', - 'varnish.backend', 'area'], - 'lines': [ - [hdr_bytes, 'header', 'incremental', 8, 1000], - [body_bytes, 'body', 'incremental', -8, 1000] - ] - } - } - - self.order.insert(0, chart_name) - self.definitions.update(chart) + def get_vbe_backends(self, data, raw): + if not self.parser.re_backend: + return + stats = self.parser.backend_stats(raw) + if not stats: + return + + for (name, param, value) in stats: + data['_'.join([name, param])] = value + if name in self.collected_vbe: + continue + self.collected_vbe.add(name) + self.add_backend_charts(name) + + def get_smf_disks(self, server_stats): + # [('SMF.', 'ssdStorage.c_req', '47686'), + # ('SMF.', 'ssdStorage.c_fail', '0'), + # ('SMF.', 'ssdStorage.c_bytes', '668102656'), + # ('SMF.', 'ssdStorage.c_freed', '140980224'), + # ('SMF.', 'ssdStorage.g_alloc', '39753'), + # ('SMF.', 'ssdStorage.g_bytes', '527122432'), + # ('SMF.', 'ssdStorage.g_space', '53159968768'), + # ('SMF.', 'ssdStorage.g_smf', '40130'), + # ('SMF.', 'ssdStorage.g_smf_frag', '311'), + # ('SMF.', 'ssdStorage.g_smf_large', '66')] + disks = [name for typ, name, _ in server_stats if typ.startswith('SMF') and name.endswith('g_space')] + if not disks: + return + for disk in disks: + disk = disk.split('.')[0] # ssdStorage + if disk in self.collected_smf: + continue + self.collected_smf.add(disk) + self.add_disk_charts(disk) + + def add_backend_charts(self, backend_name): + self.add_charts(backend_name, backend_charts_template) + + def add_disk_charts(self, disk_name): + self.add_charts(disk_name, disk_charts_template) + + def add_charts(self, name, charts_template): + order, charts = charts_template(name) + + for chart_name in order: + params = [chart_name] + charts[chart_name]['options'] + dimensions = charts[chart_name]['lines'] + + new_chart = self.charts.add_chart(params) + for dimension in dimensions: + new_chart.add_dimension(dimension) def parse_varnish_version(lines): -- cgit v1.2.3