diff options
author | Ilya Mashchenko <ilyamaschenko@gmail.com> | 2019-03-11 16:58:00 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2019-03-11 16:58:00 +0300 |
commit | 28e0d7526d51ea813ac6a28b95a4ab6997394d19 (patch) | |
tree | c1fa6f31a16c18ee7fbf349599aa40c3077f9b5a | |
parent | 30e49cdc1532ae506709db4f82d45313b7c81506 (diff) |
python loaders cleanup (#5602)
<!--
Describe the change in summary section, including rationale and degin decisions.
Include "Fixes #nnn" if you are fixing an existing issue.
In "Component Name" section write which component is changed in this PR. This
will help us review your PR quicker.
If you have more information you want to add, write them in "Additional
Information" section. This is usually used to help others understand your
motivation behind this change. A step-by-step reproduction of the problem is
helpful if there is no related issue.
-->
##### Summary
python loaders module cleanup
- move `load_module` to `python.d.plugin`
- use `load_config` in unbound module instead of YamlOrderedLoad class
- remove all classes from `loaders.py` - no longer used
##### Component Name
[`collectors/python.d.plugin/python_modules/bases/loaders`](https://github.com/netdata/netdata/blob/master/collectors/python.d.plugin/python_modules/bases/loaders.py)
##### Additional Information
-rw-r--r-- | collectors/python.d.plugin/python.d.plugin.in | 22 | ||||
-rw-r--r-- | collectors/python.d.plugin/python_modules/bases/loaders.py | 63 | ||||
-rw-r--r-- | collectors/python.d.plugin/unbound/unbound.chart.py | 10 |
3 files changed, 28 insertions, 67 deletions
diff --git a/collectors/python.d.plugin/python.d.plugin.in b/collectors/python.d.plugin/python.d.plugin.in index 8ce621e0bd..cca7346a1e 100644 --- a/collectors/python.d.plugin/python.d.plugin.in +++ b/collectors/python.d.plugin/python.d.plugin.in @@ -18,6 +18,15 @@ import re import sys import time import threading +import types + +PY_VERSION = sys.version_info[:2] + +if PY_VERSION > (3, 1): + from importlib.machinery import SourceFileLoader +else: + from imp import load_source as SourceFileLoader + ENV_NETDATA_USER_CONFIG_DIR = 'NETDATA_USER_CONFIG_DIR' ENV_NETDATA_STOCK_CONFIG_DIR = 'NETDATA_STOCK_CONFIG_DIR' @@ -73,7 +82,6 @@ sys.path.append(DIRS.pythond_packages) from bases.collection import safe_print from bases.loggers import PythonDLogger from bases.loaders import load_config -from bases.loaders import load_module as _load_module try: from collections import OrderedDict @@ -138,7 +146,10 @@ class HeartBeat(threading.Thread): def load_module(name): abs_path = os.path.join(DIRS.modules, '{0}{1}'.format(name, MODULE_SUFFIX)) - return _load_module(name, abs_path) + module = SourceFileLoader(name, abs_path) + if isinstance(module, types.ModuleType): + return module + return module.load_module() def multi_path_find(name, paths): @@ -504,6 +515,8 @@ class Plugin: self.task_queue.join() self.dequeue_results() self.result_queue.join() + self.task_queue.close() + self.result_queue.close() self.log.info('stopping checker process') worker.join() @@ -588,7 +601,8 @@ class Plugin: while True: self.runs += 1 - if threading.active_count() <= 3 and not self.auto_detection_jobs: + # threads: main + heartbeat + if threading.active_count() <= 2 and not self.auto_detection_jobs: return time.sleep(1) @@ -678,7 +692,7 @@ def main(): if cmd.trace: logger.log_traceback = True - logger.info('using python v{0}'.format(sys.version_info[:2][0])) + logger.info('using python v{0}'.format(PY_VERSION[0])) unknown_modules = set(cmd.modules_to_run) - set(AVAILABLE_MODULES) if unknown_modules: diff --git a/collectors/python.d.plugin/python_modules/bases/loaders.py b/collectors/python.d.plugin/python_modules/bases/loaders.py index 921a09f917..d8b2ec8159 100644 --- a/collectors/python.d.plugin/python_modules/bases/loaders.py +++ b/collectors/python.d.plugin/python_modules/bases/loaders.py @@ -3,7 +3,6 @@ # Author: Ilya Mashchenko (l2isbad) # SPDX-License-Identifier: GPL-3.0-or-later -import types from sys import version_info @@ -18,78 +17,22 @@ except ImportError: from yaml import SafeLoader as YamlSafeLoader -if PY_VERSION > (3, 1): - from importlib.machinery import SourceFileLoader - DEFAULT_MAPPING_TAG = 'tag:yaml.org,2002:map' -else: - from imp import load_source as SourceFileLoader - DEFAULT_MAPPING_TAG = u'tag:yaml.org,2002:map' - try: from collections import OrderedDict except ImportError: from third_party.ordereddict import OrderedDict -def dict_constructor(loader, node): - return OrderedDict(loader.construct_pairs(node)) +DEFAULT_MAPPING_TAG = 'tag:yaml.org,2002:map' if PY_VERSION > (3, 1) else u'tag:yaml.org,2002:map' -def safe_load(stream): - loader = YamlSafeLoader(stream) - try: - return loader.get_single_data() - finally: - loader.dispose() +def dict_constructor(loader, node): + return OrderedDict(loader.construct_pairs(node)) YamlSafeLoader.add_constructor(DEFAULT_MAPPING_TAG, dict_constructor) -class YamlOrderedLoader: - @staticmethod - def load_config_from_file(file_name): - opened, loaded = False, False - try: - stream = open(file_name, 'r') - opened = True - loader = YamlSafeLoader(stream) - loaded = True - parsed = loader.get_single_data() or dict() - except Exception as error: - return dict(), error - else: - return parsed, None - finally: - if opened: - stream.close() - if loaded: - loader.dispose() - - -class SourceLoader: - @staticmethod - def load_module_from_file(name, path): - try: - loaded = SourceFileLoader(name, path) - if isinstance(loaded, types.ModuleType): - return loaded, None - return loaded.load_module(), None - except Exception as error: - return None, error - - -class ModuleAndConfigLoader(YamlOrderedLoader, SourceLoader): - pass - - -def load_module(name, path): - module = SourceFileLoader(name, path) - if isinstance(module, types.ModuleType): - return module - return module.load_module() - - def load_yaml(stream): loader = YamlSafeLoader(stream) try: diff --git a/collectors/python.d.plugin/unbound/unbound.chart.py b/collectors/python.d.plugin/unbound/unbound.chart.py index adb58d4176..dade2b2041 100644 --- a/collectors/python.d.plugin/unbound/unbound.chart.py +++ b/collectors/python.d.plugin/unbound/unbound.chart.py @@ -9,7 +9,7 @@ import sys from copy import deepcopy from bases.FrameworkServices.SocketService import SocketService -from bases.loaders import YamlOrderedLoader +from bases.loaders import load_config PRECISION = 1000 @@ -169,7 +169,7 @@ def _get_perthread_info(thread): for key, value in PER_THREAD_STAT_MAP.items(): statmap[key.format(shortname=sname)] = (value[0].format(shortname=sname), value[1]) - return (charts, order, statmap) + return charts, order, statmap class Service(SocketService): @@ -205,7 +205,11 @@ class Service(SocketService): def _auto_config(self): if self.ubconf and os.access(self.ubconf, os.R_OK): self.debug('Unbound config: {0}'.format(self.ubconf)) - conf = YamlOrderedLoader.load_config_from_file(self.ubconf)[0] + conf = dict() + try: + conf = load_config(self.ubconf) + except Exception as error: + self.error("error on loading '{0}' : {1}".format(self.ubconf, error)) if self.ext is None: if 'extended-statistics' in conf['server']: self.ext = conf['server']['extended-statistics'] |