phpunit.xml


1
2
3
4
5
6
7

<phpunit bootstrap="tests/classloader.php">
<testsuites>
    <testsuite name="unit">
        <directory>./tests/unit</directory>
    </testsuite>
</testsuites>
</phpunit>
#!/usr/bin/env bash
'''':; exec "$(command -v python || command -v python3 || command -v python2 || echo "ERROR python IS NOT AVAILABLE IN THIS SYSTEM")" "$0" "$@" # '''
# -*- coding: utf-8 -*-

# Description: netdata python modules supervisor
# Author: Pawel Krupa (paulfantom)

import os
import sys
import time
import threading
from re import sub

# -----------------------------------------------------------------------------
# globals & environment setup
# https://github.com/firehol/netdata/wiki/External-Plugins#environment-variables
MODULE_EXTENSION = ".chart.py"
BASE_CONFIG = {'update_every': os.getenv('NETDATA_UPDATE_EVERY', 1),
               'priority': 90000,
               'retries': 10}

MODULES_DIR = os.path.abspath(os.getenv('NETDATA_PLUGINS_DIR',
                                        os.path.dirname(__file__)) + "/../python.d") + "/"
CONFIG_DIR = os.getenv('NETDATA_CONFIG_DIR', "/etc/netdata/")
# directories should end with '/'
if CONFIG_DIR[-1] != "/":
    CONFIG_DIR += "/"
sys.path.append(MODULES_DIR + "python_modules")

PROGRAM = os.path.basename(__file__).replace(".plugin", "")
DEBUG_FLAG = False
TRACE_FLAG = False
OVERRIDE_UPDATE_EVERY = False

# -----------------------------------------------------------------------------
# custom, third party and version specific python modules management
import msg

try:
    assert sys.version_info >= (3, 1)
    import importlib.machinery
    PY_VERSION = 3
    # change this hack below if we want PY_VERSION to be used in modules
    # import builtins
    # builtins.PY_VERSION = 3
    msg.info('Using python v3')
except (AssertionError, ImportError):
    try:
        import imp

        # change this hack below if we want PY_VERSION to be used in modules
        # import __builtin__
        # __builtin__.PY_VERSION = 2
        PY_VERSION = 2
        msg.info('Using python v2')
    except ImportError:
        msg.fatal('Cannot start. No importlib.machinery on python3 or lack of imp on python2')
# try:
#     import yaml
# except ImportError:
#     msg.fatal('Cannot find yaml library')
try:
    if PY_VERSION == 3:
        import pyyaml3 as yaml
    else:
        import pyyaml2 as yaml
except ImportError:
    msg.fatal('Cannot find yaml library')

try:
    from collections import OrderedDict
    ORDERED = True
    DICT = OrderedDict
    msg.info('YAML output is ordered')
except ImportError:
    ORDERED = False
    DICT = dict
    msg.info('YAML output is unordered')
else:
    def ordered_load(stream, Loader=yaml.Loader, object_pairs_hook=OrderedDict):
        class OrderedLoader(Loader):
            pass
        def construct_mapping(loader, node):
           loader.flatten_mapping(node)
           return object_pairs_hook(loader.construct_pairs(node))
        OrderedLoader.add_constructor(
            yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
            construct_mapping)
        return yaml.load(stream, OrderedLoader)

class PythonCharts(object):
    """
    Main class used to control every python module.
    """

    def __init__(self,
                 modules=None,
                 modules_path='../python.d/',
                 modules_configs='../conf.d/',
                 modules_disabled=None):
        """
        :param modules: list
        :param modules_path: str
        :param modules_configs: str
        :param modules_disabled: list
        """

        if modules is None:
            modules = []
        if modules_disabled is None:
            modules_disabled = []

        self.first_run = True
        # set configuration directory
        self.configs = modules_configs

        # load modules
        loaded_modules = self._load_modules(modules_path, modules, modules_disabled)

        # load configuration files
        configured_modules = self._load_configs(loaded_modules)

        # good economy and prosperity:
        self.jobs = self._create_jobs(configured_modules)  # type: list

        # enable timetable override like `python.d.plugin mysql debug 1`
        if DEBUG_FLAG and OVERRIDE_UPDATE_EVERY:
            for job in self.jobs:
                job.create_timetable(BASE_CONFIG['update_every'])

    @staticmethod
    def _import_module(path, name=None):
        """
        Try to import module using only its path.
        :param path: str
        :param name: str
        :return: object
        """

        if name is None:
            name = path.split('/')[-1]
            if name[-len(MODULE_EXTENSION):] != MODULE_EXTENSION:
                return None
            name = name[:-len(MODULE_EXTENSION)]
        try:
            if PY_VERSION == 3:
                return importlib.machinery.SourceFileLoader(name, path).load_module()
            else:
                return imp.load_source(name, path)
        except Exception as e:
            msg.error("Problem loading", name, str(e))
            return None

    def _load_modules(self, path, modules, disabled):
        """
        Load modules from 'modules' list or dynamically every file from 'path' (only .chart.py files)
        :param path: str
        :param modules: list
        :param disabled: list
        :return: list
        """

        # check if plugin directory exists
        if not os.path.isdir(path):
            msg.fatal("cannot find charts directory ", path)

        # load modules
        loaded = []
        if len(modules) > 0:
            for m in modules:
                if m in disabled:
                    continue
                mod = self._import_module(path + m + MODULE_EXTENSION)
                if mod is not None:
                    loaded.append(mod)
                else:  # exit if plugin is not found
                    msg.fatal('no modules found.')
        else:
            # scan directory specified in path and load all modules from there
            names = os.listdir(path)
            for mod in names:
                if mod.replace(MODULE_EXTENSION, "") in disabled:
                    msg.error(mod + ": disabled module ", mod.replace(MODULE_EXTENSION, ""))
                    continue
                m = self._import_module(path + mod)
                if m is not None:
                    msg.debug(mod + ": loading module '" + path + mod + "'")
                    loaded.append(m)
        return loaded

    def _load_configs(self, modules):
        """
        Append configuration in list named `config` to every module.
        For multi-job modules `config` list is created in _parse_config,
        otherwise it is created here based on BASE_CONFIG prototype with None as identifier.
        :param modules: list
        :return: list
        """
        for mod in modules:
            configfile = self.configs + mod.__name__ + ".conf"
            if os.path.isfile(configfile):
                msg.debug(mod.__name__ + ": loading module configuration: '" + configfile + "'")
                try:
                    if not hasattr(mod, 'config'):
                        mod.config = {}
                    setattr(mod,
                            'config',
                            self._parse_config(mod, read_config(configfile)))
                except Exception as e:
                    msg.error(mod.__name__ + ": cannot parse configuration file '" + configfile + "':", str(e))
            else:
                msg.error(mod.__name__ + ": configuration file '" + configfile + "' not found. Using defaults.")
                # set config if not found
                if not hasattr(mod, 'config'):
                    msg.debug(mod.__name__ + ": setting configuration for only one job")
                    mod.config = {None: {}}
                    for var in BASE_CONFIG:
                        try:
                            mod.config[None][var] = getattr(mod, var)
                        except AttributeError:
                            mod.config[None][var] = BASE_CONFIG[var]
        return modules

    @staticmethod
    def _parse_config(module, config):
        """
        Parse configuration file or extract configuration from module file.
        Example of returned dictionary:
            config = {'name': {
                            'update_every': 2,
                            'retries': 3,
                            'priority': 30000
                            'other_val': 123}}
        :param module: object
        :param config: dict
        :return: dict
        """
        if config is None:
            config = {}
        # get default values
        defaults = {}
        msg.debug(module.__name__ + ": reading configuration")
        for key in BASE_CONFIG:
            try:
                # get defaults from module config
                defaults[key] = int(config.pop(key))
            except (KeyError, ValueError):
                try:
                    # get defaults from module source code
                    defaults[key] = getattr(module, key)
                except (KeyError, ValueError, AttributeError):
                    # if above failed, get defaults from global dict
                    defaults[key] = BASE_CONFIG[key]

        # check if there are dict in config dict
        many_jobs = False
        for name in config:
            if isinstance(config[name], DICT):
                many_jobs = True
                break

        # assign variables needed by supervisor to every job configuration
        if many_jobs:
            for name in config:
                for key in defaults:
                    if key not in config[name]:
                        config[name][key] = defaults[key]
        # if only one job is needed, values doesn't have to be in dict (in YAML)
        else:
            config = {None: config.copy()}
            config[None].update(defaults)

        # return dictionary of jobs where every job has BASE_CONFIG variables
        return config

    @staticmethod
    def _create_jobs(modules):
        """
        Create jobs based on module.config dictionary and module.Service class definition.
        :param modules: list
        :return: list
        """
        jobs = []
        for module in modules:
            for name in module.config:
                # register a new job