From de6035c543296de179e359a415664106d66c3878 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Thu, 14 Jan 2021 12:50:26 +0000 Subject: Anomalies collector custom model bugfix for issue #10456 (#10459) * Fix for #10456 - Add "|" as delimiter in custom_models_dims_renamed as opposed to ".". The "|" is how the collector knows what cols to pull from the dataframe for each model. Expected colnames in the df are "model_name|dim...." so it looks for "model_name" for example. - Remove `[self.expected_cols]` check on df_allmetrics. This will avoid failing if you happen to use a dim in a custom model that belongs to a chart that has one or more `null` values from /api/v1/allmetrics. If you happen to use such a dim in your model it will still fail as we can't make a prediction when one of the features is `null`. * add filters for some irrelevant warnings * add newline at end of file * fully remove self.expected_cols since no longer used --- collectors/python.d.plugin/anomalies/anomalies.chart.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'collectors') diff --git a/collectors/python.d.plugin/anomalies/anomalies.chart.py b/collectors/python.d.plugin/anomalies/anomalies.chart.py index cacc9ce433..3691537b4b 100644 --- a/collectors/python.d.plugin/anomalies/anomalies.chart.py +++ b/collectors/python.d.plugin/anomalies/anomalies.chart.py @@ -73,7 +73,6 @@ class Service(SimpleService): self.fitted_at = {} self.df_allmetrics = pd.DataFrame() self.data_latest = {} - self.expected_cols = [] self.last_train_at = 0 self.include_average_prob = bool(self.configuration.get('include_average_prob', True)) @@ -101,7 +100,7 @@ class Service(SimpleService): self.custom_models_host_charts_dict = {} for host in self.custom_models_hosts: self.custom_models_host_charts_dict[host] = list(set([dim.split('::')[1].split('|')[0] for dim in self.custom_models_dims if dim.startswith(host)])) - self.custom_models_dims_renamed = [f"{model['name']}.{dim}" for model in self.custom_models for dim in model['dimensions'].split(',')] + self.custom_models_dims_renamed = [f"{model['name']}|{dim}" for model in self.custom_models for dim in model['dimensions'].split(',')] self.models_in_scope = list(set([f'{self.host}::{c}' for c in self.charts_in_scope] + self.custom_models_names)) self.charts_in_scope = list(set(self.charts_in_scope + self.custom_models_charts)) self.host_charts_dict = {self.host: self.charts_in_scope} @@ -245,7 +244,6 @@ class Service(SimpleService): host_charts_dict=self.host_charts_dict, host_prefix=True, host_sep='::', after=after, before=before, sort_cols=True, numeric_only=True, protocol=self.protocol, float_size='float32', user=self.username, pwd=self.password ).ffill() - self.expected_cols = list(df_train.columns) if self.custom_models: df_train = self.add_custom_models_dims(df_train) @@ -287,7 +285,7 @@ class Service(SimpleService): df_allmetrics = get_allmetrics_async( host_charts_dict=self.host_charts_dict, host_prefix=True, host_sep='::', wide=True, sort_cols=True, protocol=self.protocol, numeric_only=True, float_size='float32', user=self.username, pwd=self.password - )[self.expected_cols] + ) if self.custom_models: df_allmetrics = self.add_custom_models_dims(df_allmetrics) self.df_allmetrics = self.df_allmetrics.append(df_allmetrics).ffill().tail((max(self.lags_n.values()) + max(self.smooth_n.values()) + max(self.diffs_n.values())) * 2) -- cgit v1.2.3