add implementation for filtering

.gitignore

@@ -0,0 +1,3 @@
+*.gz
+*.tgz
+build/

Dockerfile

@@ -1,9 +1,11 @@
-FROM python:3.9
-
+FROM python:3.9-alpine
 
 ADD requirements.txt /
 RUN pip install -r /requirements.txt
 
+RUN apk update && apk upgrade && \
+    apk add --no-cache bash git openssh
+
 WORKDIR /src/pysunspec
 RUN git clone --recursive https://github.com/sunspec/pysunspec .
 RUN python -m unittest discover -v sunspec

Makefile

@@ -0,0 +1,8 @@
+VERSION = $(shell cat VERSION)
+
+.PHONY: build
+build:
+	docker build -t inosion/prometheus-sunspec-exporter:$(VERSION) .
+
+release:
+	docker save inosion/prometheus-sunspec-exporter:$(VERSION) | gzip > inosion-prometheus-sunspec-exporter-$(VERSION).tgz

README.md

@@ -1,14 +1,26 @@
 # Prometheus Sunspec Exporter
 
-- ~Alpha~ Beta - Works and is Pulling data :-D
+- Works and is Pulling data :-D
 - May have some re-coding to do around what consitutes a gauge, counter etc
 - is optimised to read only the "sunspec" model you desire (reduce call load on the device)
 - Need to run one exporter "per" modbus sunspec address/ip/port (current limitation)
 - Uses https://github.com/sunspec/pysunspec
 
-# Tested on
-- SMA SunnyBoy TriPower STL-6000 - Use address 126 (for SunSpec)
-- Fronius (Something) - SunSpec native
+# Sunspec Devices 
+  Sunspec is alliance of 100 Solar and Electricity Storage products, that provides a standard Modbu API.
+
+  With this API, a prometheus exporter was born, in approximately 8 hours.
+
+  Devices supported can be found here (Sunspec Modbus Certified List)[https://sunspec.org/sunspec-modbus-certified-products/]
+  Manufacturers supporting Sunspec:
+  - Fronius
+  - SMA
+  - Huawei
+  - ABB
+  - Sungrow
+
+  See (tested-devices)[docs/tested-devices.md] for more information.
+
 # Sample Grafana
 
 ![images/grafana_dash_sample_2021-01-10_13-28.png](images/grafana_dash_sample_2021-01-10_13-28.png)
@@ -126,7 +138,6 @@ which section, and then that becomes your set of model_id's.
      sudo systemctl start prometheus-sunspec-exporter.service 
      ```
 5. Configure your Prometheus to collect the data [install/prometheus.yml](install/prometheus.yml)
-   
 
 # Testing 
 
@@ -168,6 +179,30 @@ sunspec_Cabinet_Temperature_TmpCab_C{ip="192.168.1.70",port="502",target="126"}
 # HELP sunspec_Operating_State_St_total 
 ```
 
+# Filtering
+
+Some devices needs some tweaking, on the values they return.
+Some SMA Solar inverters return 3276.8 for NaN. 
+At night time, the DC Amps returns that value. Which is non too helpful in Grafana
+
+![amps_bad](images/filtering_example_amps_bad.png)
+
+To correct this, we can remap each returned value, before prometheus collects it. 
+
+```
+# Example of METRICFILTER
+--filter "Amps_Phase[ABC]_Aph[ABC]_A gt:3276 0.0"
+```
+METRICFILTER is a space separated 3-Tuple, <metric_regex> <function>:<args> <replace_value> 
+
+"Amps_Phase[ABC]_Aph[ABC]_A gt:3276 0.0"
+
+Which reads, when the metric matching regex, Amps_Phase[ABC]_Aph[ABC]_A is greater than 3276, set the metric as 0.
+In this example case, the inverter jumps from low 0.4, 0.5 Amps, up to 3276.7 (signed int16 Upper) represented as NaN. 
+
+This filter config removes that problem.
+
+
 # Testing with SunSpec
 
 See [testing/](testing/)

VERSION

@@ -0,0 +1 @@
+1.0.9

docs/tested-devices.md

@@ -0,0 +1,20 @@
+# Known Working / Tested Devices
+
+Search (Sunspec Modbus Certified List)[https://sunspec.org/sunspec-modbus-certified-products/] for your device.
+
+Because Sunspec is a standard, the configuration required, for the exporter, will vary only:
+
+1. Which Modbus address the device listens on
+2. The `model_ids` you need from the device (don't need the serial number 6 times a minute :-) )
+3. Any custom filterng required
+
+You won't need to map any data fields, the exporter does all that work.
+
+# Devices Working
+
+   |--------------|----------------------------|--------------------------------------------------------|---------------|
+   | Manufacturer | Model                      | Notes                                                  | Sample Config |
+   |--------------|----------------------------|--------------------------------------------------------|---------------|
+   | SMA          | SunnyBoy TriPower STL-6000 | Address 126 (add 123 to the base SMA address)          |               |
+   |              |                            | Filter required for Amps at night; NaN map to Zero (0) |               |
+   | SunSpec      |                            | Works out of the box (Fronius Modbus IS Sunspec)       |               |

sunspec_exporter/sunspec_exporter.py

@@ -4,7 +4,7 @@
 """sunspec-prometheus-exporter
 
 Usage:
-  sunspec_exporter.py start [ --port PORT ] [ --sunspec_address SUNSPEC_ADDRESS ] --sunspec_ip SUNSPEC_IP --sunspec_port SUNSPEC_PORT --sunspec_model_ids MODEL_IDS --filter METRIC_FILTER
+  sunspec_exporter.py start [ --port PORT ] [ --sunspec_address SUNSPEC_ADDRESS ] [ --filter METRIC_FILTER... ] --sunspec_ip SUNSPEC_IP --sunspec_port SUNSPEC_PORT --sunspec_model_ids MODEL_IDS
   sunspec_exporter.py query [ --sunspec_address SUNSPEC_ADDRESS ] --sunspec_ip SUNSPEC_IP --sunspec_port SUNSPEC_PORT 
 
 Options:
@@ -30,7 +30,7 @@
 
   METRICFILTER is a space separated 3-Tuple, <metric_regex> <function>:<args> <replace_value> 
   
-    --filter "Amps_Phase[A-Z]_Aph[A-Z] gt:3276 0"
+  "Amps_Phase[A-Z]_Aph[A-Z] gt:3276 0"
 
   Which reads, when the metric matching regex, Amps_Phase[A-Z]_Aph[A-Z] is greater than 3276, set the metric as 0.
   In this example case, the inverter jumps from low 0.4, 0.5 Amps, up to 3276.7 (signed int16 Upper) represented as NaN. 
@@ -53,15 +53,53 @@
 from xml.dom import minidom
 import time
 import re
+import collections
 
+Filter = collections.namedtuple('Filter', ['regex', 'fn'])
+class FnMapping:
+
+    def filter_fn(fn, *args):
+        def filter(v):
+            return fn(*args, v)
+        return filter
+
+    def gt(replacement, upper_bound, val):
+        if val > upper_bound:
+            return replacement
+        else:
+            return val
+
+    def lt(replacement, lower_bound, val):
+        if val < lower_bound:
+            return replacement
+        else:
+            return val
+
+    def gte(replacement, upper_bound, val):
+        if val >= upper_bound:
+            return replacement
+        else:
+            return val
+
+    def lte(replacement, lower_bound, val):
+        if val <= lower_bound:
+            return replacement
+        else:
+            return val
+
+    def equals(replacement, equals_val, val):
+        if val == equals_val:
+            return replacement
+        else:
+            return val
 
 # Create a metric to track time spent and requests made.
 REQUEST_TIME = Summary('sunspec_fn_collect_data',
                        'Time spent collecting the data')
 
 # Decorate function with metric.
 @REQUEST_TIME.time()
-def collect_data(sunspec_client, model_ids):
+def collect_data(sunspec_client, model_ids, filters):
 
     if sunspec_client is None:
         print("no sunspec client defined, init() call, ignoring")
@@ -111,25 +149,35 @@ def collect_data(sunspec_client, model_ids):
                             else:
                                 value = str(point.value).rstrip('\0')
 
-                            print(f"# {metric_label}{unit_label}: {value}")
-                            results[f"{metric_label}{unit_label}"] = { "value" : value, "metric_type": metric_type }
+                            final_label = f"{metric_label}{unit_label}"
+
+                            if len(filters) > 0:
+                                for x in filters:
+                                    if x.regex.match(metric_label):
+                                        old_value = value
+                                        value = x.fn(old_value)
+                                        print(f"# !! Filtered {metric_label}. {x.regex} matched. {old_value} -> {value}", flush=True)
+
+                            print(f"# {final_label}: {value}")
+                            results[f"{final_label}"] = { "value" : value, "metric_type": metric_type }
 
     return results
 
 
 class SunspecCollector(object):
     "The ip, port and target is of the modbus/sunspec device"
 
-    def __init__(self, sunspec_client, model_ids, ip, port, target):
+    def __init__(self, sunspec_client, model_ids, ip, port, target, filters):
         self.sunspec_client = sunspec_client
         self.model_ids = model_ids
         self.ip = ip
         self.port = port
         self.target = target
+        self.filters = filters
 
     def collect(self):
         # yield GaugeMetricFamily('my_gauge', 'Help text', value=7)
-        results = collect_data(self.sunspec_client, self.model_ids)
+        results = collect_data(self.sunspec_client, self.model_ids, self.filters)
         for x in results:  # call sunspec here
             the_value = results[x]["value"]
             if is_numeric(the_value):
@@ -257,8 +305,6 @@ def sunspec_test(ip, port, address):
                         print('%-40s %20s %-10s' % (label, value, str(units)))
 
 
-
-
 if __name__ == '__main__':
     # Start up the server to expose the metrics.
     arguments = docopt(__doc__, version='sunspec-prometheus-exporter 1.0')
@@ -288,6 +334,14 @@ def sunspec_test(ip, port, address):
         # remove the models that don't match what we want
         # this will make reads faster (ignore unnecessary model data sets)
 
+        filters = []
+        if arguments["--filter"] is not None:
+            for f in arguments["--filter"]:
+                (filter_metric_regex, func_n_params, replacement) = f.split(" ")
+                func_name, *parameters = func_n_params.split(":")
+                func = FnMapping.filter_fn(eval(f"FnMapping.{func_name}"), replacement, *parameters)
+                filters.append(Filter(regex=re.compile(filter_metric_regex),fn=func))
+
         print("# !!! Enumerating all models, removing from future reads unwanted ones")
         models = sunspec_client.device.models_list.copy()
         for model in models:
@@ -296,15 +350,15 @@ def sunspec_test(ip, port, address):
                 print(f"#    Removed [{name}]")
                 sunspec_client.device.models_list.remove(model)
             else:
-                print(f"#  Keeping [{name}]")
-
+                print(f"#  Will collect [{name}]")
 
         REGISTRY.register(SunspecCollector(
                 sunspec_client,
                 sunspec_model_ids,
                 sunspec_ip,
                 sunspec_port,
-                sunspec_address
+                sunspec_address,
+                filters
             )
         )