diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d427f69 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +*.gz +*.tgz +build/ diff --git a/Dockerfile b/Dockerfile index e070558..cba8460 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,11 @@ -FROM python:3.9 - +FROM python:3.9-alpine ADD requirements.txt / RUN pip install -r /requirements.txt +RUN apk update && apk upgrade && \ + apk add --no-cache bash git openssh + WORKDIR /src/pysunspec RUN git clone --recursive https://github.com/sunspec/pysunspec . RUN python -m unittest discover -v sunspec diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..da85474 --- /dev/null +++ b/Makefile @@ -0,0 +1,8 @@ +VERSION = $(shell cat VERSION) + +.PHONY: build +build: + docker build -t inosion/prometheus-sunspec-exporter:$(VERSION) . + +release: + docker save inosion/prometheus-sunspec-exporter:$(VERSION) | gzip > inosion-prometheus-sunspec-exporter-$(VERSION).tgz \ No newline at end of file diff --git a/README.md b/README.md index de547a5..c758633 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,26 @@ # Prometheus Sunspec Exporter -- ~Alpha~ Beta - Works and is Pulling data :-D +- Works and is Pulling data :-D - May have some re-coding to do around what consitutes a gauge, counter etc - is optimised to read only the "sunspec" model you desire (reduce call load on the device) - Need to run one exporter "per" modbus sunspec address/ip/port (current limitation) - Uses https://github.com/sunspec/pysunspec -# Tested on -- SMA SunnyBoy TriPower STL-6000 - Use address 126 (for SunSpec) -- Fronius (Something) - SunSpec native +# Sunspec Devices + Sunspec is alliance of 100 Solar and Electricity Storage products, that provides a standard Modbu API. + + With this API, a prometheus exporter was born, in approximately 8 hours. + + Devices supported can be found here (Sunspec Modbus Certified List)[https://sunspec.org/sunspec-modbus-certified-products/] + Manufacturers supporting Sunspec: + - Fronius + - SMA + - Huawei + - ABB + - Sungrow + + See (tested-devices)[docs/tested-devices.md] for more information. + # Sample Grafana ![images/grafana_dash_sample_2021-01-10_13-28.png](images/grafana_dash_sample_2021-01-10_13-28.png) @@ -126,7 +138,6 @@ which section, and then that becomes your set of model_id's. sudo systemctl start prometheus-sunspec-exporter.service ``` 5. Configure your Prometheus to collect the data [install/prometheus.yml](install/prometheus.yml) - # Testing @@ -168,6 +179,30 @@ sunspec_Cabinet_Temperature_TmpCab_C{ip="192.168.1.70",port="502",target="126"} # HELP sunspec_Operating_State_St_total ``` +# Filtering + +Some devices needs some tweaking, on the values they return. +Some SMA Solar inverters return 3276.8 for NaN. +At night time, the DC Amps returns that value. Which is non too helpful in Grafana + +![amps_bad](images/filtering_example_amps_bad.png) + +To correct this, we can remap each returned value, before prometheus collects it. + +``` +# Example of METRICFILTER +--filter "Amps_Phase[ABC]_Aph[ABC]_A gt:3276 0.0" +``` +METRICFILTER is a space separated 3-Tuple, : + +"Amps_Phase[ABC]_Aph[ABC]_A gt:3276 0.0" + +Which reads, when the metric matching regex, Amps_Phase[ABC]_Aph[ABC]_A is greater than 3276, set the metric as 0. +In this example case, the inverter jumps from low 0.4, 0.5 Amps, up to 3276.7 (signed int16 Upper) represented as NaN. + +This filter config removes that problem. + + # Testing with SunSpec See [testing/](testing/) \ No newline at end of file diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..66c4c22 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +1.0.9 diff --git a/docs/tested-devices.md b/docs/tested-devices.md new file mode 100644 index 0000000..1815b8b --- /dev/null +++ b/docs/tested-devices.md @@ -0,0 +1,20 @@ +# Known Working / Tested Devices + +Search (Sunspec Modbus Certified List)[https://sunspec.org/sunspec-modbus-certified-products/] for your device. + +Because Sunspec is a standard, the configuration required, for the exporter, will vary only: + +1. Which Modbus address the device listens on +2. The `model_ids` you need from the device (don't need the serial number 6 times a minute :-) ) +3. Any custom filterng required + +You won't need to map any data fields, the exporter does all that work. + +# Devices Working + + |--------------|----------------------------|--------------------------------------------------------|---------------| + | Manufacturer | Model | Notes | Sample Config | + |--------------|----------------------------|--------------------------------------------------------|---------------| + | SMA | SunnyBoy TriPower STL-6000 | Address 126 (add 123 to the base SMA address) | | + | | | Filter required for Amps at night; NaN map to Zero (0) | | + | SunSpec | | Works out of the box (Fronius Modbus IS Sunspec) | | diff --git a/sunspec_exporter/sunspec_exporter.py b/sunspec_exporter/sunspec_exporter.py index b1918c6..bd32bb2 100644 --- a/sunspec_exporter/sunspec_exporter.py +++ b/sunspec_exporter/sunspec_exporter.py @@ -4,7 +4,7 @@ """sunspec-prometheus-exporter Usage: - sunspec_exporter.py start [ --port PORT ] [ --sunspec_address SUNSPEC_ADDRESS ] --sunspec_ip SUNSPEC_IP --sunspec_port SUNSPEC_PORT --sunspec_model_ids MODEL_IDS --filter METRIC_FILTER + sunspec_exporter.py start [ --port PORT ] [ --sunspec_address SUNSPEC_ADDRESS ] [ --filter METRIC_FILTER... ] --sunspec_ip SUNSPEC_IP --sunspec_port SUNSPEC_PORT --sunspec_model_ids MODEL_IDS sunspec_exporter.py query [ --sunspec_address SUNSPEC_ADDRESS ] --sunspec_ip SUNSPEC_IP --sunspec_port SUNSPEC_PORT Options: @@ -30,7 +30,7 @@ METRICFILTER is a space separated 3-Tuple, : - --filter "Amps_Phase[A-Z]_Aph[A-Z] gt:3276 0" + "Amps_Phase[A-Z]_Aph[A-Z] gt:3276 0" Which reads, when the metric matching regex, Amps_Phase[A-Z]_Aph[A-Z] is greater than 3276, set the metric as 0. In this example case, the inverter jumps from low 0.4, 0.5 Amps, up to 3276.7 (signed int16 Upper) represented as NaN. @@ -53,7 +53,45 @@ from xml.dom import minidom import time import re +import collections +Filter = collections.namedtuple('Filter', ['regex', 'fn']) +class FnMapping: + + def filter_fn(fn, *args): + def filter(v): + return fn(*args, v) + return filter + + def gt(replacement, upper_bound, val): + if val > upper_bound: + return replacement + else: + return val + + def lt(replacement, lower_bound, val): + if val < lower_bound: + return replacement + else: + return val + + def gte(replacement, upper_bound, val): + if val >= upper_bound: + return replacement + else: + return val + + def lte(replacement, lower_bound, val): + if val <= lower_bound: + return replacement + else: + return val + + def equals(replacement, equals_val, val): + if val == equals_val: + return replacement + else: + return val # Create a metric to track time spent and requests made. REQUEST_TIME = Summary('sunspec_fn_collect_data', @@ -61,7 +99,7 @@ # Decorate function with metric. @REQUEST_TIME.time() -def collect_data(sunspec_client, model_ids): +def collect_data(sunspec_client, model_ids, filters): if sunspec_client is None: print("no sunspec client defined, init() call, ignoring") @@ -111,8 +149,17 @@ def collect_data(sunspec_client, model_ids): else: value = str(point.value).rstrip('\0') - print(f"# {metric_label}{unit_label}: {value}") - results[f"{metric_label}{unit_label}"] = { "value" : value, "metric_type": metric_type } + final_label = f"{metric_label}{unit_label}" + + if len(filters) > 0: + for x in filters: + if x.regex.match(metric_label): + old_value = value + value = x.fn(old_value) + print(f"# !! Filtered {metric_label}. {x.regex} matched. {old_value} -> {value}", flush=True) + + print(f"# {final_label}: {value}") + results[f"{final_label}"] = { "value" : value, "metric_type": metric_type } return results @@ -120,16 +167,17 @@ def collect_data(sunspec_client, model_ids): class SunspecCollector(object): "The ip, port and target is of the modbus/sunspec device" - def __init__(self, sunspec_client, model_ids, ip, port, target): + def __init__(self, sunspec_client, model_ids, ip, port, target, filters): self.sunspec_client = sunspec_client self.model_ids = model_ids self.ip = ip self.port = port self.target = target + self.filters = filters def collect(self): # yield GaugeMetricFamily('my_gauge', 'Help text', value=7) - results = collect_data(self.sunspec_client, self.model_ids) + results = collect_data(self.sunspec_client, self.model_ids, self.filters) for x in results: # call sunspec here the_value = results[x]["value"] if is_numeric(the_value): @@ -257,8 +305,6 @@ def sunspec_test(ip, port, address): print('%-40s %20s %-10s' % (label, value, str(units))) - - if __name__ == '__main__': # Start up the server to expose the metrics. arguments = docopt(__doc__, version='sunspec-prometheus-exporter 1.0') @@ -288,6 +334,14 @@ def sunspec_test(ip, port, address): # remove the models that don't match what we want # this will make reads faster (ignore unnecessary model data sets) + filters = [] + if arguments["--filter"] is not None: + for f in arguments["--filter"]: + (filter_metric_regex, func_n_params, replacement) = f.split(" ") + func_name, *parameters = func_n_params.split(":") + func = FnMapping.filter_fn(eval(f"FnMapping.{func_name}"), replacement, *parameters) + filters.append(Filter(regex=re.compile(filter_metric_regex),fn=func)) + print("# !!! Enumerating all models, removing from future reads unwanted ones") models = sunspec_client.device.models_list.copy() for model in models: @@ -296,15 +350,15 @@ def sunspec_test(ip, port, address): print(f"# Removed [{name}]") sunspec_client.device.models_list.remove(model) else: - print(f"# Keeping [{name}]") - + print(f"# Will collect [{name}]") REGISTRY.register(SunspecCollector( sunspec_client, sunspec_model_ids, sunspec_ip, sunspec_port, - sunspec_address + sunspec_address, + filters ) )