diff --git a/INSTALL.md b/INSTALL.md new file mode 100644 index 0000000..fdef1f9 --- /dev/null +++ b/INSTALL.md @@ -0,0 +1,101 @@ +# Installation guide +This guide is written to be used with a raspbian. + +## Setup network +Set a static IP adress in '/etc/dhcpcd.conf' + +``` +sudo -i +echo "interface eth0" >> /etc/dhcpcd.conf +echo "static ip_address=10.2.3.4/24" >> /etc/dhcpcd.conf +echo "static routers=10.2.3.1" >> /etc/dhcpcd.conf +echo "static domain_name_server=10.2.3.2 10.2.3.3" >> /etc/dhcpcd.conf +reboot +``` + +## Setup SSH(optional, but handy) +Edit /etc/ssh/sshd_config to fit your needs +``` +sudo vim /etc/ssh/sshd_config +``` +Start and enable ssh +``` +sudo systemctl enable --now ssh +``` +## Install oversight +Install poetry +``` +sudo pip3 install poetry +``` +Create a user +``` +sudo adduser --system oversight +``` +Change to the overside user +``` +sudo su - oversight -s /bin/bash +``` +Go to the home directory +``` +cd /home/oversight +``` +Clone the oversight git repo +``` +git clone https://github.com/feanor12/oversight.git +``` +Change working directory to the cloned repository +``` +cd oversight +``` +Checkout the django2 branch +``` +git checkout django2 +``` +Install dependencies +``` +poetry install +``` +Prepare config file +``` +cp -r example/ config +vim config/settings.py +``` +Generate Database, Superuser and static files +``` +poetry shell +env PYTHONPATH="./config" ./manage.py migrate –settings="settings" +env PYTHONPATH="./config" ./manage.py collectstatic –settings="settings" +env PYTHONPATH="./config" ./manage.py createsuperuser –settings="settings" +``` +Test server +``` +env PYTHONPATH=“./config” ./manage.py runserver –settings="settings" +firefox http://127.0.0.1:8000 +``` + +## Install Proxy +Install nginx +``` +sudo apt install nginx +``` +Copy template from oversight +``` +sudo cp /home/oversight/oversight/ansible/roles/nginx/templates/nginx.conf /etc/nginx/nginx.conf +``` +Replace the placeholders in the template and remove template if-lines +``` +sudo sed -i -e "s/inventory_hostname/my.domain.com/g" /etc/nginx/nginx.conf +sudo sed -i -e "s/{{ 443 if nginx_no_ssl is undefined else 80 }}/443/g" /etc/nginx/nginx.conf +sudo sed -i -e "/{%.*%}/d" /etc/nginx/nginx.conf +``` +Copy ssl certificates +``` +sudo mkdir /etc/nginx/ssl +sudo cp server.crt server.key dhparams.pem /etc/nginx/etc +``` + +TODO + - fix server.py + - start services + - copy systemd unit files + - start services(systemctl) diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..48a0fc4 --- /dev/null +++ b/Pipfile @@ -0,0 +1,17 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +pyserial = "*" +minimalmodbus = "*" +django = "*" +cherrypy = "*" +requests = "*" +cheroot = "*" + +[dev-packages] + +[requires] +python_version = "3.7" diff --git a/README.md b/README.md new file mode 100644 index 0000000..8948dad --- /dev/null +++ b/README.md @@ -0,0 +1,14 @@ +# oversight + +Django based web application for data logging. + +# Features +* Overview of the latest sensor values +* Detailed view with the latest values and a diagram showing the history +* Compare multiple sensors using a diagram +* Add sensor with the Django admin web interface +* API to set and get sensor data +* Logging daemon and web application are run in parallel +* Ansible scripts for easier setup on Raspberry Pi +* Easy to include new sensors + diff --git a/example/settings.py b/example/settings.py index 8da9496..6ea221c 100644 --- a/example/settings.py +++ b/example/settings.py @@ -1,5 +1,3 @@ -from django.conf.global_settings import TEMPLATE_CONTEXT_PROCESSORS - # Build paths inside the project like this: os.path.join(BASE_DIR, ...) import os BASE_DIR = os.path.dirname(os.path.dirname(__file__)) @@ -31,7 +29,7 @@ 'oversight.apps.OversightConfig', ) -MIDDLEWARE_CLASSES = ( +MIDDLEWARE = ( 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', @@ -73,6 +71,7 @@ # https://docs.djangoproject.com/en/dev/howto/static-files/ STATIC_URL = '/static/' +STATIC_ROOT = os.path.join(BASE_DIR, 'static') OVERSIGHT_KEY = 'hallo' LOGGING = { @@ -81,7 +80,7 @@ 'handlers': { 'console': { 'class': 'logging.FileHandler', - 'filename': '/dev/stderr', + 'filename': '/tmp/oversight.log', } }, 'loggers': { @@ -92,4 +91,15 @@ } } -TEMPLATE_CONTEXT_PROCESSORS = TEMPLATE_CONTEXT_PROCESSORS + ('django.core.context_processors.request',) +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages" + ] + } + }, +] diff --git a/oversight/management/commands/sensordaemon.py b/oversight/management/commands/sensordaemon.py index e7a1c52..125fda9 100644 --- a/oversight/management/commands/sensordaemon.py +++ b/oversight/management/commands/sensordaemon.py @@ -1,10 +1,10 @@ import logging import threading -import Queue -from SimpleXMLRPCServer import SimpleXMLRPCServer +from multiprocessing import Queue +from xmlrpc.server import SimpleXMLRPCServer from django.conf import settings -from django.core.management.base import NoArgsCommand +from django.core.management.base import BaseCommand import requests @@ -87,12 +87,12 @@ def worker(queue, tasks): tasks[item[0]](*item[1:]) except Exception as e: logger.error("Task failed: ", exc_info=e) - queue.task_done() + #queue.task_done() -class Command(NoArgsCommand): +class Command(BaseCommand): def handle(self, **options): - queue = Queue.Queue() + queue = Queue() server = SimpleXMLRPCServer(("localhost", 12345)) sensor_manager = SensorManager() server.register_instance(sensor_manager) diff --git a/oversight/migrations/0001_initial.py b/oversight/migrations/0001_initial.py index 318eca6..4c8a40f 100644 --- a/oversight/migrations/0001_initial.py +++ b/oversight/migrations/0001_initial.py @@ -1,10 +1,14 @@ -# encoding: utf8 -from django.db import models, migrations +# Generated by Django 2.1.1 on 2018-09-12 10:53 + +from django.db import migrations, models +import django.db.models.deletion import django.utils.timezone class Migration(migrations.Migration): + initial = True + dependencies = [ ] @@ -12,27 +16,35 @@ class Migration(migrations.Migration): migrations.CreateModel( name='LogEntry', fields=[ - (u'id', models.AutoField(verbose_name=u'ID', serialize=False, auto_created=True, primary_key=True)), + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('datetime', models.DateTimeField(default=django.utils.timezone.now)), ('value', models.CharField(max_length=255)), ], - options={ - }, - bases=(models.Model,), ), migrations.CreateModel( name='Sensor', fields=[ - (u'id', models.AutoField(verbose_name=u'ID', serialize=False, auto_created=True, primary_key=True)), + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('name', models.CharField(max_length=255)), - ('api_endpoint', models.SlugField()), + ('api_endpoint', models.SlugField(unique=True)), ('unit', models.CharField(max_length=255)), ('sensor_class', models.CharField(max_length=255)), ('params', models.TextField()), - ('current_log', models.ForeignKey(to='oversight.LogEntry', to_field=u'id', null=True)), + ('log_plot', models.BooleanField(default=False)), + ('logging_enabled', models.BooleanField(default=True)), + ('alarm_below', models.CharField(blank=True, max_length=255)), + ('alarm_above', models.CharField(blank=True, max_length=255)), + ('alarm_acked', models.BooleanField(default=True)), + ('current_log', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='oversight.LogEntry')), ], - options={ - }, - bases=(models.Model,), + ), + migrations.AddField( + model_name='logentry', + name='sensor', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='oversight.Sensor'), + ), + migrations.AlterIndexTogether( + name='logentry', + index_together={('sensor', 'datetime')}, ), ] diff --git a/oversight/migrations/0001_initial_squashed.py b/oversight/migrations/0001_initial_squashed.py deleted file mode 100644 index b989667..0000000 --- a/oversight/migrations/0001_initial_squashed.py +++ /dev/null @@ -1,69 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations -import django.utils.timezone - - -class Migration(migrations.Migration): - - replaces = [(b'oversight', '0001_initial'), (b'oversight', '0001_logentry_sensor'), (b'oversight', '0002_auto_20140131_1403'), (b'oversight', '0003_sensor_log_plot'), (b'oversight', '0004_add_dbindex_to_logentry_datetime'), (b'oversight', '0005_use_index_together'), (b'oversight', '0006_add_logging_field'), (b'oversight', '0007_add_alaram_fields')] - - dependencies = [ - ] - - operations = [ - migrations.CreateModel( - name=b'LogEntry', - fields=[ - ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), - (b'datetime', models.DateTimeField(default=django.utils.timezone.now)), - (b'value', models.CharField(max_length=255)), - ], - options={ - }, - bases=(models.Model,), - ), - migrations.CreateModel( - name=b'Sensor', - fields=[ - ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), - (b'name', models.CharField(max_length=255)), - (b'api_endpoint', models.SlugField(unique=True)), - (b'unit', models.CharField(max_length=255)), - (b'sensor_class', models.CharField(max_length=255)), - (b'params', models.TextField()), - (b'current_log', models.ForeignKey(related_name='+', blank=True, to='oversight.LogEntry', null=True)), - (b'log_plot', models.BooleanField(default=False)), - ('logging_enabled', models.BooleanField(default=True)), - ('alarm_above', models.CharField(default='', max_length=255, blank=True)), - ('alarm_acked', models.BooleanField(default=True)), - ('alarm_below', models.CharField(default='', max_length=255, blank=True)), - ], - options={ - }, - bases=(models.Model,), - ), - migrations.AddField( - model_name=b'logentry', - name=b'sensor', - field=models.ForeignKey(to=b'oversight.Sensor', default=0, to_field='id'), - preserve_default=False, - ), - migrations.AlterField( - model_name='logentry', - name='datetime', - field=models.DateTimeField(default=django.utils.timezone.now, db_index=True), - preserve_default=True, - ), - migrations.AlterField( - model_name='logentry', - name='datetime', - field=models.DateTimeField(default=django.utils.timezone.now), - preserve_default=True, - ), - migrations.AlterIndexTogether( - name='logentry', - index_together=set([(b'sensor', b'datetime')]), - ), - ] diff --git a/oversight/migrations/0001_logentry_sensor.py b/oversight/migrations/0001_logentry_sensor.py deleted file mode 100644 index 4339908..0000000 --- a/oversight/migrations/0001_logentry_sensor.py +++ /dev/null @@ -1,18 +0,0 @@ -# encoding: utf8 -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('oversight', '0001_initial'), - ] - - operations = [ - migrations.AddField( - model_name='logentry', - name='sensor', - field=models.ForeignKey(to='oversight.Sensor', to_field=u'id', default=0), - preserve_default=False, - ), - ] diff --git a/oversight/migrations/0002_auto_20140131_1403.py b/oversight/migrations/0002_auto_20140131_1403.py deleted file mode 100644 index 0c0b633..0000000 --- a/oversight/migrations/0002_auto_20140131_1403.py +++ /dev/null @@ -1,22 +0,0 @@ -# encoding: utf8 -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('oversight', '0001_logentry_sensor'), - ] - - operations = [ - migrations.AlterField( - model_name='sensor', - name='current_log', - field=models.ForeignKey(to_field=u'id', blank=True, to='oversight.LogEntry', null=True), - ), - migrations.AlterField( - model_name='sensor', - name='api_endpoint', - field=models.SlugField(unique=True), - ), - ] diff --git a/oversight/migrations/0003_sensor_log_plot.py b/oversight/migrations/0003_sensor_log_plot.py deleted file mode 100644 index d369a18..0000000 --- a/oversight/migrations/0003_sensor_log_plot.py +++ /dev/null @@ -1,18 +0,0 @@ -# encoding: utf8 -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('oversight', '0002_auto_20140131_1403'), - ] - - operations = [ - migrations.AddField( - model_name='sensor', - name='log_plot', - field=models.BooleanField(default=False), - preserve_default=True, - ), - ] diff --git a/oversight/migrations/0004_add_dbindex_to_logentry_datetime.py b/oversight/migrations/0004_add_dbindex_to_logentry_datetime.py deleted file mode 100644 index c5b2085..0000000 --- a/oversight/migrations/0004_add_dbindex_to_logentry_datetime.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations -import django.utils.timezone - - -class Migration(migrations.Migration): - - dependencies = [ - ('oversight', '0003_sensor_log_plot'), - ] - - operations = [ - migrations.AlterField( - model_name='logentry', - name='datetime', - field=models.DateTimeField(default=django.utils.timezone.now, db_index=True), - ), - ] diff --git a/oversight/migrations/0005_use_index_together.py b/oversight/migrations/0005_use_index_together.py deleted file mode 100644 index 4e24a2f..0000000 --- a/oversight/migrations/0005_use_index_together.py +++ /dev/null @@ -1,24 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations -import django.utils.timezone - - -class Migration(migrations.Migration): - - dependencies = [ - ('oversight', '0004_add_dbindex_to_logentry_datetime'), - ] - - operations = [ - migrations.AlterField( - model_name='logentry', - name='datetime', - field=models.DateTimeField(default=django.utils.timezone.now), - ), - migrations.AlterIndexTogether( - name='logentry', - index_together=set([(b'sensor', b'datetime')]), - ), - ] diff --git a/oversight/migrations/0006_add_logging_field.py b/oversight/migrations/0006_add_logging_field.py deleted file mode 100644 index 7e23e8c..0000000 --- a/oversight/migrations/0006_add_logging_field.py +++ /dev/null @@ -1,25 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('oversight', '0005_use_index_together'), - ] - - operations = [ - migrations.AddField( - model_name='sensor', - name='logging_enabled', - field=models.BooleanField(default=True), - preserve_default=True, - ), - migrations.AlterField( - model_name='sensor', - name=b'current_log', - field=models.ForeignKey(related_name=b'+', blank=True, to='oversight.LogEntry', null=True), - ), - ] diff --git a/oversight/migrations/0007_add_alaram_fields.py b/oversight/migrations/0007_add_alaram_fields.py deleted file mode 100644 index 38b04dd..0000000 --- a/oversight/migrations/0007_add_alaram_fields.py +++ /dev/null @@ -1,32 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('oversight', '0006_add_logging_field'), - ] - - operations = [ - migrations.AddField( - model_name='sensor', - name='alarm_above', - field=models.CharField(default='', max_length=255, blank=True), - preserve_default=False, - ), - migrations.AddField( - model_name='sensor', - name='alarm_acked', - field=models.BooleanField(default=True), - preserve_default=True, - ), - migrations.AddField( - model_name='sensor', - name='alarm_below', - field=models.CharField(default='', max_length=255, blank=True), - preserve_default=False, - ), - ] diff --git a/oversight/models.py b/oversight/models.py index f526942..6ca395a 100644 --- a/oversight/models.py +++ b/oversight/models.py @@ -2,7 +2,8 @@ from django.core.exceptions import ValidationError from django.db import models -from django.utils.module_loading import import_by_path +#from django.utils.module_loading import import_by_path +from django.utils.module_loading import import_string from django.utils.timezone import now @@ -12,7 +13,9 @@ class Sensor(models.Model): unit = models.CharField(max_length=255) sensor_class = models.CharField(max_length=255) params = models.TextField() - current_log = models.ForeignKey('LogEntry', null=True, blank=True, + current_log = models.ForeignKey('LogEntry', null=True, + blank=True, + on_delete=models.CASCADE, related_name='+') log_plot = models.BooleanField(default=False) logging_enabled = models.BooleanField(default=True) @@ -30,7 +33,7 @@ def clean(self): @property def backend(self): params = json.loads(self.params) - return import_by_path(self.sensor_class)(**params) + return import_string(self.sensor_class)(**params) @property def frozen(self): @@ -40,10 +43,14 @@ def frozen(self): def __unicode__(self): return self.name + def __str__(self): + return self.name + + class LogEntry(models.Model): datetime = models.DateTimeField(default=now) - sensor = models.ForeignKey(Sensor) + sensor = models.ForeignKey(Sensor,on_delete=models.CASCADE) value = models.CharField(max_length=255) class Meta: diff --git a/oversight/root_urls.py b/oversight/root_urls.py index c0b55b3..c67961a 100644 --- a/oversight/root_urls.py +++ b/oversight/root_urls.py @@ -1,11 +1,14 @@ -from django.conf.urls import patterns, include, url +from django.conf.urls import include, url from django.views.generic.base import RedirectView +from django.conf import settings +from django.conf.urls.static import static + from django.contrib import admin -urlpatterns = patterns('', +urlpatterns = [ url(r'^oversight/', include('oversight.urls')), - url(r'^admin/', include(admin.site.urls)), + url(r'^admin/', admin.site.urls), url(r'^$', RedirectView.as_view(pattern_name='oversight_index')), -) +]#+ static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) diff --git a/oversight/sensors/base.py b/oversight/sensors/base.py index fc7a814..3430205 100644 --- a/oversight/sensors/base.py +++ b/oversight/sensors/base.py @@ -1,4 +1,5 @@ from contextlib import contextmanager +from six import string_types import threading @@ -20,7 +21,7 @@ def api(self, action, args): return self.write(self.from_string(args[0])) def to_string(self, value): - if not isinstance(value, basestring): + if not isinstance(value, string_types): value = repr(value) return value diff --git a/oversight/sensors/eurotherm.py b/oversight/sensors/eurotherm.py index 9ce8fa4..409bfea 100644 --- a/oversight/sensors/eurotherm.py +++ b/oversight/sensors/eurotherm.py @@ -1,26 +1,29 @@ from .base import Sensor import minimalmodbus -# Yikes, but we can't change this otherwise -minimalmodbus.BAUDRATE = 9600 +minimalmodbus.CLOSE_PORT_AFTER_EACH_CALL = True +import logging +logger = logging.getLogger(__name__) class EuroTherm(Sensor): def __init__(self, port, number_of_decimals, register=1): self.port = port self.number_of_decimals = number_of_decimals self.register = register + self.baudrate = 9600 + self.instrument = minimalmodbus.Instrument(self.port, 1) + self.instrument.serial.baudrate = self.baudrate def read(self): - instrument = minimalmodbus.Instrument(self.port, 1) - data = instrument.read_register(self.register, self.number_of_decimals, - signed=True) - #instrument.serial.close() + data = self.instrument.read_register(self.register, + self.number_of_decimals, + signed=True) return round(data, 2) def write(self, value): - instrument = minimalmodbus.Instrument(self.port, 1) - instrument.write_register(self.register, value, self.number_of_decimals, - signed=True) - #instrument.serial.close() + self.instrument.write_register(self.register, + value, + self.number_of_decimals, + signed=True) return '' diff --git a/oversight/sensors/pressure.py b/oversight/sensors/pressure.py index a837e05..d02b5bc 100644 --- a/oversight/sensors/pressure.py +++ b/oversight/sensors/pressure.py @@ -5,8 +5,8 @@ from .base import Sensor -CR = '\n' -ENQ = '\x05' +CR = b'\n' +ENQ = b'\x05' sleeptime_rs = 0.05 @@ -18,7 +18,7 @@ def __init__(self, port, sensor): def read(self): ser = serial.Serial(self.port, 9600, timeout=5, parity='N', bytesize=8, stopbits=1) - ser.write('P'+bytes(self.sensor)+CR) + ser.write(b'P'+bytes(self.sensor,"utf-8")+CR) time.sleep(sleeptime_rs) ser.readline() # read acknowledgement time.sleep(sleeptime_rs) @@ -26,7 +26,7 @@ def read(self): time.sleep(sleeptime_rs) value = ser.readline().strip() # read value ser.close() - return self.from_string(value[2:]) + return self.from_string(value[2:].decode("utf-8")) def to_string(self, value): return '%.2e' % value diff --git a/oversight/sensors/web.py b/oversight/sensors/web.py new file mode 100644 index 0000000..2fef6e9 --- /dev/null +++ b/oversight/sensors/web.py @@ -0,0 +1,20 @@ +import decimal + +from .base import Sensor +from requests import get + +class WebSensor(Sensor): + def __init__(self, url): + self.url = url + + def read(self): + res = get(self.url) + if(res.status_code == 200): + return self.from_string(res.text) + return self.from_string("-1") + + def to_string(self, value): + return '%.2e' % value + + def from_string(self, value): + return decimal.Decimal(value) diff --git a/oversight/templates/oversight/index.html b/oversight/templates/oversight/index.html index 1a383a6..73fed78 100644 --- a/oversight/templates/oversight/index.html +++ b/oversight/templates/oversight/index.html @@ -1,4 +1,5 @@ {% extends 'oversight/page.html' %} +{% load ktoc %} {% block content %} {{ block.super }} @@ -35,7 +36,10 @@