Merge pull request #423 from ODM2/release/0.10.0

Release/0.10.0

release_commands.txt

@@ -1,4 +0,0 @@
-dataloaderinterface/change_datetimes_to_utc
-dataloaderinterface/generate_new_models_data
-dataloaderinterface/set_leafpackdb_defaults
-accounts/migrate_odm2user_model

requirements.txt

@@ -1,7 +1,7 @@
 beautifulsoup4==4.5.1
 codegen==1.0
 coverage==4.2
-Django==1.11.15
+Django==1.11.20
 django-debug-toolbar==1.6
 django-discover-runner==1.0
 django-webtest==1.8.0
@@ -23,3 +23,6 @@ python-crontab==2.2.8
 oauthlib==2.0.*
 django-reset-migrations==0.3.1
 google-api-python-client==1.6.7
+django-admin-select2==1.0.1
+pandas==0.23.4
+influxdb==5.2.1

src/WebSDL/settings/base.py

@@ -52,6 +52,7 @@
     'dataloaderinterface.apps.DataloaderinterfaceConfig',
     'hydroshare',
     'leafpack',
+    'django_admin_select2',
     'django.contrib.admin',
     'django.contrib.auth',
     'django.contrib.contenttypes',
@@ -122,6 +123,8 @@
         'TEST': database['test'] if 'test' in database else {},
     }
 
+INFLUX_CONNECTION = data['influx_connection']
+
 # Password validation
 # https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators
 
@@ -217,3 +220,5 @@
 GOOGLE_API_CONF = data.get('google_api_conf', None)
 
 AUTH_USER_MODEL = 'accounts.User'
+
+DEBUG = True if 'debug_mode' in data and data['debug_mode'] == "True" else False

src/WebSDL/settings/linux_sandbox.py → src/WebSDL/settings/linux_server.py

@@ -1,7 +1,5 @@
 from WebSDL.settings.base import *
 
-DEBUG = False
-
 ALLOWED_HOSTS = ['127.0.0.1', 'localhost']
 if "host" in data:
     ALLOWED_HOSTS.append(data["host"])

src/WebSDL/settings/settings_template.json

@@ -1,5 +1,6 @@
 {
   "secret_key": "{{django secret key}}",
+  "debug_mode": "{{True/False, False by default}}",
   "static_root": "{{static files root directory}}",
   "host": "{{server domain name}}",
   "host_alt": ["{{any other server domain names}}", "{{...}}"],

src/WebSDL/wsgi.py

@@ -12,7 +12,7 @@
 
 from django.core.wsgi import get_wsgi_application
 
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "WebSDL.settings.linux_sandbox")
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "WebSDL.settings.linux_server")
 
 crontab_jobs.start_jobs()
 

src/accounts/templates/registration/register.html

@@ -3,7 +3,7 @@
 {% load static %}
 
 {% block page_title %}
-    <title>Join the EnviroDIY Data Portal</title>
+    <title>Join Monitor My Watershed</title>
 {% endblock %}
 
 {% block content %}

src/dataloader/helpers.py

@@ -0,0 +1,86 @@
+from datetime import datetime
+
+import pandas as pd
+import numpy as np
+
+from django.conf import settings
+from django.db.models import F
+
+from influxdb import DataFrameClient
+from influxdb.exceptions import InfluxDBClientError
+
+from dataloader.models import TimeSeriesResultValue
+
+
+class InfluxHelper(object):
+    class MissingConnectionException(Exception):
+        """Client is not defined or connected"""
+        pass
+
+    def __init__(self, *args, **kwargs):
+        self.client = None
+        self.batch_size = 10000
+        self.connection_info = settings.INFLUX_CONNECTION
+
+    def connect_to_dataframe_client(self):
+        self.client = DataFrameClient(
+            host=self.connection_info['host'],
+            port=self.connection_info['port'],
+            username=self.connection_info['username'],
+            password=self.connection_info['password'],
+            database=self.connection_info['database']
+        )
+
+    def recreate_database(self):
+        read_user = self.connection_info['read_username']
+        database_name = self.connection_info['database']
+
+        if not self.client:
+            raise InfluxHelper.MissingConnectionException('InfluxDB client is not connected.')
+
+        self.client.drop_database(database_name)
+        self.client.create_database(database_name)
+        self.client.grant_privilege('read', database_name, read_user)
+
+    def write_all_sensor_values(self, sensor):
+        self.write_sensor_values(sensor, datetime.min)
+
+    def get_series_last_value(self, identifier):
+        query_string = 'select last(DataValue), time from {identifier}'.format(identifier=identifier)
+        result = self.client.query(query_string, database=self.connection_info['database'])
+        if result and len(result) == 1:
+            dataframe = result[identifier]  # type: pd.DataFrame
+            return dataframe.first_valid_index().to_pydatetime()
+
+    def write_sensor_values(self, sensor, starting_datetime):
+        values = TimeSeriesResultValue.objects.filter(value_datetime__gt=starting_datetime, result_id=sensor.result_id)\
+            .annotate(DateTime=F('value_datetime'))\
+            .annotate(UTCOffset=F('value_datetime_utc_offset'))\
+            .annotate(DataValue=F('data_value'))
+        values_dataframe = self.prepare_data_values(values)
+        if values_dataframe.empty:
+            return 0
+
+        result = self.add_dataframe_to_database(values_dataframe, sensor.influx_identifier)
+        del values_dataframe
+        return result
+
+    def prepare_data_values(self, values_queryset):
+        dataframe = pd.DataFrame.from_records(values_queryset.values('DateTime', 'UTCOffset', 'DataValue'))
+        if dataframe.empty:
+            return dataframe
+
+        dataframe['DateTime'] = pd.to_datetime(dataframe['DateTime'])
+        dataframe.set_index(['DateTime'], inplace=True)
+        dataframe['DataValue'] = pd.to_numeric(dataframe['DataValue'], errors='coerce').astype(np.float64)
+        dataframe['UTCOffset'] = pd.to_numeric(dataframe['UTCOffset'], errors='coerce').astype(np.float64)
+        dataframe.dropna(how='any', inplace=True)
+        return dataframe
+
+    def add_dataframe_to_database(self, dataframe, identifier):
+        try:
+            write_success = self.client.write_points(dataframe, identifier, time_precision='s', batch_size=self.batch_size)
+            return len(dataframe) if write_success else 0
+        except InfluxDBClientError as e:
+            print 'Error while writing to database {}: {}'.format(identifier, e.message)
+            return 0

src/dataloader/models.py

@@ -714,7 +714,7 @@ class Unit(models.Model):
     unit_link = models.CharField(db_column='unitslink', blank=True, max_length=255)
 
     def __str__(self):
-        return '%s: %s (%s)' % (self.unit_type_id, self.unit_name, self.unit_abbreviation)
+        return '%s - %s (%s)' % (self.unit_name, self.unit_abbreviation, self.unit_type_id)
 
     def __repr__(self):
         return "<Unit('%s', '%s', '%s', '%s')>" % (
@@ -723,7 +723,7 @@ def __repr__(self):
 
     class Meta:
         db_table = 'units'
-        ordering = ['unit_type_id', 'unit_name']
+        ordering = ['unit_name', 'unit_type_id']
 
 
 @python_2_unicode_compatible

src/dataloaderinterface/admin.py

@@ -31,12 +31,13 @@ class SiteRegistrationAdmin(admin.ModelAdmin):
 
 @admin.register(Organization)
 class OrganizationAdmin(admin.ModelAdmin):
-    pass
+    list_display = ('organization_code', 'organization_name', 'organization_type', 'organization_link')
 
 
 @admin.register(EquipmentModel)
 class EquipmentModelAdmin(admin.ModelAdmin):
     sensor_fields = ['equipment_model_id']
+    list_display = ('model_name', 'model_manufacturer', 'model_description', 'model_link')
 
     def save_model(self, request, obj, form, change):
         if change:
@@ -48,6 +49,7 @@ def save_model(self, request, obj, form, change):
 @admin.register(Variable)
 class VariableAdmin(admin.ModelAdmin):
     sensor_fields = ['variable_id']
+    list_display = ('variable_code', 'variable_name', 'variable_type', 'no_data_value')
 
     def save_model(self, request, obj, form, change):
         if change:
@@ -59,6 +61,7 @@ def save_model(self, request, obj, form, change):
 @admin.register(Unit)
 class UnitAdmin(admin.ModelAdmin):
     sensor_fields = ['unit_id']
+    list_display = ('unit_name', 'unit_type', 'unit_abbreviation', 'unit_link')
 
     def save_model(self, request, obj, form, change):
         if change:
@@ -69,6 +72,8 @@ def save_model(self, request, obj, form, change):
 
 @admin.register(InstrumentOutputVariable)
 class InstrumentOutputVariableAdmin(admin.ModelAdmin):
+    list_display = ('variable', 'model', 'instrument_raw_output_unit', 'instrument_accuracy')
+
     def get_changelist_form(self, request, **kwargs):
         return super(InstrumentOutputVariableAdmin, self).get_changelist_form(request, **kwargs)
 

src/dataloaderinterface/forms.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-
+from django.contrib.auth import get_user_model
 from django.forms import NumberInput
 from django.forms.widgets import HiddenInput
 
@@ -14,6 +14,12 @@
     'Storm sewer', 'Stream gage', 'Tidal stream', 'Water quality station', 'Weather station', 'Wetland', 'Other'
 ]
 
+user_affiliations = [
+    affiliation[0]
+    for affiliation
+    in get_user_model().objects.filter(affiliation_id__isnull=False).values_list('affiliation_id')
+]
+
 
 class SiteTypeSelect(forms.Select):
     site_types = {
@@ -43,7 +49,7 @@ def label_from_instance(self, obj):
 
 class SiteRegistrationForm(forms.ModelForm):
     affiliation_id = forms.ModelChoiceField(
-        queryset=Affiliation.objects.for_display(),
+        queryset=Affiliation.objects.filter(affiliation_id__in=(user_affiliations)).for_display(),
         required=False,
         help_text='Select the user that deployed or manages the site',
         label='Deployed By'
@@ -91,13 +97,15 @@ class Meta:
         help_texts = {
             'organization_code': 'Enter a brief, but unique code to identify your organization (e.g., "USU" or "USGS")',
             'organization_name': 'Enter the name of your organization',
-            'organization_description': 'Enter a description for your organization'
+            'organization_description': 'Enter a description for your organization',
+            'organization_link': 'Enter a URL that links to the organization website'
         }
         fields = [
             'organization_code',
             'organization_name',
             'organization_type',
-            'organization_description'
+            'organization_description',
+            'organization_link'
         ]
 
 

src/dataloaderinterface/management/commands/check_data_loss.py

@@ -9,7 +9,8 @@
 
 
 class Command(BaseCommand):
-    help = ''
+    help = 'Checks for sites that haven\'t received any data for longer than the threshold ' \
+           'set by the user who set up the alert, and sends an email alert.'
 
     @staticmethod
     def send_email(email_address, subject, message):
@@ -46,4 +47,4 @@ def handle(self, *args, **options):
             success = Command.send_email(site_alert.user.email, subject, message)
             if success:
                 site_alert.last_alerted = datetime.utcnow()
-                site_alert.save()
+                site_alert.save()