Merge pull request #241 from USACE/fix/null-measurement-values

handle null/undefined/nan/empty string for measurement values

api/internal/handler/datalogger_telemetry.go

@@ -178,7 +178,7 @@ func getCR6Handler(h *TelemetryHandler, dl model.Datalogger, rawJSON []byte) ech
 					delete(eqtFields, f.Name)
 					continue
 				}
-				items[j] = model.Measurement{TimeseriesID: *row.TimeseriesID, Time: t, Value: v}
+				items[j] = model.Measurement{TimeseriesID: *row.TimeseriesID, Time: t, Value: model.FloatNanInf(v)}
 			}
 
 			mcs[i] = model.MeasurementCollection{TimeseriesID: *row.TimeseriesID, Items: items}

api/internal/handler/measurement_inclinometer.go

@@ -51,7 +51,7 @@ func (h *ApiHandler) ListInclinometerMeasurements(c echo.Context) error {
 	}
 
 	for idx := range im.Inclinometers {
-		values, err := h.InclinometerMeasurementService.ListInclinometerMeasurementValues(ctx, tsID, im.Inclinometers[idx].Time, cm.Value)
+		values, err := h.InclinometerMeasurementService.ListInclinometerMeasurementValues(ctx, tsID, im.Inclinometers[idx].Time, float64(cm.Value))
 		if err != nil {
 			return httperr.InternalServerError(err)
 		}

api/internal/handler/measurement_test.go

@@ -51,7 +51,7 @@ const createMeasurementsObjectBody = `{
     "timeseries_id": "869465fc-dc1e-445e-81f4-9979b5fadda9",
     "items": [
     	    {"time": "2020-06-01T00:00:00Z", "value": 10.00},
-            {"time": "2020-06-02T01:00:00Z", "value": 11.10},
+            {"time": "2020-06-02T01:00:00Z", "value": null},
             {"time": "2020-06-03T02:00:00Z", "value": 10.20},
             {"time": "2020-06-04T03:00:00Z", "value": 10.30},
             {"time": "2020-06-05T04:00:00Z", "value": 10.40}
@@ -65,15 +65,15 @@ const createMeasurementsArrayBody = `[
 		{"time": "2020-06-01T00:00:00Z", "value": 10.00},
 	        {"time": "2020-06-02T01:00:00Z", "value": 11.10},
 	        {"time": "2020-06-03T02:00:00Z", "value": 10.20},
-	        {"time": "2020-06-04T03:00:00Z", "value": 10.30},
+	        {"time": "2020-06-04T03:00:00Z", "value": null},
 	        {"time": "2020-06-05T04:00:00Z", "value": 10.40}
 	]
     },
     {
         "timeseries_id": "9a3864a8-8766-4bfa-bad1-0328b166f6a8",
         "items": [
 		{"time": "2020-06-01T00:00:00Z", "value": 10.00},
-	        {"time": "2020-06-02T01:00:00Z", "value": 11.10},
+	        {"time": "2020-06-02T01:00:00Z", "value": null},
 	        {"time": "2020-06-03T02:00:00Z", "value": 10.20},
 	        {"time": "2020-06-04T03:00:00Z", "value": 10.30},
 	        {"time": "2020-06-05T04:00:00Z", "value": 10.40}
@@ -86,7 +86,7 @@ const createMeasurementsArrayBody = `[
 	        {"time": "2020-06-02T01:00:00Z", "value": 11.10},
 	        {"time": "2020-06-03T02:00:00Z", "value": 10.20},
 	        {"time": "2020-06-04T03:00:00Z", "value": 10.30},
-	        {"time": "2020-06-05T04:00:00Z", "value": 10.40}
+	        {"time": "2020-06-05T04:00:00Z", "value": null}
 	]
     }
 ]`

api/internal/model/datalogger_parser.go

@@ -2,9 +2,7 @@ package model
 
 import (
 	"encoding/csv"
-	"encoding/json"
 	"log"
-	"math"
 	"os"
 )
 
@@ -43,25 +41,6 @@ type Field struct {
 	Settable bool   `json:"settable"`
 }
 
-type FloatNanInf float64
-
-func (j *FloatNanInf) UnmarshalJSON(v []byte) error {
-	switch string(v) {
-	case `"NAN"`, "NAN":
-		*j = FloatNanInf(math.NaN())
-	case `"INF"`, "INF":
-		*j = FloatNanInf(math.Inf(1))
-	default:
-		var fv float64
-		if err := json.Unmarshal(v, &fv); err != nil {
-			*j = FloatNanInf(math.NaN())
-			return nil
-		}
-		*j = FloatNanInf(fv)
-	}
-	return nil
-}
-
 // ParseTOA5 parses a Campbell Scientific TOA5 data file that is simlar to a csv.
 // The unique properties of TOA5 are that the meatdata are stored in header of file (first 4 lines of csv)
 func ParseTOA5(filename string) ([][]string, error) {

api/internal/model/measurement.go

@@ -3,7 +3,9 @@ package model
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"math"
+	"strings"
 	"time"
 
 	"github.com/USACE/instrumentation-api/api/internal/util"
@@ -46,13 +48,40 @@ func (cc *TimeseriesMeasurementCollectionCollection) UnmarshalJSON(b []byte) err
 
 // Measurement is a time and value associated with a timeseries
 type Measurement struct {
-	TimeseriesID uuid.UUID `json:"-" db:"timeseries_id"`
-	Time         time.Time `json:"time"`
-	Value        float64   `json:"value"`
-	Error        string    `json:"error,omitempty"`
+	TimeseriesID uuid.UUID   `json:"-" db:"timeseries_id"`
+	Time         time.Time   `json:"time"`
+	Value        FloatNanInf `json:"value"`
+	Error        string      `json:"error,omitempty"`
 	TimeseriesNote
 }
 
+type FloatNanInf float64
+
+func (j FloatNanInf) MarshalJSON() ([]byte, error) {
+	if math.IsNaN(float64(j)) || math.IsInf(float64(j), 0) {
+		return []byte("null"), nil
+	}
+
+	return []byte(fmt.Sprintf("%f", float64(j))), nil
+}
+
+func (j *FloatNanInf) UnmarshalJSON(v []byte) error {
+	switch strings.ToLower(string(v)) {
+	case `"nan"`, "nan", "", "null", "undefined":
+		*j = FloatNanInf(math.NaN())
+	case `"inf"`, "inf":
+		*j = FloatNanInf(math.Inf(1))
+	default:
+		var fv float64
+		if err := json.Unmarshal(v, &fv); err != nil {
+			*j = FloatNanInf(math.NaN())
+			return nil
+		}
+		*j = FloatNanInf(fv)
+	}
+	return nil
+}
+
 // MeasurementLean is the minimalist representation of a timeseries measurement
 // a key value pair where key is the timestamp, value is the measurement { <time.Time>: <float32> }
 type MeasurementLean map[time.Time]float64
@@ -79,7 +108,7 @@ func (m Measurement) getTime() time.Time {
 }
 
 func (m Measurement) getValue() float64 {
-	return m.Value
+	return float64(m.Value)
 }
 
 // Should only ever be one

api/internal/model/timeseries_process.go

@@ -187,7 +187,7 @@ func (mrc *ProcessTimeseriesResponseCollection) CollectSingleTimeseries(threshol
 				mmts[i] = Measurement{
 					TimeseriesID: t.TimeseriesID,
 					Time:         m.Time,
-					Value:        m.Value,
+					Value:        FloatNanInf(m.Value),
 					Error:        m.Error,
 				}
 			}
@@ -548,7 +548,7 @@ func queryInclinometerTimeseriesMeasurements(ctx context.Context, q *Queries, f
 			log.Println(err)
 		}
 		for i := range tt2[idx].Measurements {
-			values, err := q.ListInclinometerMeasurementValues(ctx, t.TimeseriesID, tt2[idx].Measurements[i].Time, cm.Value)
+			values, err := q.ListInclinometerMeasurementValues(ctx, t.TimeseriesID, tt2[idx].Measurements[i].Time, float64(cm.Value))
 			if err != nil {
 				return nil, err
 			}

api/internal/service/dcsloader.go

@@ -71,7 +71,7 @@ func (s dcsLoaderService) ParseCsvMeasurementCollection(r io.Reader) ([]model.Me
 				Items:        make([]model.Measurement, 0),
 			}
 		}
-		mcMap[tsid].Items = append(mcMap[tsid].Items, model.Measurement{TimeseriesID: tsid, Time: t, Value: v})
+		mcMap[tsid].Items = append(mcMap[tsid].Items, model.Measurement{TimeseriesID: tsid, Time: t, Value: model.FloatNanInf(v)})
 		mCount++
 	}
 

api/internal/service/measurement.go

@@ -38,7 +38,7 @@ type noteCbk func(context.Context, uuid.UUID, time.Time, model.TimeseriesNote) e
 func createMeasurements(ctx context.Context, mc []model.MeasurementCollection, mmtFn mmtCbk, noteFn noteCbk) error {
 	for _, c := range mc {
 		for _, m := range c.Items {
-			if err := mmtFn(ctx, c.TimeseriesID, m.Time, m.Value); err != nil {
+			if err := mmtFn(ctx, c.TimeseriesID, m.Time, float64(m.Value)); err != nil {
 				return err
 			}
 			if m.Masked != nil || m.Validated != nil || m.Annotation != nil {

api/internal/service/measurement_inclinometer.go

@@ -105,7 +105,7 @@ func (s inclinometerMeasurementService) CreateTimeseriesConstant(ctx context.Con
 		}
 
 		measurement.Time = time.Now()
-		measurement.Value = value
+		measurement.Value = model.FloatNanInf(value)
 		measurements = append(measurements, measurement)
 		mc.TimeseriesID = tsNew.ID
 		mc.Items = measurements