From 387bd1719eab84e7e90881d3ce1c251945518267 Mon Sep 17 00:00:00 2001 From: Dennis Smith Date: Fri, 20 Sep 2024 12:04:50 -0400 Subject: [PATCH] handle null/undefined/nan/empty string for measurement values --- api/internal/handler/datalogger_telemetry.go | 2 +- .../handler/measurement_inclinometer.go | 2 +- api/internal/handler/measurement_test.go | 8 ++-- api/internal/model/datalogger_parser.go | 21 ---------- api/internal/model/measurement.go | 39 ++++++++++++++++--- api/internal/model/timeseries_process.go | 4 +- api/internal/service/dcsloader.go | 2 +- api/internal/service/measurement.go | 2 +- .../service/measurement_inclinometer.go | 2 +- 9 files changed, 45 insertions(+), 37 deletions(-) diff --git a/api/internal/handler/datalogger_telemetry.go b/api/internal/handler/datalogger_telemetry.go index dda06a31..a32a35f4 100644 --- a/api/internal/handler/datalogger_telemetry.go +++ b/api/internal/handler/datalogger_telemetry.go @@ -178,7 +178,7 @@ func getCR6Handler(h *TelemetryHandler, dl model.Datalogger, rawJSON []byte) ech delete(eqtFields, f.Name) continue } - items[j] = model.Measurement{TimeseriesID: *row.TimeseriesID, Time: t, Value: v} + items[j] = model.Measurement{TimeseriesID: *row.TimeseriesID, Time: t, Value: model.FloatNanInf(v)} } mcs[i] = model.MeasurementCollection{TimeseriesID: *row.TimeseriesID, Items: items} diff --git a/api/internal/handler/measurement_inclinometer.go b/api/internal/handler/measurement_inclinometer.go index dd78cf94..85b813f7 100644 --- a/api/internal/handler/measurement_inclinometer.go +++ b/api/internal/handler/measurement_inclinometer.go @@ -51,7 +51,7 @@ func (h *ApiHandler) ListInclinometerMeasurements(c echo.Context) error { } for idx := range im.Inclinometers { - values, err := h.InclinometerMeasurementService.ListInclinometerMeasurementValues(ctx, tsID, im.Inclinometers[idx].Time, cm.Value) + values, err := h.InclinometerMeasurementService.ListInclinometerMeasurementValues(ctx, tsID, im.Inclinometers[idx].Time, float64(cm.Value)) if err != nil { return httperr.InternalServerError(err) } diff --git a/api/internal/handler/measurement_test.go b/api/internal/handler/measurement_test.go index f84be498..cbb88dc9 100644 --- a/api/internal/handler/measurement_test.go +++ b/api/internal/handler/measurement_test.go @@ -51,7 +51,7 @@ const createMeasurementsObjectBody = `{ "timeseries_id": "869465fc-dc1e-445e-81f4-9979b5fadda9", "items": [ {"time": "2020-06-01T00:00:00Z", "value": 10.00}, - {"time": "2020-06-02T01:00:00Z", "value": 11.10}, + {"time": "2020-06-02T01:00:00Z", "value": null}, {"time": "2020-06-03T02:00:00Z", "value": 10.20}, {"time": "2020-06-04T03:00:00Z", "value": 10.30}, {"time": "2020-06-05T04:00:00Z", "value": 10.40} @@ -65,7 +65,7 @@ const createMeasurementsArrayBody = `[ {"time": "2020-06-01T00:00:00Z", "value": 10.00}, {"time": "2020-06-02T01:00:00Z", "value": 11.10}, {"time": "2020-06-03T02:00:00Z", "value": 10.20}, - {"time": "2020-06-04T03:00:00Z", "value": 10.30}, + {"time": "2020-06-04T03:00:00Z", "value": null}, {"time": "2020-06-05T04:00:00Z", "value": 10.40} ] }, @@ -73,7 +73,7 @@ const createMeasurementsArrayBody = `[ "timeseries_id": "9a3864a8-8766-4bfa-bad1-0328b166f6a8", "items": [ {"time": "2020-06-01T00:00:00Z", "value": 10.00}, - {"time": "2020-06-02T01:00:00Z", "value": 11.10}, + {"time": "2020-06-02T01:00:00Z", "value": null}, {"time": "2020-06-03T02:00:00Z", "value": 10.20}, {"time": "2020-06-04T03:00:00Z", "value": 10.30}, {"time": "2020-06-05T04:00:00Z", "value": 10.40} @@ -86,7 +86,7 @@ const createMeasurementsArrayBody = `[ {"time": "2020-06-02T01:00:00Z", "value": 11.10}, {"time": "2020-06-03T02:00:00Z", "value": 10.20}, {"time": "2020-06-04T03:00:00Z", "value": 10.30}, - {"time": "2020-06-05T04:00:00Z", "value": 10.40} + {"time": "2020-06-05T04:00:00Z", "value": null} ] } ]` diff --git a/api/internal/model/datalogger_parser.go b/api/internal/model/datalogger_parser.go index 32dfdc7e..411404f7 100644 --- a/api/internal/model/datalogger_parser.go +++ b/api/internal/model/datalogger_parser.go @@ -2,9 +2,7 @@ package model import ( "encoding/csv" - "encoding/json" "log" - "math" "os" ) @@ -43,25 +41,6 @@ type Field struct { Settable bool `json:"settable"` } -type FloatNanInf float64 - -func (j *FloatNanInf) UnmarshalJSON(v []byte) error { - switch string(v) { - case `"NAN"`, "NAN": - *j = FloatNanInf(math.NaN()) - case `"INF"`, "INF": - *j = FloatNanInf(math.Inf(1)) - default: - var fv float64 - if err := json.Unmarshal(v, &fv); err != nil { - *j = FloatNanInf(math.NaN()) - return nil - } - *j = FloatNanInf(fv) - } - return nil -} - // ParseTOA5 parses a Campbell Scientific TOA5 data file that is simlar to a csv. // The unique properties of TOA5 are that the meatdata are stored in header of file (first 4 lines of csv) func ParseTOA5(filename string) ([][]string, error) { diff --git a/api/internal/model/measurement.go b/api/internal/model/measurement.go index ae873a08..18029e7e 100644 --- a/api/internal/model/measurement.go +++ b/api/internal/model/measurement.go @@ -3,7 +3,9 @@ package model import ( "context" "encoding/json" + "fmt" "math" + "strings" "time" "github.com/USACE/instrumentation-api/api/internal/util" @@ -46,13 +48,40 @@ func (cc *TimeseriesMeasurementCollectionCollection) UnmarshalJSON(b []byte) err // Measurement is a time and value associated with a timeseries type Measurement struct { - TimeseriesID uuid.UUID `json:"-" db:"timeseries_id"` - Time time.Time `json:"time"` - Value float64 `json:"value"` - Error string `json:"error,omitempty"` + TimeseriesID uuid.UUID `json:"-" db:"timeseries_id"` + Time time.Time `json:"time"` + Value FloatNanInf `json:"value"` + Error string `json:"error,omitempty"` TimeseriesNote } +type FloatNanInf float64 + +func (j FloatNanInf) MarshalJSON() ([]byte, error) { + if math.IsNaN(float64(j)) || math.IsInf(float64(j), 0) { + return []byte("null"), nil + } + + return []byte(fmt.Sprintf("%f", float64(j))), nil +} + +func (j *FloatNanInf) UnmarshalJSON(v []byte) error { + switch strings.ToLower(string(v)) { + case `"nan"`, "nan", "", "null", "undefined": + *j = FloatNanInf(math.NaN()) + case `"inf"`, "inf": + *j = FloatNanInf(math.Inf(1)) + default: + var fv float64 + if err := json.Unmarshal(v, &fv); err != nil { + *j = FloatNanInf(math.NaN()) + return nil + } + *j = FloatNanInf(fv) + } + return nil +} + // MeasurementLean is the minimalist representation of a timeseries measurement // a key value pair where key is the timestamp, value is the measurement { : } type MeasurementLean map[time.Time]float64 @@ -79,7 +108,7 @@ func (m Measurement) getTime() time.Time { } func (m Measurement) getValue() float64 { - return m.Value + return float64(m.Value) } // Should only ever be one diff --git a/api/internal/model/timeseries_process.go b/api/internal/model/timeseries_process.go index 81b5a9ad..d2bd6e5f 100644 --- a/api/internal/model/timeseries_process.go +++ b/api/internal/model/timeseries_process.go @@ -187,7 +187,7 @@ func (mrc *ProcessTimeseriesResponseCollection) CollectSingleTimeseries(threshol mmts[i] = Measurement{ TimeseriesID: t.TimeseriesID, Time: m.Time, - Value: m.Value, + Value: FloatNanInf(m.Value), Error: m.Error, } } @@ -548,7 +548,7 @@ func queryInclinometerTimeseriesMeasurements(ctx context.Context, q *Queries, f log.Println(err) } for i := range tt2[idx].Measurements { - values, err := q.ListInclinometerMeasurementValues(ctx, t.TimeseriesID, tt2[idx].Measurements[i].Time, cm.Value) + values, err := q.ListInclinometerMeasurementValues(ctx, t.TimeseriesID, tt2[idx].Measurements[i].Time, float64(cm.Value)) if err != nil { return nil, err } diff --git a/api/internal/service/dcsloader.go b/api/internal/service/dcsloader.go index a8a2f4c5..f1fd6880 100644 --- a/api/internal/service/dcsloader.go +++ b/api/internal/service/dcsloader.go @@ -71,7 +71,7 @@ func (s dcsLoaderService) ParseCsvMeasurementCollection(r io.Reader) ([]model.Me Items: make([]model.Measurement, 0), } } - mcMap[tsid].Items = append(mcMap[tsid].Items, model.Measurement{TimeseriesID: tsid, Time: t, Value: v}) + mcMap[tsid].Items = append(mcMap[tsid].Items, model.Measurement{TimeseriesID: tsid, Time: t, Value: model.FloatNanInf(v)}) mCount++ } diff --git a/api/internal/service/measurement.go b/api/internal/service/measurement.go index e2a08ee3..909185a0 100644 --- a/api/internal/service/measurement.go +++ b/api/internal/service/measurement.go @@ -38,7 +38,7 @@ type noteCbk func(context.Context, uuid.UUID, time.Time, model.TimeseriesNote) e func createMeasurements(ctx context.Context, mc []model.MeasurementCollection, mmtFn mmtCbk, noteFn noteCbk) error { for _, c := range mc { for _, m := range c.Items { - if err := mmtFn(ctx, c.TimeseriesID, m.Time, m.Value); err != nil { + if err := mmtFn(ctx, c.TimeseriesID, m.Time, float64(m.Value)); err != nil { return err } if m.Masked != nil || m.Validated != nil || m.Annotation != nil { diff --git a/api/internal/service/measurement_inclinometer.go b/api/internal/service/measurement_inclinometer.go index 746f5a23..8805a046 100644 --- a/api/internal/service/measurement_inclinometer.go +++ b/api/internal/service/measurement_inclinometer.go @@ -105,7 +105,7 @@ func (s inclinometerMeasurementService) CreateTimeseriesConstant(ctx context.Con } measurement.Time = time.Now() - measurement.Value = value + measurement.Value = model.FloatNanInf(value) measurements = append(measurements, measurement) mc.TimeseriesID = tsNew.ID mc.Items = measurements