🐛 support Never in json read/write

Serializing to string tries to follow timestamp norms. However, the two infinity times are set as beyond regular date ranges and cause it to crash. We decided to use integers to represent the unix time (which is intuitive from reading it) and is much faster to read/write. However, due to limitations in JSON precision for integers (RFC 7159) we had to limit the range and eagerly interpret things to be either Never-times (future and past). Signed-off-by: Dominik Richter <[email protected]>
mondoohq · Sep 18, 2023 · 46cd566 · 46cd566
1 parent 72b59e2
commit 46cd566
Show file tree

Hide file tree

Showing 3 changed files with 21 additions and 7 deletions.
diff --git a/llx/primitives.go b/llx/primitives.go
@@ -93,10 +93,10 @@ func TimePrimitive(t *time.Time) *Primitive {
 }
 
 // NeverFutureTime is an indicator for what we consider infinity when looking at time
-var NeverFutureTime = time.Unix(1<<63-1, 0)
+var NeverFutureTime = time.Unix(1<<63-1, 0).UTC()
 
 // NeverPastTime is an indicator for what we consider negative infinity when looking at time
-var NeverPastTime = time.Unix(-(1<<63 - 1), 0)
+var NeverPastTime = time.Unix(-(1<<63 - 1), 0).UTC()
 
 // NeverFuturePrimitive is the special time primitive for the infinite future time
 var NeverFuturePrimitive = TimePrimitive(&NeverFutureTime)

diff --git a/llx/rawdata.go b/llx/rawdata.go
@@ -33,6 +33,12 @@ func (r *RawData) MarshalJSON() ([]byte, error) {
 		return json.Marshal(errData{Error: r.Error.Error()})
 	}
 
+	if r.Type == types.Time {
+		tv := r.Value.(*time.Time)
+		ut := tv.Unix()
+		return json.Marshal(RawData{Type: r.Type, Value: ut})
+	}
+
 	type rd2 RawData
 	return json.Marshal((*rd2)(r))
 }
@@ -44,11 +50,17 @@ func (r *RawData) UnmarshalJSON(data []byte) error {
 		case types.Int:
 			r.Value = int64(r.Value.(float64))
 		case types.Time:
-			v, err := time.Parse(time.RFC3339, r.Value.(string))
-			if err != nil {
-				return errors.New("failed to parse time into raw data: " + err.Error())
+			tv := r.Value.(float64)
+			// JSON serialization of numbers is limited to 1**53 precision, see:
+			// https://stackoverflow.com/questions/13502398/json-integers-limit-on-size#comment80159722_13502497
+			if tv > (1 << 53) {
+				r.Value = &NeverFutureTime
+			} else if tv < (-(1 << 53)) {
+				r.Value = &NeverPastTime
+			} else {
+				v := time.Unix(int64(tv), 0).UTC()
+				r.Value = &v
 			}
-			r.Value = &v
 		}
 		return nil
 	}

diff --git a/llx/rawdata_test.go b/llx/rawdata_test.go
@@ -190,8 +190,10 @@ func TestRawData_JSON(t *testing.T) {
 		RegexData(""),
 		RegexData("r"),
 		TimeData(time.Time{}),
+		TimeData(NeverFutureTime),
+		TimeData(NeverPastTime),
 		// TODO: the raw comparison here does not come out right, because of nano time
-		// TimeData(now),
+		// TimeData(now.UTC()),
 		ArrayData([]interface{}{"a", "b"}, types.String),
 		MapData(map[string]interface{}{"a": "b"}, types.String),
 		{Error: errors.New("test")},