diff --git a/dozer-api/src/rest/api_generator.rs b/dozer-api/src/rest/api_generator.rs index 067e47a6a7..0b3cd8cf7c 100644 --- a/dozer-api/src/rest/api_generator.rs +++ b/dozer-api/src/rest/api_generator.rs @@ -18,8 +18,7 @@ use crate::sql::datafusion::SQLExecutor; use crate::CacheEndpoint; use crate::{auth::Access, errors::ApiError}; use dozer_types::grpc_types::health::health_check_response::ServingStatus; -use dozer_types::json_types::field_to_json_value; -use dozer_types::serde_json::{json, Value}; +use dozer_types::json_types::{field_to_json_value, json, JsonValue}; use self::extractor::QueryExpressionExtractor; @@ -90,10 +89,8 @@ pub async fn list( } // Generated get function for health check -pub async fn health_route() -> Result { - let status = ServingStatus::Serving; - let resp = json!({ "status": status.as_str_name() }).to_string(); - Ok(HttpResponse::Ok().body(resp)) +pub async fn health_route() -> HttpResponse { + HttpResponse::Ok().json(json!({ "status": ServingStatus::Serving.as_str_name() })) } pub async fn count( @@ -132,7 +129,7 @@ fn get_records_map( access: Option>, cache_endpoint: ReqData>, exp: &mut QueryExpression, -) -> Result>, ApiError> { +) -> Result>, ApiError> { let mut maps = vec![]; let cache_reader = &cache_endpoint.cache_reader(); let records = get_records( @@ -153,7 +150,7 @@ fn get_records_map( fn record_to_map( record: CacheRecord, schema: &Schema, -) -> Result, CannotConvertF64ToJson> { +) -> Result, CannotConvertF64ToJson> { let mut map = IndexMap::new(); for (field_def, field) in schema.fields.iter().zip(record.record.values) { @@ -161,10 +158,10 @@ fn record_to_map( map.insert(field_def.name.clone(), val); } - map.insert("__dozer_record_id".to_string(), Value::from(record.id)); + map.insert("__dozer_record_id".to_string(), JsonValue::from(record.id)); map.insert( "__dozer_record_version".to_string(), - Value::from(record.version), + JsonValue::from(record.version), ); Ok(map) diff --git a/dozer-deno/src/runtime/conversion.rs b/dozer-deno/src/runtime/conversion.rs index 649a7040ab..3494cf1039 100644 --- a/dozer-deno/src/runtime/conversion.rs +++ b/dozer-deno/src/runtime/conversion.rs @@ -1,3 +1,5 @@ +use std::ops::Deref; + use deno_runtime::{ deno_core::{ anyhow::{bail, Context as _}, @@ -5,23 +7,23 @@ use deno_runtime::{ }, deno_napi::v8::{self, HandleScope, Local}, }; -use dozer_types::serde_json::{self, Number}; +use dozer_types::json_types::{DestructuredJson, JsonObject, JsonValue}; pub fn to_v8<'s>( scope: &mut HandleScope<'s>, - value: serde_json::Value, + value: JsonValue, ) -> Result, AnyError> { - match value { - serde_json::Value::Null => Ok(v8::null(scope).into()), - serde_json::Value::Bool(value) => Ok(v8::Boolean::new(scope, value).into()), - serde_json::Value::Number(value) => { - let value = value.as_f64().context("number is not a f64")?; + match value.destructure() { + DestructuredJson::Null => Ok(v8::null(scope).into()), + DestructuredJson::Bool(value) => Ok(v8::Boolean::new(scope, value).into()), + DestructuredJson::Number(value) => { + let value = value.to_f64().context("number is not a f64")?; Ok(v8::Number::new(scope, value).into()) } - serde_json::Value::String(value) => Ok(v8::String::new(scope, &value) - .context(format!("failed to create string {}", value))? + DestructuredJson::String(value) => Ok(v8::String::new(scope, &value) + .context(format!("failed to create string {}", value.deref()))? .into()), - serde_json::Value::Array(values) => { + DestructuredJson::Array(values) => { let array = v8::Array::new(scope, values.len() as i32); for (index, value) in values.into_iter().enumerate() { let value = to_v8(scope, value)?; @@ -29,11 +31,11 @@ pub fn to_v8<'s>( } Ok(array.into()) } - serde_json::Value::Object(map) => { + DestructuredJson::Object(map) => { let object = v8::Object::new(scope); for (key, value) in map.into_iter() { let key = v8::String::new(scope, &key) - .context(format!("failed to create key {}", key))?; + .context(format!("failed to create key {}", key.deref()))?; let value = to_v8(scope, value)?; object.set(scope, key.into(), value); } @@ -45,18 +47,18 @@ pub fn to_v8<'s>( pub fn from_v8<'s>( scope: &mut HandleScope<'s>, value: Local<'s, v8::Value>, -) -> Result { +) -> Result { if value.is_null_or_undefined() { - Ok(serde_json::Value::Null) + Ok(JsonValue::NULL) } else if value.is_boolean() { - Ok(serde_json::Value::Bool(value.boolean_value(scope))) + Ok(value.boolean_value(scope).into()) } else if value.is_number() { - Ok(serde_json::Value::Number( - Number::from_f64(value.number_value(scope).context("number is not a f64")?) - .context("f64 number cannot be represented in JSON")?, - )) + Ok(value + .number_value(scope) + .context("number is not a f64")? + .into()) } else if let Ok(value) = TryInto::>::try_into(value) { - Ok(serde_json::Value::String(value.to_rust_string_lossy(scope))) + Ok(value.to_rust_string_lossy(scope).into()) } else if let Ok(value) = TryInto::>::try_into(value) { let mut values = Vec::new(); for index in 0..value.length() { @@ -64,11 +66,11 @@ pub fn from_v8<'s>( let value = from_v8(scope, value)?; values.push(value); } - Ok(serde_json::Value::Array(values)) + Ok(values.into()) } else if let Ok(value) = TryInto::>::try_into(value) { - let mut map = serde_json::Map::new(); + let mut map = JsonObject::new(); let Some(keys) = value.get_own_property_names(scope, Default::default()) else { - return Ok(serde_json::Value::Object(map)); + return Ok(map.into()); }; for index in 0..keys.length() { let key = keys.get_index(scope, index).unwrap(); @@ -77,7 +79,7 @@ pub fn from_v8<'s>( let value = from_v8(scope, value)?; map.insert(key, value); } - Ok(serde_json::Value::Object(map)) + Ok(map.into()) } else { bail!("cannot convert v8 value to JSON because its type is not supported") } diff --git a/dozer-deno/src/runtime/mod.rs b/dozer-deno/src/runtime/mod.rs index b82b5a8edd..96b0f41cce 100644 --- a/dozer-deno/src/runtime/mod.rs +++ b/dozer-deno/src/runtime/mod.rs @@ -17,8 +17,8 @@ use deno_runtime::{ deno_napi::v8::{self, undefined, Function, Global, Local}, }; use dozer_types::{ + json_types::JsonValue, log::{error, info}, - serde_json::Value, thiserror, }; use tokio::{ @@ -118,8 +118,8 @@ impl Runtime { pub async fn call_function( &mut self, id: NonZeroI32, - args: Vec, - ) -> Result { + args: Vec, + ) -> Result { let (return_sender, return_receiver) = oneshot::channel(); if self .work_sender @@ -140,7 +140,7 @@ impl Runtime { } // Return type is actually `!` - async fn propagate_panic(&mut self) -> Result { + async fn propagate_panic(&mut self) -> Result { self.handle .take() .expect("runtime panicked before and cannot be used again") @@ -191,8 +191,8 @@ async fn load_functions( enum Work { CallFunction { id: NonZeroI32, - args: Vec, - return_sender: oneshot::Sender>, + args: Vec, + return_sender: oneshot::Sender>, }, } @@ -253,9 +253,9 @@ fn do_work(runtime: &mut JsRuntime, work: Work, functions: &HashMap, + args: Vec, functions: &HashMap>, -) -> Result { +) -> Result { let function = functions .get(&function) .context(format!("function {} not found", function))?; @@ -268,7 +268,7 @@ fn call_function( let result = Local::new(scope, function).call(scope, recv.into(), &args); result .map(|value| from_v8(scope, value)) - .unwrap_or(Ok(Value::Null)) + .unwrap_or(Ok(JsonValue::NULL)) } mod conversion; diff --git a/dozer-lambda/src/js/worker/mod.rs b/dozer-lambda/src/js/worker/mod.rs index 17d4f01bee..9d82bbcb6d 100644 --- a/dozer-lambda/src/js/worker/mod.rs +++ b/dozer-lambda/src/js/worker/mod.rs @@ -2,9 +2,8 @@ use std::{num::NonZeroI32, sync::Arc}; use dozer_log::tokio::runtime::Runtime; use dozer_types::{ - json_types::field_to_json_value, + json_types::{field_to_json_value, json, JsonObject, JsonValue}, log::error, - serde_json::{json, Value}, types::{Field, Operation}, }; @@ -46,10 +45,10 @@ impl Worker { } } -fn create_record_json_value(field_names: Vec, values: Vec) -> Value { - let mut record = Value::Object(Default::default()); +fn create_record_json_value(field_names: Vec, values: Vec) -> JsonValue { + let mut record = JsonObject::new(); for (field_name, value) in field_names.into_iter().zip(values.into_iter()) { - record[field_name] = field_to_json_value(value); + record.insert(field_name, field_to_json_value(value)); } - record + record.into() } diff --git a/dozer-log-js/src/mapper.rs b/dozer-log-js/src/mapper.rs index 0b290ea018..1a47418645 100644 --- a/dozer-log-js/src/mapper.rs +++ b/dozer-log-js/src/mapper.rs @@ -1,7 +1,8 @@ +use std::ops::Deref; + use dozer_log::replication::LogOperation; use dozer_types::{ - json_types::field_to_json_value, - serde_json::Value, + json_types::{field_to_json_value, DestructuredJson, JsonValue}, types::{Field, Operation, Record, Schema}, }; use neon::{ @@ -89,13 +90,13 @@ fn map_value<'a, C: Context<'a>>(value: Field, cx: &mut C) -> JsResult<'a, JsVal map_json_value(field_to_json_value(value), cx) } -fn map_json_value<'a, C: Context<'a>>(value: Value, cx: &mut C) -> JsResult<'a, JsValue> { - match value { - Value::Null => Ok(cx.null().upcast()), - Value::Bool(b) => Ok(cx.boolean(b).upcast()), - Value::Number(n) => Ok(cx.number(n.as_f64().unwrap_or(f64::NAN)).upcast()), - Value::String(s) => Ok(cx.string(s).upcast()), - Value::Array(a) => { +fn map_json_value<'a, C: Context<'a>>(value: JsonValue, cx: &mut C) -> JsResult<'a, JsValue> { + match value.destructure() { + DestructuredJson::Null => Ok(cx.null().upcast()), + DestructuredJson::Bool(b) => Ok(cx.boolean(b).upcast()), + DestructuredJson::Number(n) => Ok(cx.number(n.to_f64().unwrap_or(f64::NAN)).upcast()), + DestructuredJson::String(s) => Ok(cx.string(s.deref()).upcast()), + DestructuredJson::Array(a) => { let result = cx.empty_array(); for (i, v) in a.into_iter().enumerate() { let v = map_json_value(v, cx)?; @@ -103,7 +104,7 @@ fn map_json_value<'a, C: Context<'a>>(value: Value, cx: &mut C) -> JsResult<'a, } Ok(result.upcast()) } - Value::Object(o) => { + DestructuredJson::Object(o) => { let result = cx.empty_object(); for (k, v) in o.into_iter() { let v = map_json_value(v, cx)?; diff --git a/dozer-sql/expression/src/javascript/evaluate.rs b/dozer-sql/expression/src/javascript/evaluate.rs index a9be6761b2..96ffbd6b90 100644 --- a/dozer-sql/expression/src/javascript/evaluate.rs +++ b/dozer-sql/expression/src/javascript/evaluate.rs @@ -2,8 +2,6 @@ use std::{num::NonZeroI32, sync::Arc}; use dozer_deno::deno_runtime::deno_core::error::AnyError; use dozer_types::{ - errors::types::DeserializationError, - json_types::{json_value_to_serde_json, serde_json_to_json_value}, thiserror, types::{Field, FieldType, Record, Schema, SourceDefinition}, }; @@ -34,8 +32,6 @@ pub enum Error { CreateRuntime(#[from] dozer_deno::RuntimeError), #[error("failed to evaluate udf: {0}")] Evaluate(#[source] AnyError), - #[error("failed to convert json: {0}")] - JsonConversion(#[source] DeserializationError), } impl Udf { @@ -105,11 +101,8 @@ async fn evaluate_impl( let mut runtime = runtime.lock().await; let result = runtime - .call_function(function, vec![json_value_to_serde_json(&arg)]) + .call_function(function, vec![arg]) .await .map_err(Error::Evaluate)?; - drop(runtime); - - let result = serde_json_to_json_value(result).map_err(Error::JsonConversion)?; Ok(Field::Json(result)) } diff --git a/dozer-types/src/json_types.rs b/dozer-types/src/json_types.rs index e65a241d10..8a196a9b77 100644 --- a/dozer-types/src/json_types.rs +++ b/dozer-types/src/json_types.rs @@ -6,7 +6,7 @@ use chrono::SecondsFormat; use ordered_float::OrderedFloat; use prost_types::value::Kind; use prost_types::{ListValue, Struct, Value as ProstValue}; -use serde_json::{Map, Value}; +use serde_json::Value; use ijson::{Destructured, DestructuredRef, IArray, INumber, IObject, IValue}; @@ -88,43 +88,43 @@ pub(crate) fn json_cmp(l: &JsonValue, r: &JsonValue) -> std::cmp::Ordering { } } -fn convert_x_y_to_object((x, y): &(OrderedFloat, OrderedFloat)) -> Value { - let mut m = Map::new(); - m.insert("x".to_string(), Value::from(x.0)); - m.insert("y".to_string(), Value::from(y.0)); - Value::Object(m) +fn convert_x_y_to_object((x, y): (OrderedFloat, OrderedFloat)) -> JsonValue { + let mut object = JsonObject::new(); + object.insert("x", x.0); + object.insert("y", y.0); + object.into() } -fn convert_duration_to_object(d: &DozerDuration) -> Value { - let mut m = Map::new(); - m.insert("value".to_string(), Value::from(d.0.as_nanos().to_string())); - m.insert("time_unit".to_string(), Value::from(d.1.to_string())); - Value::Object(m) +fn convert_duration_to_object(duration: DozerDuration) -> JsonValue { + let mut object = JsonObject::new(); + object.insert("value", duration.0.as_nanos().to_string()); + object.insert("time_unit", duration.1.to_string()); + object.into() } /// Should be consistent with `convert_cache_type_to_schema_type`. -pub fn field_to_json_value(field: Field) -> Value { +pub fn field_to_json_value(field: Field) -> JsonValue { match field { - Field::UInt(n) => Value::from(n), - Field::U128(n) => Value::String(n.to_string()), - Field::Int(n) => Value::from(n), - Field::I128(n) => Value::String(n.to_string()), - Field::Float(n) => Value::from(n.0), - Field::Boolean(b) => Value::from(b), - Field::String(s) => Value::from(s), - Field::Text(n) => Value::from(n), - Field::Binary(b) => Value::from(b), - Field::Decimal(n) => Value::String(n.to_string()), - Field::Timestamp(ts) => Value::String(ts.to_rfc3339_opts(SecondsFormat::Millis, true)), - Field::Date(n) => Value::String(n.format(DATE_FORMAT).to_string()), - Field::Json(b) => json_value_to_serde_json(&b), - Field::Point(point) => convert_x_y_to_object(&point.0.x_y()), - Field::Duration(d) => convert_duration_to_object(&d), - Field::Null => Value::Null, + Field::UInt(n) => n.into(), + Field::U128(n) => n.to_string().into(), + Field::Int(n) => n.into(), + Field::I128(n) => n.to_string().into(), + Field::Float(n) => n.0.into(), + Field::Boolean(b) => b.into(), + Field::String(s) => s.into(), + Field::Text(n) => n.into(), + Field::Binary(b) => b.into(), + Field::Decimal(n) => n.to_string().into(), + Field::Timestamp(ts) => ts.to_rfc3339_opts(SecondsFormat::Millis, true).into(), + Field::Date(n) => n.format(DATE_FORMAT).to_string().into(), + Field::Json(b) => b, + Field::Point(point) => convert_x_y_to_object(point.0.x_y()), + Field::Duration(d) => convert_duration_to_object(d), + Field::Null => JsonValue::NULL, } } -pub fn json_value_to_serde_json(value: &JsonValue) -> Value { +fn json_value_to_serde_json(value: &JsonValue) -> Value { // Note that while this cannot fail, the other way might, as our internal JSON // representation does not support `inf`, `-inf` and NaN ijson::from_value(value).expect("Json to Json conversion should never fail") @@ -241,7 +241,8 @@ mod tests { let value = field_to_json_value(field.clone()); // Convert the JSON value back to a Field. - let deserialized = json_value_to_field(value, field_type, true).unwrap(); + let deserialized = + json_value_to_field(json_value_to_serde_json(&value), field_type, true).unwrap(); assert_eq!(deserialized, field); }