data-converted: always sort keys in output JSON (#2343)

This allows us to remove most uses of `jq` in the tests.
calyxir · Nov 6, 2024 · fd43d16 · fd43d16
1 parent 25f4b57
commit fd43d16
Show file tree

Hide file tree

Showing 3 changed files with 39 additions and 39 deletions.
diff --git a/interp/tests/runt.toml b/interp/tests/runt.toml
@@ -7,7 +7,7 @@ ver = "0.4.1"
 name = "unit"
 paths = ["unit/*.futil"]
 cmd = """
-../../target/debug/cider {} -l ../../ --dump-registers | ../../target/debug/cider-data-converter --to json | jq --sort-keys
+../../target/debug/cider {} -l ../../ --dump-registers | ../../target/debug/cider-data-converter --to json
 """
 timeout = 10
 expect_dir = "unit"
@@ -16,10 +16,9 @@ expect_dir = "unit"
 name = "multi-comp"
 paths = ["multi-comp/*.futil"]
 cmd = """
-fud2 --from calyx --to jq \
+fud2 --from calyx --to dat \
          --through cider \
          -s cider.flags="--dump-registers" \
-         -s jq.flags="--sort-keys" \
          {}
 """
 timeout = 10
@@ -38,10 +37,9 @@ name = "complex"
 paths = ["complex/*.futil"]
 
 cmd = """
-fud2 --from calyx --to jq \
+fud2 --from calyx --to dat \
          --through cider \
          -s cider.flags="--dump-registers" \
-         -s jq.flags="--sort-keys" \
          {}
 """
 timeout = 10
@@ -51,11 +49,10 @@ expect_dir = "complex"
 name = "primitives"
 paths = ["primitives/*.futil"]
 cmd = """
-fud2 --from calyx --to jq \
+fud2 --from calyx --to dat \
          --through cider \
          -s calyx.args="--log off" \
          -s cider.flags="--dump-registers" \
-         -s jq.flags="--sort-keys" \
          {}
 """
 timeout = 10
@@ -65,7 +62,7 @@ expect_dir = "primitives"
 name = "par to seq"
 paths = ["control/par_reg.futil", "control/par_mem.futil"]
 cmd = """
-../../target/debug/calyx {} -p par-to-seq -l ../../ | ../../target/debug/cider -l ../../  --dump-registers | ../../target/debug/cider-data-converter --to json | jq --sort-keys
+../../target/debug/calyx {} -p par-to-seq -l ../../ | ../../target/debug/cider -l ../../  --dump-registers | ../../target/debug/cider-data-converter --to json
 """
 timeout = 10
 expect_dir = "par-to-seq"
@@ -74,10 +71,9 @@ expect_dir = "par-to-seq"
 name = "control"
 paths = ["control/*.futil", "control/iteration/*.futil"]
 cmd = """
-fud2 --from calyx --to jq \
+fud2 --from calyx --to dat \
          --through cider \
          -s cider.flags="--dump-registers" \
-         -s jq.flags="--sort-keys" \
          {}
 """
 timeout = 10
@@ -87,25 +83,24 @@ expect_dir = "control"
 name = "invoke"
 paths = ["control/invoke/*.futil"]
 cmd = """
-fud2 {} --from calyx --to jq --through cider -s sim.data={}.data -s calyx.args="--log off" -s jq.flags="--sort-keys"
+fud2 {} --from calyx --to dat --through cider -s sim.data={}.data -s calyx.args="--log off"
 """
 timeout = 10
 
 [[tests]]
 name = "invoke compiled"
 paths = ["control/invoke/*.futil"]
 cmd = """
-fud2 {} --from calyx --to jq --through cider \
+fud2 {} --from calyx --to dat --through cider \
  -s cider.calyx-passes=" -p compile-invoke" \
- -s sim.data={}.data -s calyx.args="--log off" \
- -s jq.flags="--sort-keys"
+ -s sim.data={}.data -s calyx.args="--log off"
 """
 
 [[tests]]
 name = "fully structural"
 paths = ["control/*.futil", "control/iteration/*.futil"]
 cmd = """
-../../target/debug/calyx {} -d pre-opt -d post-opt -p simplify-with-control -l ../../ --log off | ../../target/debug/cider -l ../../  --dump-registers | ../../target/debug/cider-data-converter --to json | jq --sort-keys
+../../target/debug/calyx {} -d pre-opt -d post-opt -p simplify-with-control -l ../../ --log off | ../../target/debug/cider -l ../../  --dump-registers | ../../target/debug/cider-data-converter --to json
 """
 expect_dir = "control"
 # timeout = 10
@@ -123,11 +118,10 @@ expect_dir = "tests/lowered/"
 name = "correctness dynamic"
 paths = ["../../tests/correctness/*.futil"]
 cmd = """
-fud2 --from calyx --to jq \
+fud2 --from calyx --to dat \
          --through cider \
          -s sim.data={}.data \
          -s calyx.args="--log off" \
-         -s jq.flags="--sort-keys" \
          {}
 """
 
@@ -142,13 +136,12 @@ cmd = """
 name = "correctness lowered"
 paths = ["../../tests/correctness/*.futil"]
 cmd = """
-fud2 --from calyx --to jq \
+fud2 --from calyx --to dat \
          --through cider \
          -s sim.data={}.data \
          -s calyx.args="--log off" \
          -s cider.calyx-passes="-p all" \
          -s cider.flags="--no-verify" \
-         -s jq.flags="--sort-keys" \
          {}
 """
 timeout = 60
@@ -157,36 +150,33 @@ timeout = 60
 name = "correctness ieee754-float"
 paths = ["../../tests/correctness/ieee754-float/*.futil"]
 cmd = """
-fud2 --from calyx --to jq \
+fud2 --from calyx --to dat \
          --through cider \
          -s sim.data={}.data \
          -s calyx.args="--log off" \
-         -s jq.flags="--sort-keys" \
          {}
 """
 
 [[tests]]
 name = "correctness ref cells"
 paths = ["../../tests/correctness/ref-cells/*.futil"]
 cmd = """
-fud2 --from calyx --to jq \
+fud2 --from calyx --to dat \
          --through cider \
          -s sim.data={}.data \
          -s calyx.args="--log off" \
-         -s jq.flags="--sort-keys" \
          {}
 """
 
 [[tests]]
 name = "correctness ref cells compiled"
 paths = ["../../tests/correctness/ref-cells/*.futil"]
 cmd = """
-fud2 --from calyx --to jq \
+fud2 --from calyx --to dat \
          --through cider \
          -s sim.data={}.data \
          -s calyx.args="--log off" \
          -s cider.calyx-passes=" -p compile-invoke" \
-         -s jq.flags="--sort-keys" \
          {}
 """
 
@@ -198,48 +188,44 @@ paths = [
   "../../tests/correctness/numeric-types/fixed-point/*.futil",
 ]
 cmd = """
-fud2 --from calyx --to jq \
+fud2 --from calyx --to dat \
          --through cider \
          -s sim.data={}.data \
          -s calyx.args="--log off" \
          -s cider.converter-flags="-r --legacy-quotes" \
-         -s jq.flags="--sort-keys" \
          {}
 """
 
 [[tests]]
 name = "[frontend] tcam testing"
 paths = ["../../tests/correctness/tcam/*.futil"]
 cmd = """
-fud2 --from calyx --to jq \
+fud2 --from calyx --to dat \
       --through cider \
       -s calyx.args="--log off" \
       -s sim.data={}.data \
-      -s jq.flags="--sort-keys" \
       {}
 """
 
 [[tests]]
 name = "benchmark misc"
 paths = ["benchmarks/*.futil"]
 cmd = """
-fud2 --from calyx --to jq \
+fud2 --from calyx --to dat \
          --through cider \
          -s sim.data={}.data \
          -s calyx.args="--log off" \
-         -s jq.flags="--sort-keys" \
          {}
 """
 
 [[tests]]
 name = "benchmark polybench"
 paths = ["benchmarks/polybench/*.fuse"]
 cmd = """
-fud2 --from dahlia --to jq \
+fud2 --from dahlia --to dat \
          --through cider \
          -s sim.data={}.data \
          -s calyx.args="--log off" \
-          -s jq.flags="--sort-keys" \
          {}
 """
 timeout = 180
@@ -265,7 +251,6 @@ timeout = 180
 #       -s interpreter.flags "--raw " \
 #       -s verilog.data {}.data \
 #       -s jq.expr ".main" \
-#       -s jq.flags "--sort-keys " \
 #       {} -q
 # """
 # expect_dir = "tests/ntt-results/"
@@ -281,7 +266,6 @@ timeout = 180
 #             --through cider \
 #             -s sim.data={}.data \
 #             -s calyx.args="--log off" \
-#             -s jq.flags "--sort-keys " \
 #             {}
 # """
 
@@ -293,6 +277,5 @@ timeout = 180
 #             --through interpreter-out \
 #             -s interpreter.flags "--raw " \
 #             -s verilog.data {}.data \
-#             -s jq.flags "--sort-keys " \
 #             -s jq.expr ".main"
 # """
diff --git a/interp/tests/unit/uninitialized-port-returns-zero.expect b/interp/tests/unit/uninitialized-port-returns-zero.expect
@@ -1,3 +1,5 @@
+---CODE---
+1
 ---STDERR---
 Error: Attempted to write an undefined value to register or memory named "main.reg0"
 Error: Input is not a valid data dump
diff --git a/tools/cider-data-converter/src/json_data.rs b/tools/cider-data-converter/src/json_data.rs
@@ -1,9 +1,9 @@
-use std::{collections::HashMap, num::ParseFloatError, str::FromStr};
-
 use interp::serialization::Dimensions;
 use num_bigint::{BigInt, ParseBigIntError};
-use serde::{self, Deserialize, Serialize};
+use serde::{self, Deserialize, Serialize, Serializer};
 use serde_json::Number;
+use std::collections::BTreeMap;
+use std::{collections::HashMap, num::ParseFloatError, str::FromStr};
 use thiserror::Error;
 
 #[derive(Debug, Serialize, Deserialize, Clone, Copy, Eq, PartialEq)]
@@ -537,10 +537,25 @@ pub struct JsonData(pub HashMap<String, JsonDataEntry>);
 #[serde(untagged)]
 /// A structure meant to mimic the old style of data dump printing.
 pub enum JsonPrintDump {
+    #[serde(serialize_with = "ordered_map")]
     Normal(HashMap<String, ParseVec>),
+    #[serde(serialize_with = "ordered_map")]
     Quoted(HashMap<String, PrintVec>),
 }
 
+/// For use with serde's [serialize_with] attribute
+/// see: https://stackoverflow.com/questions/42723065/how-to-sort-hashmap-keys-when-serializing-with-serde
+fn ordered_map<S, K: Ord + Serialize, V: Serialize>(
+    value: &HashMap<K, V>,
+    serializer: S,
+) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+{
+    let ordered: BTreeMap<_, _> = value.iter().collect();
+    ordered.serialize(serializer)
+}
+
 impl JsonPrintDump {
     #[must_use]
     pub fn as_normal(&self) -> Option<&HashMap<String, ParseVec>> {