WIP: use datumaro for labels and make labels agnostic of annotation type

examples/cpp/gRPC/src/queryImages.cpp

@@ -81,7 +81,7 @@ flatbuffers::Offset<seerep::fb::Point> createPoint(flatbuffers::grpc::MessageBui
   return pointBuilder.Finish();
 }
 
-flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<seerep::fb::LabelsWithInstanceWithCategory>>>
+flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<seerep::fb::LabelCategory>>>
 createLabelWithInstance(flatbuffers::grpc::MessageBuilder& mb, const seerep::fb::BoundingBox2DLabeled* bb,
                         const flatbuffers::String* category)
 {

seerep_com/CMakeLists.txt

@@ -16,7 +16,6 @@ find_package(Flatbuffers REQUIRED)
 list(APPEND PROTOBUF_IMPORT_DIRS ${SeerepMsgs_PROTOBUF_IMPORT_DIRS})
 
 set(MY_PROTO_FILES
-    protos/transfer_sensor_msgs.proto
     protos/meta_operations.proto
     protos/tf_service.proto
     protos/image_service.proto
@@ -32,7 +31,6 @@ set(MY_FBS_FILES
     fbs/meta_operations.fbs
     fbs/point_service.fbs
     fbs/tf_service.fbs
-    fbs/transfer_sensor_msgs.fbs
     fbs/point_cloud_service.fbs
     fbs/camera_intrinsics_service.fbs
 )

seerep_com/fbs/image_service.fbs

@@ -1,7 +1,7 @@
 
 include "image.fbs";
 include "query.fbs";
-include "boundingboxes2d_labeled_stamped.fbs";
+include "label_category.fbs";
 
 include "server_response.fbs";
 
@@ -10,5 +10,5 @@ namespace seerep.fb;
 rpc_service ImageService {
   GetImage(seerep.fb.Query):seerep.fb.Image (streaming: "server");
   TransferImage(seerep.fb.Image):seerep.fb.ServerResponse  (streaming: "client");
-  AddBoundingBoxes2dLabeled(seerep.fb.BoundingBoxes2DLabeledStamped):seerep.fb.ServerResponse   (streaming: "client");
+  AddBoundingBoxes2dLabeled(seerep.fb.LabelCategory):seerep.fb.ServerResponse   (streaming: "client");
 }

seerep_com/fbs/meta_operations.fbs

@@ -7,8 +7,7 @@ include "boundingbox.fbs";
 include "time_interval.fbs";
 include "uuid_datatype_pair.fbs";
 include "uuid_datatype_with_category.fbs";
-include "labels.fbs";
-include "categories.fbs";
+include "string_vector.fbs";
 
 namespace seerep.fb;
 
@@ -19,6 +18,6 @@ rpc_service MetaOperations {
   DeleteProject(seerep.fb.ProjectInfo):seerep.fb.Empty;
   GetOverallTimeInterval(seerep.fb.UuidDatatypePair):seerep.fb.TimeInterval;
   GetOverallBoundingBox(seerep.fb.UuidDatatypePair):seerep.fb.Boundingbox;
-  GetAllCategories(seerep.fb.UuidDatatypePair):seerep.fb.Categories;
-  GetAllLabels(seerep.fb.UuidDatatypeWithCategory):seerep.fb.Labels;
+  GetAllCategories(seerep.fb.UuidDatatypePair):seerep.fb.StringVector;
+  GetAllLabels(seerep.fb.UuidDatatypeWithCategory):seerep.fb.StringVector;
 }

seerep_com/fbs/point_cloud_service.fbs

@@ -1,7 +1,7 @@
 
 include "point_cloud_2.fbs";
 include "query.fbs";
-include "boundingboxes_labeled_stamped.fbs";
+include "label_category.fbs";
 
 include "server_response.fbs";
 
@@ -10,5 +10,5 @@ namespace seerep.fb;
 rpc_service PointCloudService {
     GetPointCloud2(seerep.fb.Query):seerep.fb.PointCloud2 (streaming: "server");
     TransferPointCloud2(seerep.fb.PointCloud2):seerep.fb.ServerResponse (streaming: "client");
-    AddBoundingBoxesLabeled(seerep.fb.BoundingBoxesLabeledStamped):seerep.fb.ServerResponse (streaming: "client");
+    AddBoundingBoxesLabeled(seerep.fb.LabelCategory):seerep.fb.ServerResponse (streaming: "client");
 }

seerep_com/fbs/tf_service.fbs

@@ -3,13 +3,13 @@ include "transform_stamped.fbs";
 include "transform_stamped_query.fbs";
 
 include "server_response.fbs";
-include "frame_infos.fbs";
+include "string_vector.fbs";
 include "frame_query.fbs";
 
 namespace seerep.fb;
 
 rpc_service TfService {
   TransferTransformStamped(seerep.fb.TransformStamped):seerep.fb.ServerResponse (streaming: "client");
-  GetFrames(seerep.fb.FrameQuery):seerep.fb.FrameInfos;
+  GetFrames(seerep.fb.FrameQuery):seerep.fb.StringVector;
   GetTransformStamped(seerep.fb.TransformStampedQuery):seerep.fb.TransformStamped;
 }

seerep_com/protos/meta_operations.proto

@@ -10,16 +10,15 @@ import "projectCreation.proto";
 import "uuid_datatype_pair.proto";
 import "time_interval.proto";
 import "boundingbox.proto";
-import "labels.proto";
-import "categories.proto";
 import "uuid_datatype_with_category.proto";
+import "string_vector.proto";
 
 service MetaOperations
 {
   rpc CreateProject(ProjectCreation) returns (ProjectInfo);
   rpc GetProjects(google.protobuf.Empty) returns (ProjectInfos);
   rpc GetOverallTimeInterval(UuidDatatypePair) returns (TimeInterval);
   rpc GetOverallBoundingBox(UuidDatatypePair) returns (Boundingbox);
-  rpc GetAllCategories(UuidDatatypePair) returns (Categories);
-  rpc GetAllLabels(UuidDatatypeWithCategory) returns (Labels);
+  rpc GetAllCategories(UuidDatatypePair) returns (StringVector);
+  rpc GetAllLabels(UuidDatatypeWithCategory) returns (StringVector);
 }

seerep_com/protos/tf_service.proto

@@ -4,15 +4,14 @@ package seerep.pb;
 
 import "transform_stamped.proto";
 import "transform_stamped_query.proto";
-// import "query.proto";
 
 import "server_response.proto";
-import "frame_infos.proto";
 import "frame_query.proto";
+import "string_vector.proto";
 
 service TfService
 {
   rpc TransferTransformStamped(TransformStamped) returns (ServerResponse);
-  rpc GetFrames(FrameQuery) returns (FrameInfos);
+  rpc GetFrames(FrameQuery) returns (StringVector);
   rpc GetTransformStamped(TransformStampedQuery) returns (TransformStamped);
 }

seerep_hdf5/seerep_hdf5_core/include/seerep_hdf5_core/hdf5_core_general.h

@@ -8,7 +8,8 @@
 #include <seerep_msgs/aabb.h>
 #include <seerep_msgs/dataset_indexable.h>
 #include <seerep_msgs/geodetic_coordinates.h>
-#include <seerep_msgs/labels_with_instance_with_category.h>
+#include <seerep_msgs/label.h>
+#include <seerep_msgs/label_category.h>
 
 // std
 #include <boost/geometry.hpp>
@@ -147,23 +148,23 @@ class Hdf5CoreGeneral
   template <class T>
   void readHeader(const std::string& id, HighFive::AnnotateTraits<T>& object, seerep_core_msgs::Header& header);
 
-  // BoundingBoxes
-  /**
-   * @brief read the labels and instances of a dataset of all categories and add them to the category-labels/instances map
-   *
-   * @param [in] datatypeGroup the data type
-   * @param [in] uuid the uuid of the dataset
-   * @param [in,out] labelsWithInstancesWithCategory the map from category to the instances of the category
-   */
-  void readBoundingBoxLabeledAndAddToLabelsWithInstancesWithCategory(
-      const std::string& datatypeGroup, const std::string& uuid,
-      std::unordered_map<std::string, std::vector<seerep_core_msgs::LabelWithInstance>>& labelsWithInstancesWithCategory);
-  void readBoundingBoxLabeled(const std::string& datatypeGroup, const std::string& uuid,
-                              std::vector<std::string>& labelCategories,
-                              std::vector<std::vector<std::string>>& labelsPerCategory,
-                              std::vector<std::vector<float>>& labelConfidencesPerCategory,
-                              std::vector<std::vector<std::vector<double>>>& boundingBoxesPerCategory,
-                              std::vector<std::vector<std::string>>& instancesPerCategory, bool loadBoxes = true);
+  // // BoundingBoxes
+  // /**
+  //  * @brief read the labels and instances of a dataset of all categories and add them to the category-labels/instances map
+  //  *
+  //  * @param [in] datatypeGroup the data type
+  //  * @param [in] uuid the uuid of the dataset
+  //  * @param [in,out] labelsWithInstancesWithCategory the map from category to the instances of the category
+  //  */
+  // void readBoundingBoxLabeledAndAddToLabelsWithInstancesWithCategory(
+  //     const std::string& datatypeGroup, const std::string& uuid,
+  //     std::unordered_map<std::string, std::vector<seerep_core_msgs::LabelWithInstance>>& labelsWithInstancesWithCategory);
+  // void readBoundingBoxLabeled(const std::string& datatypeGroup, const std::string& uuid,
+  //                             std::vector<std::string>& labelCategories,
+  //                             std::vector<std::vector<std::string>>& labelsPerCategory,
+  //                             std::vector<std::vector<float>>& labelConfidencesPerCategory,
+  //                             std::vector<std::vector<std::vector<double>>>& boundingBoxesPerCategory,
+  //                             std::vector<std::vector<std::string>>& instancesPerCategory, bool loadBoxes = true);
 
   // Labels General
   /**
@@ -174,15 +175,14 @@ class Hdf5CoreGeneral
    * @param [in] uuid the uuid of the dataset
    * @param [in,out] labelsWithInstancesWithCategory the map from category to the instances of the category
    */
-  void readLabelsGeneralAndAddToLabelsWithInstancesWithCategory(
+  void readLabelsAndAddToLabelsPerCategory(
       const std::string& datatypeGroup, const std::string& uuid,
-      std::unordered_map<std::string, std::vector<seerep_core_msgs::LabelWithInstance>>& labelsWithInstancesWithCategory);
-  void readLabelsGeneral(
-      const std::string& datatypeGroup, const std::string& uuid, std::vector<std::string>& labelCategories,
-      std::vector<std::vector<seerep_core_msgs::LabelWithInstance>>& labelsWithInstancesGeneralPerCategory);
-  void writeLabelsGeneral(
-      const std::string& datatypeGroup, const std::string& uuid,
-      const std::vector<seerep_core_msgs::LabelsWithInstanceWithCategory>& labelsWithInstanceWithCategory);
+      std::unordered_map<std::string, std::vector<seerep_core_msgs::Label>>& labelsCategoryMap);
+  void readLabels(const std::string& datatypeGroup, const std::string& uuid, std::vector<std::string>& labelCategories,
+                  std::vector<std::vector<seerep_core_msgs::Label>>& labelsPerCategory,
+                  std::vector<std::string>& datumaroJsonPerCategory);
+  void writeLabels(const std::string& datatypeGroup, const std::string& uuid,
+                   const std::vector<seerep_core_msgs::LabelCategory>& LabelsCategory);
   // ################
   //  Project
   // ################
@@ -273,11 +273,8 @@ class Hdf5CoreGeneral
   void getLabelCategories(std::string id, std::string labelType, std::vector<std::string>& matchingLabelCategory);
 
 private:
-  void readLabel(const std::string& id, const std::string labelType, std::vector<std::string>& labels);
-  void readlabelConfidences(const std::string& id, const std::string labelType, std::vector<float>& labelConfidences);
-  void readBoundingBoxes(const std::string& id, const std::string boundingBoxType,
-                         std::vector<std::vector<double>>& boundingBoxes);
-  void readInstances(const std::string& id, const std::string InstanceType, std::vector<std::string>& instances);
+  template <class T>
+  T readDataset(const std::string& path);
 
 public:
   // header attribute keys
@@ -303,13 +300,11 @@ class Hdf5CoreGeneral
 
   // dataset names
   inline static const std::string RAWDATA = "rawdata";
-  inline static const std::string LABELGENERAL = "labelGeneral";
-  inline static const std::string LABELGENERALCONFIDENCES = "labelGeneralConfidences";
-  inline static const std::string LABELGENERALINSTANCES = "labelGeneralInstances";
-  inline static const std::string LABELBB = "labelBB";
-  inline static const std::string LABELBBCONFIDENCES = "labelBBConfidences";
-  inline static const std::string LABELBBBOXESWITHROTATION = "labelBBBoxesWithRotation";
-  inline static const std::string LABELBBINSTANCES = "labelBBInstances";
+  inline static const std::string LABEL = "label";
+  inline static const std::string LABEL_ID_DATUMARO = "labelIdDatumaro";
+  inline static const std::string LABELINSTANCES = "labelInstances";
+  inline static const std::string LABELINSTANCES_ID_DATUMARO = "labelInstancesIdDatumaro";
+  inline static const std::string DATUMARO_JSON = "datumaroJson";
   inline static const std::string POINTS = "points";
 
 protected:

seerep_hdf5/seerep_hdf5_core/include/seerep_hdf5_core/hdf5_core_image.h

@@ -11,7 +11,7 @@
 
 // seerep_msgs
 #include <seerep_msgs/dataset_indexable.h>
-#include <seerep_msgs/labels_with_instance_with_category.h>
+#include <seerep_msgs/label_category.h>
 
 // std
 #include <boost/geometry.hpp>
@@ -86,11 +86,9 @@ class Hdf5CoreImage : public virtual Hdf5CoreGeneral, public Hdf5CoreDatatypeInt
    * @brief Write generals labels based on C++ data structures to HdF5
    *
    * @param uuid uuid of the image data group
-   * @param labelsWithInstanceWithCategory vector of labels with instances in multiple categories
+   * @param labelCategory vector of labels with instances in multiple categories
    */
-  void writeLabelsGeneral(
-      const std::string& uuid,
-      const std::vector<seerep_core_msgs::LabelsWithInstanceWithCategory>& labelsWithInstanceWithCategory);
+  void writeLabels(const std::string& uuid, const std::vector<seerep_core_msgs::LabelCategory>& labelCategory);
 
   /**
    * @brief Write the general attributes of an image to hdf5

seerep_hdf5/seerep_hdf5_core/include/seerep_hdf5_core/impl/hdf5_core_general.hpp

@@ -88,4 +88,14 @@ void Hdf5CoreGeneral::readHeader(const std::string& id, HighFive::AnnotateTraits
   header.frameId = Hdf5CoreGeneral::readFrameId(object, HEADER_FRAME_ID, id);
   header.sequence = Hdf5CoreGeneral::readAttributeFromHdf5<int32_t>(object, HEADER_SEQ, id);
 }
+
+template <class T>
+T Hdf5CoreGeneral::readDataset(const std::string& path)
+{
+  checkExists(path);
+  HighFive::DataSet datasetInstances = m_file->getDataSet(path);
+  T data;
+  datasetInstances.read(data);
+  return data;
+}
 }  // namespace seerep_hdf5_core