~

.vscode/settings.json

@@ -70,6 +70,12 @@
         "variant": "cpp",
         "any": "cpp",
         "fstream": "cpp",
-        "*.inc": "cpp"
+        "*.inc": "cpp",
+        "csignal": "cpp",
+        "codecvt": "cpp",
+        "coroutine": "cpp",
+        "source_location": "cpp",
+        "future": "cpp",
+        "__nullptr": "cpp"
     }
 }

mediapipe/calculators/tflite/BUILD

@@ -484,6 +484,16 @@ cc_library(
     alwayslink = 1,
 )
 
+cc_library(
+    name = "tflite_tensors_to_bytestream_calculator",
+    srcs = ["tflite_tensors_to_bytestream_calculator.cc"],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/port:ret_check",
+        "@org_tensorflow//tensorflow/lite:framework",
+    ],
+    alwayslink = 1,
+)
 # To run this with native GPU on Linux, use:
 # bazel test //mediapipe/calculators/tflite:tflite_inference_calculator_test --copt=-DTFLITE_GPU_EXTRA_GLES_DEPS --copt=-DMESA_EGL_NO_X11_HEADERS --copt=-DEGL_NO_X11 --config=grte_v5 --test_strategy=local
 cc_test(

mediapipe/calculators/tflite/tflite_tensors_to_bytestream_calculator.cc

@@ -0,0 +1,85 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "tensorflow/lite/interpreter.h"
+#include <sstream>
+
+namespace mediapipe {
+
+constexpr char kTensorsTag[] = "TENSORS";
+constexpr char kBytesTag[] = "BYTES";
+
+// A calculator for converting TFLite tensors to a byte stream.
+//
+// Input:
+//  TENSORS - Vector of TfLiteTensor of type kTfLiteFloat32. Only the first
+//            tensor will be used.
+// Output:
+//  BYTES - Converted byte stream.
+//
+// Usage example:
+// node {
+//   calculator: "TfLiteTensorsToByteStreamCalculator"
+//   input_stream: "TENSORS:tensors"
+//   output_stream: "BYTES:bytes"
+// }
+class TfLiteTensorsToByteStreamCalculator : public CalculatorBase {
+ public:
+    static absl::Status GetContract(CalculatorContract* cc) {
+        RET_CHECK(cc->Inputs().HasTag(kTensorsTag));
+        RET_CHECK(cc->Outputs().HasTag(kBytesTag));
+
+        cc->Inputs().Tag(kTensorsTag).Set<std::vector<TfLiteTensor>>();
+        cc->Outputs().Tag(kBytesTag).Set<std::string>();
+
+        return absl::OkStatus();
+    }
+
+    absl::Status Open(CalculatorContext* cc) override {
+        cc->SetOffset(TimestampDiff(0));
+        return absl::OkStatus();
+    }
+
+    absl::Status Process(CalculatorContext* cc) override {
+        RET_CHECK(!cc->Inputs().Tag(kTensorsTag).IsEmpty());
+
+        const auto& input_tensors =
+            cc->Inputs().Tag(kTensorsTag).Get<std::vector<TfLiteTensor>>();
+        // TODO: Add option to specify which tensor to take from.
+        const TfLiteTensor* raw_tensor = &input_tensors[0];
+        const float* raw_floats = raw_tensor->data.f;
+        int num_values = 1;
+        for (int i = 0; i < raw_tensor->dims->size; ++i) {
+            RET_CHECK_GT(raw_tensor->dims->data[i], 0);
+            num_values *= raw_tensor->dims->data[i];
+        }
+
+        // Create a byte stream from the float array
+        std::ostringstream oss;
+        // Write the size of the vector (4 bytes).
+        oss.write(reinterpret_cast<const char*>(&num_values), sizeof(num_values));
+        // Write the float data (4 bytes per float).
+        oss.write(reinterpret_cast<const char*>(raw_floats), sizeof(float) * num_values);
+        std::string bytes = oss.str();
+
+        cc->Outputs()
+            .Tag(kBytesTag)
+            .AddPacket(MakePacket<std::string>(bytes).At(cc->InputTimestamp()));
+        return absl::OkStatus();
+    }
+    };
+    REGISTER_CALCULATOR(TfLiteTensorsToByteStreamCalculator);
+}  // namespace mediapipe

mediapipe/examples/desktop/BUILD

@@ -37,6 +37,7 @@ cc_library(
         "@com_google_absl//absl/log:absl_log",
         "@com_google_absl//absl/strings",
     ],
+    linkopts = ["-lcurl"]
 )
 
 cc_library(

mediapipe/examples/desktop/custom/BUILD

@@ -4,6 +4,7 @@ package(default_visibility = ["//visibility:public"])
 
 cc_binary(
     name = "custom",
+    linkstatic = True,
     deps = [
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/formats:image_frame",

mediapipe/examples/desktop/demo_run_graph_main.cc

@@ -29,7 +29,7 @@
 #include "mediapipe/framework/port/status.h"
 #include "mediapipe/util/resource_util.h"
 
-constexpr char kInputStream[] = "input_video";
+constexpr char kInputStream[] = "input_video_cpu";
 constexpr char kOutputStream[] = "transformed_input_video_cpu";
 constexpr char kWindowName[] = "MediaPipe";
 

mediapipe/examples/desktop/simple_run_graph_main.cc

@@ -1,24 +1,21 @@
-// Copyright 2019 The MediaPipe Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-// A simple main function to run a MediaPipe graph.
 #include <cstdlib>
 #include <fstream>
 #include <iostream>
 #include <map>
-#include <string>
 #include <vector>
+#include <stdexcept>
+#include <asio.hpp>
+#include <asio/ts/internet.hpp>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <netdb.h>
+#include <sstream>
+#include <cstdio>
+#include <cstring>
+#include <curl/curl.h>
 #include "mediapipe/framework/port/opencv_core_inc.h"
 #include "mediapipe/framework/port/opencv_features2d_inc.h"
 #include "mediapipe/framework/port/opencv_imgproc_inc.h"
@@ -44,8 +41,6 @@ ABSL_FLAG(std::string, input_side_packets, "",
           "for the CalculatorGraph. All values will be treated as the "
           "string type even if they represent doubles, floats, etc.");
 
-// Local file output flags.
-// Output stream
 ABSL_FLAG(std::string, output_stream, "",
           "The output stream to output to the local file in csv format.");
 ABSL_FLAG(std::string, output_stream_file, "",
@@ -54,13 +49,71 @@ ABSL_FLAG(std::string, output_stream_file, "",
 ABSL_FLAG(bool, strip_timestamps, false,
           "If true, only the packet contents (without timestamps) will be "
           "written into the local file.");
-// Output side packets
 ABSL_FLAG(std::string, output_side_packets, "",
           "A CSV of output side packets to output to local file.");
 ABSL_FLAG(std::string, output_side_packets_file, "",
           "The name of the local file to output all side packets specified "
           "with --output_side_packets. ");
 
+// Function to send HTTP POST request with binary data
+absl::Status SendHttpPostRequest(const std::string &url, const std::string &data)
+{
+  CURL *curl = curl_easy_init();
+  if (!curl)
+  {
+    return absl::InternalError("Failed to initialize curl");
+  }
+
+  curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
+  curl_easy_setopt(curl, CURLOPT_POST, 1L); // Use POST method
+
+  // Set headers for binary data
+  struct curl_slist *headers = NULL;
+  headers = curl_slist_append(headers, "Content-Type: application/octet-stream"); // Indicate binary data
+  curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
+
+  // Provide the binary data and its size
+  curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data.data());
+  curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, data.size());
+  auto start_time = std::chrono::high_resolution_clock::now();
+  CURLcode res = curl_easy_perform(curl);
+  if (res != CURLE_OK)
+  {
+    curl_slist_free_all(headers);
+    curl_easy_cleanup(curl);
+    return absl::InternalError(absl::StrCat("curl_easy_perform() failed: ", curl_easy_strerror(res)));
+  }
+  auto end_time = std::chrono::high_resolution_clock::now();
+
+  // Calculate RTT in milliseconds
+  auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end_time - start_time);
+  ABSL_LOG(INFO) << "RTT: " << duration.count() << "ms";
+  curl_slist_free_all(headers);
+  curl_easy_cleanup(curl);
+  return absl::OkStatus();
+}
+
+// absl::Status OutputStreamToLocalFile(mediapipe::OutputStreamPoller &poller)
+// {
+//   mediapipe::Packet packet;
+//   while (poller.Next(&packet))
+//   {
+//     const auto &byte_string = packet.Get<std::string>();
+//     // ABSL_LOG(INFO) << "Float Result" << byte_string;
+//     // Send the byte string using your SendHttpPostRequest function
+//     absl::Status status = SendHttpPostRequest("https://us-central1-development-382019.cloudfunctions.net/mp-dump-test", byte_string);
+//     if (!status.ok())
+//     {
+//       ABSL_LOG(ERROR) << "Error sending HTTP POST request: " << status.message();
+//     }
+//     else
+//     {
+//       ABSL_LOG(INFO) << "HTTP POST request sent successfully.";
+//     }
+//   }
+//   return absl::OkStatus();
+// }
+
 absl::Status OutputStreamToLocalFile(mediapipe::OutputStreamPoller &poller)
 {
   std::ofstream file;
@@ -72,14 +125,11 @@ absl::Status OutputStreamToLocalFile(mediapipe::OutputStreamPoller &poller)
     // Check if the packet contains KeyPoin
     absl::StrAppend(&output_data, "TIME:::", packet.Timestamp().Value(), "\n");
     const auto &descriptors = packet.Get<std::vector<float>>();
-    std::stringstream ss;
-    for (size_t i = 0; i < descriptors.size(); ++i)
+    for (const float &value : descriptors)
     {
-      if (i != 0)
-        ss << ",";
-      ss << descriptors[i];
+      absl::StrAppend(&output_data, value);
+      absl::StrAppend(&output_data, ",");
     }
-    absl::StrAppend(&output_data, ss.str());
     absl::StrAppend(&output_data, "\n");
     file << output_data;
   }
@@ -89,6 +139,7 @@ absl::Status OutputStreamToLocalFile(mediapipe::OutputStreamPoller &poller)
 
 absl::Status OutputSidePacketsToLocalFile(mediapipe::CalculatorGraph &graph)
 {
+
   if (!absl::GetFlag(FLAGS_output_side_packets).empty() &&
       !absl::GetFlag(FLAGS_output_side_packets_file).empty())
   {

mediapipe/graphs/custom/BUILD

@@ -27,6 +27,7 @@ cc_library(
         "//mediapipe/calculators/tflite:tflite_converter_calculator",
         "//mediapipe/calculators/tflite:tflite_inference_calculator",
         "//mediapipe/calculators/tflite:tflite_tensors_to_floats_calculator",
+        "//mediapipe/calculators/tflite:tflite_tensors_to_bytestream_calculator",
     ],
 )
 

mediapipe/graphs/custom/feature_extraction_desktop.pbtxt

@@ -1,7 +1,5 @@
-
-output_stream: "floats"
-
-max_queue_size: 100
+output_stream: "bytes"
+max_queue_size: 5
 
 profiler_config {
   trace_enabled: true
@@ -29,9 +27,9 @@ node: {
   output_stream: "IMAGE:transformed_input_video_cpu"
   node_options: {
     [type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] {
-      output_width: 640
-      output_height: 480
-      scale_mode: STRETCH
+      output_width: 224
+      output_height: 224
+      scale_mode: FIT
     }
   }
 }
@@ -42,14 +40,31 @@ node {
   output_stream: "TENSORS:cropped_image_tensor"
   options: {
     [mediapipe.TfLiteConverterCalculatorOptions.ext] {
-      zero_center: true
+      zero_center: false
+    }
+  }
+}
+
+
+node {
+  calculator: "FlowLimiterCalculator"
+  input_stream: "cropped_image_tensor"
+  input_stream: "FINISHED:bytes"
+  input_stream_info: {
+    tag_index: 'FINISHED'
+    back_edge: true
+  }
+  options : {
+    [mediapipe.FlowLimiterCalculatorOptions.ext] {
+      max_in_flight: 1
     }
   }
+  output_stream: "throttled_image_tensor"
 }
 
 node {
   calculator: "TfLiteInferenceCalculator"
-  input_stream: "TENSORS:cropped_image_tensor"
+  input_stream: "TENSORS:throttled_image_tensor"
   output_stream: "TENSORS:tensors"
   options: {
     [mediapipe.TfLiteInferenceCalculatorOptions.ext] {
@@ -62,6 +77,6 @@ node {
 node {
   calculator: "TfLiteTensorsToFloatsCalculator"
   input_stream: "TENSORS:tensors"
-  output_stream: "FLOATS:floats"
+  output_stream: "FLOATS:bytes"
 }
-num_threads: 32
+num_threads: 2