Skip to content

Commit d0b9800

Browse files
Debaditya SenDebaditya Sen
Debaditya Sen
authored and
Debaditya Sen
committed
final working track + scan
1 parent 89a1681 commit d0b9800

File tree

14 files changed

+528
-427
lines changed

14 files changed

+528
-427
lines changed

‎mediapipe/calculators/core/gate_calculator.cc

+229-191
Large diffs are not rendered by default.

‎mediapipe/calculators/image/feature_detector_calculator_custom.cc

+1
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ namespace mediapipe
4747

4848
absl::Status Process(CalculatorContext *cc) override
4949
{
50+
ABSL_LOG(INFO) << "FeatureDetection runnning";
5051
const Timestamp &timestamp = cc->InputTimestamp();
5152
if (timestamp == Timestamp::PreStream())
5253
{

‎mediapipe/calculators/image/overlay_calculator.cc

+11-27
Original file line numberDiff line numberDiff line change
@@ -46,22 +46,11 @@ namespace mediapipe
4646
// Handle the first non-empty packet from SECONDARY_IMAGE
4747
if (!cc->Inputs().Tag("SECONDARY_IMAGE").IsEmpty())
4848
{
49-
ABSL_LOG(INFO) << "SECONDARY_IMAGE is not empty";
5049
if (cc->Inputs().Tag("PRIMARY_IMAGE").IsEmpty())
5150
{
5251
ABSL_LOG(INFO) << "PRIMARY_IMAGE is empty";
53-
// cv::Mat image(640, 480, CV_8UC3, cv::Scalar(255, 255, 255));
54-
// std::unique_ptr<ImageFrame> output_image_frame =
55-
// absl::make_unique<ImageFrame>(ImageFormat::SRGB, image.cols,
56-
// image.rows,
57-
// ImageFrame::kGlDefaultAlignmentBoundary);
58-
// image.copyTo(formats::MatView(output_image_frame.get()));
59-
// cc->Outputs()
60-
// .Tag("OVERLAYED_IMAGE")
61-
// .Add(output_image_frame.release(), cc->InputTimestamp());
6252
return absl::OkStatus();
6353
}
64-
ABSL_LOG(INFO) << "PRIMARY_IMAGE is not empty";
6554
const auto &secondary_image =
6655
cc->Inputs().Tag("SECONDARY_IMAGE").Get<ImageFrame>();
6756
const auto &primary_image =
@@ -100,7 +89,6 @@ namespace mediapipe
10089
const ImageFrame &secondary_image)
10190
{
10291
// Process the overlay
103-
ABSL_LOG(INFO) << "ProcessAndCopy executing";
10492
OverlayImages(cc, primary_image, secondary_image);
10593
ABSL_LOG(INFO) << "ProcessAndCopy done";
10694
return absl::OkStatus();
@@ -123,7 +111,6 @@ namespace mediapipe
123111
{
124112
cv::Mat primary_view = formats::MatView(&primary_image);
125113
cv::Mat secondary_view = formats::MatView(&secondary_image);
126-
127114
// Convert to grayscale for feature detection.
128115
cv::Mat primary_gray, secondary_gray;
129116
cv::cvtColor(primary_view, primary_gray, cv::COLOR_BGR2GRAY);
@@ -133,10 +120,8 @@ namespace mediapipe
133120
std::vector<cv::KeyPoint> primary_keypoints, secondary_keypoints;
134121
cv::Mat primary_descriptors, secondary_descriptors;
135122

136-
ABSL_LOG(INFO) << "Running ORB1";
137123
feature_detector_->detectAndCompute(
138124
primary_gray, cv::noArray(), primary_keypoints, primary_descriptors);
139-
ABSL_LOG(INFO) << "Running ORB2";
140125
feature_detector_->detectAndCompute(
141126
secondary_gray, cv::noArray(), secondary_keypoints,
142127
secondary_descriptors);
@@ -166,7 +151,6 @@ namespace mediapipe
166151
return;
167152
}
168153

169-
ABSL_LOG(INFO) << "Finding GoodMatch";
170154
const float ratio_thresh = 0.7f;
171155
std::vector<cv::DMatch> good_matches;
172156
for (size_t i = 0; i < knn_matches.size(); i++)
@@ -179,18 +163,16 @@ namespace mediapipe
179163
}
180164

181165
// Find homography matrix.
182-
ABSL_LOG(INFO) << "Finding Homography";
183166
std::vector<cv::Point2f> primary_points, secondary_points;
184167
for (const auto &match : good_matches)
185168
{
186169
primary_points.push_back(primary_keypoints[match.queryIdx].pt);
187170
secondary_points.push_back(secondary_keypoints[match.trainIdx].pt);
188171
}
189172
cv::Mat homography;
190-
if (primary_points.size() >= 4 && secondary_points.size() >= 4)
173+
if (primary_points.size() >= 8 && secondary_points.size() >= 8)
191174
{
192-
homography = cv::findHomography(secondary_points, primary_points,
193-
cv::RANSAC);
175+
homography = cv::findHomography(secondary_points, primary_points, cv::RANSAC);
194176
}
195177
else
196178
{
@@ -207,7 +189,7 @@ namespace mediapipe
207189
if (!homography.empty())
208190
{
209191
cv::warpPerspective(secondary_view, warped_secondary_image, homography,
210-
primary_view.size());
192+
primary_view.size(), cv::INTER_NEAREST, cv::BORDER_CONSTANT, 0);
211193
}
212194
else
213195
{
@@ -222,12 +204,14 @@ namespace mediapipe
222204
cv::Mat overlayed_image;
223205
if (!warped_secondary_image.empty())
224206
{
225-
overlayed_image = primary_view.clone(); // Create a copy of the primary
226-
// image.
227-
// You can use different blending modes here, e.g.,
228-
// cv::addWeighted(overlayed_image, 0.5, warped_secondary_image, 0.5, 0,
229-
// overlayed_image); // Blend with 50% transparency
230-
warped_secondary_image.copyTo(overlayed_image, warped_secondary_image);
207+
overlayed_image = primary_view.clone();
208+
cv::Mat mask;
209+
cv::cvtColor(warped_secondary_image, mask, cv::COLOR_BGR2GRAY);
210+
cv::threshold(mask, mask, 0, 0, cv::THRESH_BINARY_INV);
211+
cv::Mat inverted_mask;
212+
cv::bitwise_not(mask, inverted_mask);
213+
// Use the mask to overlay the warped secondary image
214+
warped_secondary_image.copyTo(overlayed_image, inverted_mask);
231215
}
232216
else
233217
{

‎mediapipe/calculators/image/reranking_calculator.cc

+3-1
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ namespace mediapipe
4545

4646
absl::Status Process(CalculatorContext *cc) override
4747
{
48+
ABSL_LOG(INFO) << "RerankCalculator runnning";
4849
if (cc->Inputs().Tag("FEATURES").IsEmpty() || cc->Inputs().Tag("QFEATURES").IsEmpty())
4950
{
5051
auto output = absl::make_unique<int>(-1);
@@ -88,11 +89,12 @@ namespace mediapipe
8889
}
8990
}
9091
match_counts[selectionIndex] = localMatchCount;
92+
ABSL_LOG(INFO) << "SCORES: " << match_counts[selectionIndex];
9193
}
9294

9395
// Find the index with the maximum match count
9496
int max_index = -1;
95-
int max_count = 4;
97+
int max_count = 8;
9698
for (int i = 0; i < match_counts.size(); ++i)
9799
{
98100
if (match_counts[i] > max_count)

‎mediapipe/examples/android/solutions/.gradle/nb-cache/subprojects.ser

0 Bytes
Binary file not shown.

‎mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/MainActivity.java

+15-7
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@
1313
// limitations under the License.
1414
package com.google.mediapipe.apps.basic;
1515

16+
import android.graphics.Color;
17+
1618
import android.content.pm.ApplicationInfo;
1719
import android.content.pm.PackageManager;
1820
import android.content.pm.PackageManager.NameNotFoundException;
@@ -187,7 +189,9 @@ protected void onCreate(Bundle savedInstanceState) {
187189
applicationInfo.metaData.getBoolean("flipFramesVertically", FLIP_FRAMES_VERTICALLY));
188190

189191
PermissionHelper.checkAndRequestCameraPermissions(this);
190-
192+
processor.getGraph().addPacketToInputStream("match_image", processor.getPacketCreator().createRgbImageFrame(
193+
Bitmap.createBitmap(640, 480, Bitmap.Config.ARGB_8888)), System.currentTimeMillis());
194+
processor.getGraph().addPacketToInputStream("enable_scanning", processor.getPacketCreator().createBool(true), System.currentTimeMillis());
191195
// Add packet callbacks for new outputs
192196
processor.addPacketCallback(
193197
"output_tensor_floats",
@@ -223,7 +227,7 @@ protected void onCreate(Bundle savedInstanceState) {
223227
Packet imagePacket = processor.getPacketCreator().createRgbImageFrame(bitmap);
224228
processor.getGraph().addPacketToInputStream("match_image", imagePacket,
225229
System.currentTimeMillis());
226-
processor.getGraph().closeInputStream("scanning_frame");
230+
processor.getGraph().addPacketToInputStream("enable_scanning", processor.getPacketCreator().createBool(false), System.currentTimeMillis());
227231
bitmap.recycle();
228232
} catch (ProtocolException ex) {
229233
} catch (IOException ex) {
@@ -414,11 +418,15 @@ public void onResponse(Call call, Response response) {
414418
} catch (IOException e) {
415419
Log.e(TAG, "Error reading server response: " + e.getMessage());
416420
} finally {
417-
processingLock.tryLock();
418-
processing = false;
419-
imgIdx = images;
420-
processingLock.unlock();
421-
response.close();
421+
try {
422+
processingLock.tryLock();
423+
processing = false;
424+
imgIdx = images;
425+
processingLock.unlock();
426+
response.close();
427+
} catch (Exception e) {
428+
response.close();
429+
}
422430
}
423431
}
424432
});

‎mediapipe/examples/android/src/java/com/google/mediapipe/apps/custom/BUILD

+1-1
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ android_binary(
6767
"inputVideoStreamName": "input_video_gpu",
6868
"outputVideoStreamName": "output_video",
6969
"flipFramesVertically": "True",
70-
"converterNumBuffers": "6",
70+
"converterNumBuffers": "8",
7171
},
7272
multidex = "native",
7373
deps = [

‎mediapipe/framework/input_side_packet_handler.cc

+67-58
Original file line numberDiff line numberDiff line change
@@ -20,71 +20,80 @@
2020
#include "mediapipe/framework/port/status_builder.h"
2121
#include "mediapipe/framework/tool/fill_packet_set.h"
2222

23-
namespace mediapipe {
23+
namespace mediapipe
24+
{
2425

25-
absl::Status InputSidePacketHandler::PrepareForRun(
26-
const PacketTypeSet* input_side_packet_types,
27-
const std::map<std::string, Packet>& all_side_packets,
28-
std::function<void()> input_side_packets_ready_callback,
29-
std::function<void(absl::Status)> error_callback) {
30-
int missing_input_side_packet_count;
31-
prev_input_side_packets_ = std::move(input_side_packets_);
32-
MP_ASSIGN_OR_RETURN(
33-
input_side_packets_,
34-
tool::FillPacketSet(*input_side_packet_types, all_side_packets,
35-
&missing_input_side_packet_count));
26+
absl::Status InputSidePacketHandler::PrepareForRun(
27+
const PacketTypeSet *input_side_packet_types,
28+
const std::map<std::string, Packet> &all_side_packets,
29+
std::function<void()> input_side_packets_ready_callback,
30+
std::function<void(absl::Status)> error_callback)
31+
{
32+
int missing_input_side_packet_count;
33+
prev_input_side_packets_ = std::move(input_side_packets_);
34+
MP_ASSIGN_OR_RETURN(
35+
input_side_packets_,
36+
tool::FillPacketSet(*input_side_packet_types, all_side_packets,
37+
&missing_input_side_packet_count));
3638

37-
input_side_packet_types_ = input_side_packet_types;
38-
missing_input_side_packet_count_.store(missing_input_side_packet_count,
39-
std::memory_order_relaxed);
40-
input_side_packets_ready_callback_ =
41-
std::move(input_side_packets_ready_callback);
42-
error_callback_ = std::move(error_callback);
43-
return absl::OkStatus();
44-
}
39+
input_side_packet_types_ = input_side_packet_types;
40+
missing_input_side_packet_count_.store(missing_input_side_packet_count,
41+
std::memory_order_relaxed);
42+
input_side_packets_ready_callback_ =
43+
std::move(input_side_packets_ready_callback);
44+
error_callback_ = std::move(error_callback);
45+
return absl::OkStatus();
46+
}
4547

46-
bool InputSidePacketHandler::InputSidePacketsChanged() {
47-
return prev_input_side_packets_ == nullptr ||
48-
input_side_packets_ == nullptr ||
49-
*input_side_packets_ != *prev_input_side_packets_;
50-
}
48+
bool InputSidePacketHandler::InputSidePacketsChanged()
49+
{
50+
return prev_input_side_packets_ == nullptr ||
51+
input_side_packets_ == nullptr ||
52+
*input_side_packets_ != *prev_input_side_packets_;
53+
}
5154

52-
void InputSidePacketHandler::Set(CollectionItemId id, const Packet& packet) {
53-
absl::Status status = SetInternal(id, packet);
54-
if (!status.ok()) {
55-
TriggerErrorCallback(status);
55+
void InputSidePacketHandler::Set(CollectionItemId id, const Packet &packet)
56+
{
57+
absl::Status status = SetInternal(id, packet);
58+
if (!status.ok())
59+
{
60+
TriggerErrorCallback(status);
61+
}
5662
}
57-
}
5863

59-
absl::Status InputSidePacketHandler::SetInternal(CollectionItemId id,
60-
const Packet& packet) {
61-
RET_CHECK_GT(missing_input_side_packet_count_, 0);
62-
Packet& side_packet = input_side_packets_->Get(id);
64+
absl::Status InputSidePacketHandler::SetInternal(CollectionItemId id,
65+
const Packet &packet)
66+
{
67+
// RET_CHECK_GT(missing_input_side_packet_count_, 0);
68+
Packet &side_packet = input_side_packets_->Get(id);
6369

64-
if (!side_packet.IsEmpty()) {
65-
return mediapipe::AlreadyExistsErrorBuilder(MEDIAPIPE_LOC)
66-
<< "Input side packet with id " << id << " was already set.";
67-
}
68-
absl::Status result = input_side_packet_types_->Get(id).Validate(packet);
69-
if (!result.ok()) {
70-
return mediapipe::StatusBuilder(result, MEDIAPIPE_LOC).SetPrepend()
71-
<< absl::StrCat(
72-
"Packet type mismatch on calculator input side packet with "
73-
"id ",
74-
id.value(), ": ");
70+
// if (!side_packet.IsEmpty()) {
71+
// return mediapipe::AlreadyExistsErrorBuilder(MEDIAPIPE_LOC)
72+
// << "Input side packet with id " << id << " was already set.";
73+
// }
74+
absl::Status result = input_side_packet_types_->Get(id).Validate(packet);
75+
if (!result.ok())
76+
{
77+
return mediapipe::StatusBuilder(result, MEDIAPIPE_LOC).SetPrepend()
78+
<< absl::StrCat(
79+
"Packet type mismatch on calculator input side packet with "
80+
"id ",
81+
id.value(), ": ");
82+
}
83+
side_packet = packet;
84+
if (missing_input_side_packet_count_.fetch_sub(
85+
1, std::memory_order_acq_rel) == 1)
86+
{
87+
input_side_packets_ready_callback_();
88+
}
89+
return absl::OkStatus();
7590
}
76-
side_packet = packet;
77-
if (missing_input_side_packet_count_.fetch_sub(
78-
1, std::memory_order_acq_rel) == 1) {
79-
input_side_packets_ready_callback_();
80-
}
81-
return absl::OkStatus();
82-
}
8391

84-
void InputSidePacketHandler::TriggerErrorCallback(
85-
const absl::Status& status) const {
86-
ABSL_CHECK(error_callback_);
87-
error_callback_(status);
88-
}
92+
void InputSidePacketHandler::TriggerErrorCallback(
93+
const absl::Status &status) const
94+
{
95+
ABSL_CHECK(error_callback_);
96+
error_callback_(status);
97+
}
8998

90-
} // namespace mediapipe
99+
} // namespace mediapipe

0 commit comments

Comments
 (0)