From 2d5dac7fed3dc6dba5b4bcae89e9d3b6da6342c2 Mon Sep 17 00:00:00 2001 From: Debaditya Sen Date: Fri, 9 Aug 2024 22:46:39 +0530 Subject: [PATCH] motionvector --- .vscode/settings.json | 14 +- .../solutions-1214149408/project-info.ser | Bin 0 -> 55803 bytes ...73242814EAEAB93B6A628E41DCF6A4C252F832C623 | 1 + mediapipe/calculators/image/BUILD | 26 ++- .../image/frame_joiner_calculator.cc | 79 ++++++++ .../calculators/image/reranking_calculator.cc | 2 +- ...y_calculator.cc => tracking_calculator.cc} | 159 ++++++++++++---- .../.gradle/nb-cache/subprojects.ser | Bin 1280 -> 1280 bytes .../com/google/mediapipe/apps/custom/BUILD | 2 +- .../framework/port/opencv_features2d_inc.h | 4 +- mediapipe/graphs/custom/BUILD | 13 +- .../feature_extraction_desktop copy.pbtxt | 178 ++++++++++++++++++ .../custom/feature_extraction_desktop.pbtxt | 102 +++++++++- mediapipe/graphs/custom/subgraphs/BUILD | 10 +- .../custom/subgraphs/scanning_subgraph.pbtxt | 16 +- .../custom/subgraphs/tracking_subgraph.pbtxt | 17 +- 16 files changed, 554 insertions(+), 69 deletions(-) create mode 100644 bin/mediapipe/examples/android/solutions/.gradle/nb-cache/solutions-1214149408/project-info.ser create mode 100644 bin/mediapipe/examples/android/solutions/.gradle/nb-cache/trust/D574649DA6D8FAEA3AFAF373242814EAEAB93B6A628E41DCF6A4C252F832C623 create mode 100644 mediapipe/calculators/image/frame_joiner_calculator.cc rename mediapipe/calculators/image/{overlay_calculator.cc => tracking_calculator.cc} (55%) create mode 100644 mediapipe/graphs/custom/feature_extraction_desktop copy.pbtxt diff --git a/.vscode/settings.json b/.vscode/settings.json index 17d18d8..940b1c0 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -103,7 +103,15 @@ "*.ipp": "cpp", "slist": "cpp", "shared_mutex": "cpp", - "__mutex_base": "cpp" + "__mutex_base": "cpp", + "fft": "cpp", + "*.traits": "cpp", + "nnls": "cpp", + "format": "cpp", + "__memory": "cpp", + "filesystem": "cpp", + "alignedvector3": "cpp", + "*.expression": "cpp" }, "java.compile.nullAnalysis.mode": "automatic", "java.completion.favoriteStaticMembers": [ @@ -137,7 +145,9 @@ "bazel-bin": false, "bazel-mediapipe": false, "bazel-out": false, - "bazel-testlogs": false + "bazel-testlogs": false, + "bin": false, + "com": false }, "java.configuration.updateBuildConfiguration": "automatic" } \ No newline at end of file diff --git a/bin/mediapipe/examples/android/solutions/.gradle/nb-cache/solutions-1214149408/project-info.ser b/bin/mediapipe/examples/android/solutions/.gradle/nb-cache/solutions-1214149408/project-info.ser new file mode 100644 index 0000000000000000000000000000000000000000..a7bbef500c6bb092e1b5d38e06dd2e732d37a6e9 GIT binary patch literal 55803 zcmeHw3#^>kUEf@Ly-z!9?|L8J>}D^!-h>3teD9YpA?vKY8#j)#v7L~{3g&xWk3Dy0 znE9@KZ%}zi1zPn9ZJ-od+LB78fdZvG8X*J}p$bvdMlB#j0U-nh6$+F#YAV$J{^#+1 z=X;IsojaRIBw1xVcV^Bx|MP#p&iTIo4|_-Ev))%sW}Aa?nn!6ko)5Mr$@VCn4>o6E zGD-(=7~f0>=c0K&3*-F!aDMw0`s6fwy)e#ayFLEDwL`u8{!;Iue7Kd)^Kff=vG>58 zbT%JO#+Q2cC1D=sy(cf-3h#v4aH3sJXTxwb{8$)`(v3^KN9L35S)9H)#KL*+QU0^- zd^o}n^NpQpuebKCPhUzO!%C+B@O$qK@awb5tu)Rrj&W-=W@-OT9<=rZ4Qo=`;uAdG9d}%P1UgYFG2waJ&g{?;B#;DxB{Nr}*N*>1=`> zZOy-?_jSEPmwFF;4Xv2(=Dpw&VAdp(HGyP}ucw96p>{cXjbU8CcA5d3?_$;mX6bY? z%lV``(!jK;+w-e>AX7gd&2om%zWg5U;CMV_vBaqr^CHNul61uO~T|=aKdDE z5yZYZOXoKyqhx3Lu9rC*+m z{}UJ9bnJKjAH&3bQ>;tWII=U%dmq<6I0sK^Ng9O-=n~G;u_icOW1K#hr0JVlM(o%( z2Zjc84#IIVn+%gdKADW>1GX_qK6_vebj`92)AZBGx9lv@9WQmF%rLDW2~x{TZO65v z&@$shjOw#dgInS3?xCKx!>Ids-usCqK_rpL)A4*V%SMyCTKTpQ22A@~X&#agYWlz! z=>1u2YF-!yk(C;5;ybY!hN+WgnTJidj%7HZ<;6kfSw`47SHGC6v5SAXc+P2hQ4m6W)>QmW4U2u_+Db!zH2368n_8qJkd3EYzMgiXvN@0 z+hZ6e5XDwszMDm!RvX{v;~QZA zi~Ui#cJ^~o{`s9k6WJ+e*533c0ni0O%8W^U{{y5YHc;2OSdVpsL?T~Pny zHvnqb)pR5Q4gxRA{0!R7joi?T!ywSXBVK4j|2v86XQ}Ciy2}A(L%U%8&Q-yx&6C@G z2;0_VT!87C&;hocrIsFqiJlsT)aB9 z7LLQwZa$3Xuzk@iob77y3~q)*IFwrmr3=~1g4FSS--MZt<2cF^*ESze>LSjF*36nlA}4`FuZz{Mv)gMju*qFuywc^L7Zf|ndq(yXTu8g#9cI1 z$4Pdq)%BHNA)J#Y-m^{hCtZCF6arSFlG5&B*MvK-!l=5Jpa znv-M;?sl>TL6qX>1g>kCffG6KguomLDLTj?7(0f$42`S@DJ-}c)+K-cd}S~;)dj`6 zw@Z8Nn1SsEz7>TYFdIbyJb4|STN-&;1lI6FBXlCe)5N$gkiT_RAd@LjF={1x01q;R zk}xt0_%rn&3PUGz!Z@`;ybHb5vc#}1@K)#jjKbZ?c1{o$?^I6$R>JdO)BH4u+{{D( zLPF&^MuHFo?gXrgWkJzZM|Xk!cUFegK4Bf=_%*0Q1@@SyD44pCpaN)Y3`~D@Nqagkk5^=L|h)mq= z2}EeD@qhaBJOOw1ZzZ>V#69)^PFU(1R+Kun3$Za$7g-0}^$`QRfgX8jl)0vphKPb& zra%BhQ#9<=uPh09mw|;w9M7}KY>Uf3VlKU(q|;d%hk2Uxx96!EY1q&TvEw?X7uzNx z4bMjS8AJ|3kHpFnA6v6Bkbx9!IW1B&3#V;t^5<5&Ng^4g6=#}!LoRcSblPMFQW0zs z(l0vF5Rn<`hLdHEn<7tv_|Ny9zzSmztY)XK>G>`qShr;&5UiZ2v7=>mATiwJt~m(j zTN?f^{J6R1A*{ANAKaek9vnHxH9{DYz)TXj6Hq$vCh&|l^TXK47-({2D|oBIT8{iMqbE_#HIq0(QpB^u%mu$8 zM`I%;CQu_vjY>! zI|sp{6X>=&5Z-jq;QsGf5@7;HXp+tCaF%HDjnJ2x3rVvy5HO81SWVXrkh2ZIa=vdE zdTRM@Z27RQj%mS~)ra@l@CLA-S`yd}_1~fy14Z#1w!2yqFyVHC!$Wi>!*o!f+ zL_q>+uPids7=D-QdXW!5&<{c*(hWGdt_iOObrig3C7^8&!sbqFeQ!P zy@o!#?GTW|3yU&63-ruK0v)CapSrf75^6~-|JwlgAepa){GbU|unj$O4W#UK-w#6s z2GDD;0Rgh(h;ow5wEe&Vx74_)gA0E6;9NlY2PBgQQxti~JA^+Ck!Hw6g-Rdz{ctqB z313Oe3>~Q#B;?{KvTRRxGab=m7Wv4S!?{I`&4I`6#mWq*ODQunu)XRUPdL5(F^U^` zCdLk8h?$oe7LuYy7Wzn!ng$XxFux`|6Bqx%(xs^RvSA$=n7?*#9N!vlWpj}JvS=l-qm1kTZ`HMY$Rp4x(dfsv<$%;W z%ssM{aRTSq3*tC1kg+iF|17pae3D{}>HzT|;!^QK?#@udNBMBzzzy+H;zm{qalVhR z-_C3kdJx`H;ADs~V-(^{gqLo_+z>tzEQpZ}7X0odS&+mfnrx5b-8h`*aGs-hdV9D- zxe3ETVapgeu4cPH5TG<&sK6|VOxyKr*!K*|+Q?9~w~`2T7gS{{gQ0gE_YH+~Hn~%U z6vZlwYC&*{Lbyr^Vz(3teket%#i7WBq_3IA$ezcrP{rVONDm@4!j>oDCJD6o#~eW2 zSsJ2B-|_(wYE}p;4d_ERP2h?8feSY;c3rr7E^@ql7${D|8i@ScOOl*?;hn*7Oifjc zv5G$o%|&@CLF^HFIvh;LL*)|Y#0Wv$$OCYO<0E}$`IT|NUcL2YS=fg8V0%2=8V-{A zP|LuP%nSCB;LU`_y$s9urP6k;2Z_2P)BTstRs+FjO(*;4cL~oETjxM zS!0x@2e;-(8p?=Mc~AD)LH!q93&`ao6ND%(NswK(TnE`O)A4m=TTsfeBFJ?)u+>U` z*OD|=6gTg3(!%k0g2p{`!WAwr^b83yYSx(pew#7 zcUeG5|AbT4n0PbKr+|Ap8K>iXuF;=c=s<)TcLuIzxV8oVHZ%+v8N*9qnoT>^sc;h##FCs!pmycSXK6MfB_(p?Xjeie5Z2p5RxU$S3ID?g+|-NUdjxKfpz+N| ztSNKkVmJh=H9x*=W7}Rz>C_X#y*S8_6+_Hmi&E5ZZ6t|(l=t8sBUhJXs5M$J1Tkt) zWno_AZaZ9&WjR>Y3Cdzxgx1MRqPc>@?CK~mqlgD-^UVY(HyjuwD6PaxLeCD->qH!q zs6#t|{gwl;ML5n!GbkUt+Dx=8Ed$O_53%8b*lud0*9Or~Nd5;*m5UZglynJfF^Y** z({ldJC4mA4LD+d8UC+u}b6{%`e91Jl2(&PS9~YPo8h~tQCf!80giis5fsP<$9L&1i zkfMLHJY*Hy1S3iv+usI3vDneKqYGVN`l;_)E(hy|#;dUZrXQ4!9I5wKB zkZM8**PH-ig$WuvSl03rcM7M8nb%ENMvc8t%;jnU(%GH`3@7LUvrVX1lrKy;cBzjT z4)!1p0%Q^*HipG%P2GPGCwY@PpR?ioCU(3-4*5>hpKXt+M;^|+H83?;cONAIA3i*q z6yd)o9vT2`GtAJ@iq17SD{hpUVPz~J#n=WG-`J|7Cp9Xljsm{#pn!meKUjSevARLc zCIR|>Le%Pz@j|DB)Tm|Q((BTy|7=MPUStROvJ&Xf0dJxvgy0FXA6SUkZ4!FVfnGpE zBdQ`n9H8Z(T5avH0?RrsojH9{ry`?|786R%CFC(%kTV;$)`I4Q>w?lg+Bs|wJ+9FG zh(%aspihH4^7%iqBw`i83b&E2y%UbMk+Q2jgKH+kI~8l zE>078Emn+dxf`K+4*rfhW&vwpmIhD%=)pIT&SsMtkTp&<_VxyQe?#2?mqpb5^%xx# z2w{-O0T)NG4DfR(yh6t)dKOUsfokrWrb8_M>XKVv%$-cOv|D&Fp)Yi3G?W$L1Ndo- zu6D}j8U`4`@=!7f5!#@yBBKN^+9%i`K!iaJq%NCWWDt>xr3ger@4b)SI7{j_#yp$E{-fqWRwJD~A4O>AWK0Uqow4_xH122=-80wNj^2DUJm_#3ST^EtEts)$fr zA&Mf7=K^OAT7c2ihZYL7ayjs}O|)|vI{N6M#&GZ^jvLg&)n(hIs!9vJ*6C<8yfr{q z#2kGMlRK$>3-Ia#LKJ#U49zrFuqnVMF zqZ?o^YioB>oY0zU{GW@cnSvwPAq;eN#y;vvc7P@gw09!wZ9;yKwe=ky&=XLKrQi z=J_BdzfGW)Eix-QlsP9E8fc^KO{C=KGfQ_+n*ohA)}n(5M2{`YClA_!_msf*hyF*a z3o?13=Vy8%Mxio?Q4Pc`n@q0oq8hSm2H`I-VA7Ke2`3Z1r3jZ%B1m9+P_Rdr1sWkk z6M%-*V4p@d09@9KQjo}N23a^9ZO>4Pr~VoyCul}Tb>2u&@t`5?uF}wpC-RrPK^)>o?UZX*wgH%n0 z4}Qy>M7K~sV*k`!1PgH=(rt!8?k*@|A=^*q%{&hkbF{0%`HhnRi5YNuV?0RJ3>ubo z4T$GSAa91-blUaS1bjc-3P=4p4(s5!Sq*T**D@THb5TVPS@8_Lm1quuFX?y=vYP~W zgpyU}px>x-A~GR<1Lw>7k>E8JQ;sH^o79tE|J#6KvC;JbXBX0I!%at38V5Dd8>L%r z5Ti5Hfx&Ye^eof|1Jb5`IY7;n0ZRC$5-kUt{{@Vo$qZ=BP<@7hzHpf(XH@yILpHU%-Vp?U?38#ejqw4DVjtz zAN+_k5o`$zFWP8PY{3~w-$9Mij&T+vOI;U?+BQdr`?_r2gRokQfAgKY+%lV5OpQyh zB}f_CW(*@38+Pm=O&7Toi=p<5JdFuojK*|8wrsphntHxH9zZ{%&0pjH$kxXqJ|`>Y zYYBoy&q5~$+R;Ls#R?)il!bmsH?%V3YH_RuwadWmn1z|^rraN-;e#M2UZ6q;okO1^ zQS&O^(@}SNPvUE!{Fb7cuH(#ag3~NfhE^ZwHe@_aj?1PCU9nKRWG)Fry6}e3Gxp&OM z!EY$pWudS5M(xemT%iHwtQ{L?VsUoCMXd(L069kVTUsc9rf9?0(I1c)uzZFEMN!Q$ z@JRrCYr=Wi6uD&b64AgK)$o)L2?Vp=Fab(YST^t7kAukoGVdKLm&d&qxM76)kB8~} z>UQ)xzqEt_@Y`6A^29hth2O9-^ylEbAOf&w*4E|7NQU9&cmmrH&j(_{D=FO{a*c4M zVphQsef%5rf_*=XC8yhLb9wb<4G;{uGim|gGoByf90#39%6re%U|~2=zNec}XdnaN zcWI^HV$0FX@6pR|)60LOm*2kO&6C`)+zb$IEV;6H5^fFcmC=n_A?;v39-jZ?DX4}bQ9b~xOusCt?6hRzWW?K z6=?0jfn`Z7J{qjCJuRnrs#xzDIwxP1J!*30qjRuOE~uT0+hOR=n!&pz%RXMrz_!NZ zjIac_@5>={dG9!5GB%3ebVFZrij;`O)^TR%;yB)ppbN&E^v!267CT8}1i64ApTv_< z-utrFU09PzeDN)P z&bbP_f8;oemd;9y(z`-V zH)_)~RS@I03*$ROB;dEu?r={G??;N?r7T}caYM3jK%1Sp^BQ^~Cg{z_No*OYK{Vc6iNb?;IV2iv~ zxa-UB2}dcpP%OjEoA+j~htw@ULl9mC#zraKPjzil&PP9k4So~S@+n(rB+du7bh=~Pbsd3WxDOI+LKdc5(`ZhqEgnGOjXZ7oj6u%{4^M<&nerj@ ztbkth^`ApK2MTF!-PvNDjA&>XBHtrDEuz-oDp>Ryyd(DA{93njydNF?in4ggfU2*- zC6+wLilM!?u6~d6grVY|#*Pzo)q^;)Hu{=@D8(GW{F3 zrDKqFWI%Rbn@zTP)B|_@AyTfS3z&>TdXB2U>fSuQSvy@(e!U_{?(NXH&s4rG{h>q5 zwNyMOsU02+M?7Mu(Z31dd#1+!=eZZPBI$Z(ZT8YONh_UGGJqTG(w++0>e!qjS1??q z2gkOAjWcW`IJvrwGZ<_RxCO`Q)60`F^KMC)>DLsp)91qFVG-?uAQb3&TFHM%?q!ti zfc0?RJM!Y~bQcc%r_a6i`i+a1uU)uu`P`)&H{N&g@{QLoeegB>^Pvk@u3miqa#0xhvN$oZr|ONAl9u^c6blw6VbtI3+h=*y+aC8!*Q=SclAfV?$n+6mCuj z;u=?eLTUgJBfFg@8>&MJO&iWOm4S#7&+!{GKU|%Sh{o?RsZW^qPD?t*NQkg#ys9L{ z-h1$jZb)q|R(XwIp2~u1$pY~%+i5!Efi&W>gy{5HVE$AK=08MDPSxa7DFa9&oh+`9 zQZ`57^K2XT{B-f{0-P1U)&~2dAK+d?Q2qEo5dneoy!U+n!&^J1Zu?v7{qrpIf>;(Y z&v^5lAH4Qz-_Lt*s{q$c0!W@>_jol@PfrzZWP1c8>!HN~PUkQy|C276Xbu(6-61q@#?G)*;-HDNbqu zq2*uc4lZhj=%aAWF1OVXO)%$o&Tk46s5vQDRK>HRqN-SUy!<_PKH;7Lf8{jXs@)hP zsz#!Sf}eFUWc_Ugbo~Mt)M7cDZ-dh(=iVfnnD?LyWcugEFj+(!nN@-2h)3JVdNbv{ zXv9&u76vSb|KC%GC&=Sc)X|J7B@&$0&E^ z+b|sn^pVDsWJzVz9~8G+6p&;<{Un`lt;E2+R(x3;2*i< zUtaZ+#R_8G`_JBlG;p?9wNSB{**cKpf?S2hd1QM`=>&E~IoO)CzU9Kl zDpq$Yyc*yF^2ocy$TwLVfiV;(v_lvlpr8H#qXNDBtK!`cf^E)LeAu-grY}#@%fBvu z_9OWktLAcY{Y9q>aab8Dl-Q}~Xcv-+aQCD1LKG@e@JrC=p|e6CGbtkK^!|Y6tx)IB zKJQWk7QKUXJv8sCgzI8CWkYm`M1@ozObThL9CAD#sA(7Z0E^eTA*MuAa-}GXs+4aZ zU&6khQDU|BAq`DXVKEdS8gc9r+;;M8%RICYQwtwyFkbL^XS~mux~{5)0!<4t2r8Q_ zrXK4tD6@-85bn{lM2T{Wf@be*(9HS5q1pZtx_hVBab~-fGUXB!J8`x?NkOwm8#MF2 zaA?-9P+9P$Abc4ec|r$N<#F0y+Md?egg=> zTh3UQ9hKwNeGid50C*k?IfXj4S^OHln-sYpeN zfccn5>cl_v`S(};0#LUtxG7a|yEScB+)^Gsx1rI!U|r0s&*4}va!KiX*%j%`m$=H> z99VnG63qJ+MCA)+l&Wm)!*y=%{ogl0mF-Vg?L!G2EVN!sll!Q2u>|UmoTY&h!@WPL zV|eY<7Y>GFwM~=tI`3V7A4lJIrTuRSTAVmrpG4AS?a2mRzUvEzE~Gi3AJ9dQqAI%X z#5 zZOb|4k5I)y3@-?CzCoD(^?4`EUr@LFbL&`RHP!{$Ze$i4iX%)dEhYABWWinUZE)8Q zf04VEtt+>C7NxMXQ@VZBH8>dUzhA@|)E&C^r;IOnQ`5R=YC5q!>pgy*JtLjhlKIK8 zjUV{*hkoz(pZnU^=yBe7dV7z0sMmgxUZ`ny?HBUi4RnH2YwLVHznQKN`RNV(n@`rm zb(m{mEjtpUQg|D{!FEhaR)Qj$$5cqtq$RsrkEk z@2e||#HzDP1P-z7aJ(L+>)d%vlY^1-Cm51{3|)n_b>;CZY9~v!apCBCw7V~2F2L%4 zXN&$pb5X^!UuQ;szldG1=pPqY{-eD2T4lG`qA=?FU9ZY5KZKqUmO)u(hfZn1Pok&v zA{tqSIJNPybPkCer+3$1V=cDptjblBFKTn<%9kDey*75XK{e4n2D4G@6-ZK$&{?q1Hk#&joQ4w&gT8K+Pwe1nD^)N-d7d#qJf=n zh&L*h3J0zh{)k6NG98dvf3;Zom-F70%F0qJP`mSrR{M`w?GT-~SoJ zc;(Y>g}YDi7()9d{o92_sDV$gZ8 z{wY}!X*>8&Nca?9dXV%lDF`GBjI6apTqW=#JLh~K-N8j&tQ|3SHyF>#eGY1H(8Pe? z&|C7}nI;5u#9-l>#gim-^*L!rI6U%uA-?%9DzvmKBCdDJcG_!xrs~3v{r8bD31j*=2fWsd};nGj3p6E zBzY?u0vV{{@WXd15N~XV1#$2~To$phk*H^ea7d%L&0%AMh{|Bt*pLU+=|j?Sls(v{ z8y>3@Y-}{8s93dzSmO6BlPH~&A)P*@y6WHyLFCs6!6`@R*W0vAuq^G7($B9C6P z8S>tX5)^inWFaSLc9VGASC@*J*31x!SSRvs^UzHgr;bOz+>9j4I~8k??btENuhvw zSb@2aOa=TI%bYUMYD`!08872tz&(I{M(Xe;e-gBC^B?16ZQLutPZ$cgNl4w z&I~~+&fXR$Q(KKfO@zcmCm8_k-%;ni1VM5Q4TCRzzMJi9n&i6m(Lu{S1&V@sf;npn z=L$*%6-pJVqXug`n^g6+GLf?G)Ae<0ob&>@aAk8_UF$0|9pz?+9~B3N*4}}BbY|d` zl?B3|Wtb}zO6yfAPxJXYx#rE18r^#9RDJonz91)rMOZ3#)qhktA<90KeZR1<9=QTC zzP(GBMWJI}09?Jxh0Hq_V=9Fp3&HCe=w#>GF7JJ*)FX;kTZD<{d zs~Xe4ykG|fE|_!KVRi*3o|MQew3(2h?@<<4T%wAq<4#X|bO)ck?UooCQlSW+qk7{@ znU_7lC0pOPTt3>G+TkG zJIW1q1_$h92Ck%^aD;TbRa#dcRQ~j7O zu7MO4_vW-qPd62-vjH*H4=9$veGHY0b#O<*EWU}GSooJUdE3|!ci?PnbX=rUj3?NI zY88_6$8}i*G;@Gd#h)4`VKRBWVbzCL@Lc?YE&HO*r?nqh?QU$ z+7Yi^g*lcBq2Mkgy99lRz1jKI@LbbNTj$GPbp?hY7OPqK;YL3rvJF|(}FuyhqZzye4(bvXzVL8uwjSK7v{^F zda_ut;mY!)LL1(=x zm&%|MBD8>9Rc-Mzib%JC40082cIi>k9yA!iy2O{$D{G(PYu9F1esGT+)dn(5!fCE> zM1zxFY_M*VRx}gi_n)Qxyc1TjGN;PPdGDo$Jgc%$3w~9ssy{2$gS>U9uH~L>z+J!G zby3!=3i4x;a;57{TZW=5Q-mZ=9hkvIqKjFrMi_;n&lQL)?398ALBT?%Rh?AbLD@f0 z%u~)xK{gR?oo;-p_38`R{;;wt`TFc0U+z?kg~$`3?sK>-we8Rc%zte2nsF((bVX%` zQ)bE@r8UXtQ*RX{IDdg&o(1Q!@VOJ|r|WxJkl;GXl?A^GpU?_DC<^?QvL$^*rI^Ol zV)FzU(yT8nrgxP#1rJroS84#Urbs8s)FrLN+9}v=kAKoyk#CJH1VxebctNs&x-dg` zQOKjU;jc9JTit@8>fls0c78rO!ieV*|rY{jkw5A`Gn_iD` z3RmHNRL)qt%609R2)$pX6po9eS;a2Yw3X^ASLg)6lDJmOzLL}B-C@#4-mFNQln*7y z1^bC7A+$(aQy{wCe)ArTT5@3}>j9UBr8z28s{f)a+mddSyHJ$d#r<$iYe7-LKjQ&5 zVNLxg3-ZFJf?VahRwXJFmym*{Nojhk@>vV58-AvH!?eU_kcVt276n@R4U|veM>M~B ziaw$XW|Z4ml|2^hUX_A}nP>>H{3>Nua@f>IjgVVcIFt%I)-aAd4m<7KuI~%*n^!`= zRf=O7ht?C3%5I*MzgDw-S7qjmkOj4Hf@4J4l)&O~RpTqOHeL1F?!@?hX9%OLRTqgu z)fWk!YB-O%*WQGFhC*j9@F_%}P&;(bOWbs%S zaE6uPqSUDJBCQIV9A#oq<-xb_SrIY(6QVv%?|!|_I4NRFmq)Mt8ohj2+!xn~mgq}* zT*cbEke?)%LP)|vG8J`)zMV=1hD-b@8jVUyZ|l|=0~w1cIIminnh7d>K+?xGh>A%F z5r0H4=h5SMl1q<>N_Sf>qWx)Va$KV(NA%_v!EtFVo5pB9Yk3aLe^{S=!+-hi&;HS& z-r7fcN5Hh(>CP07ySQiDUodGFA5{JuLm zp-49R*RR)(&K13d!qhQOfu2W&2N4af=D55uoF&%^i~6#mtM-!HfhqPUs=Ez zh5C>#SDW#?u4Cc*`R#$N-Xlp-d&Q6Zmc!4(gY4?)Jp2+uBL-lh$*T$b>oyjDXdZ_n zY63=^>RT6VM>6_#7lYW-fbg6%r1E3-)GvG_<6@b9An?xYO?a*w7A8HxAgRtO(%cHo z8OB~aDAg@La-zh7-r~a5ln??R6kaAp#kRc{%e0!k2o8n_R@~c~T+x2O5QTQAhZ|}u zpdj7e`f6#nZ?3XgS#9a!mY1YyELih=-N|6cIsER6m*H%&=~vAJCLW}0;kk{i} z1&MzkbnlsUeoa;N2ja8J9*bWVBPm5z9mhVYjbfh_V+8gTZE&$vX zE69(EpUv+Mv%Crw|494-Bs?XoCmG#U^O0pli5sWjqe5F1{yF1^)nC00k>Fhmg&~`- zMwcX6ga_^hg;&|GL>5~WvZcpvZKJQQe`|XRUkcAV+mmJSFpxdWuwUV)mh3W_EqNJv z_1Ho?z(qekj4fxqy&5JHFdYUT!aOyS&MR;~Y$w38&YcXas9DzbUDSg%>IMLEUb}Mc z{G|&wK6v%Ql^gGQ|NAZ|2@_Sxcbd4)PWvzF6XD%J<1iId>XUAR=Y5o{CpuF0T?ZiZ zyvIa8c_xc{+{r307E|KvBlGDbzFjqCih5iDgG=Rz#afet#KVLzXwR|(wMdV*%fVja zv5u&ttLvU{uO<$qJqH=D@8e)GcDW}_NAB)pO)HEKrC(GQIr84sRo-gPRqI`7_^QuF zojS~h6biDMC+Sol-J(F@#mP3m0uGLv_`$MZfwgbOuemp_u;6OU8lD`VdZCU`ws4UP z9#a-u0lzyQ9$FuH32j|6QIU3Ko7tat_*{2UlFjz&0VAOR&&5tMlb!`Y&+Z%W;xL}p z^6CdKU8+~A6;!Q+E>>n*X`CSpYaH5G0VxdO%zzZNEaO0I@~^`>a99?iiEB}g1p4ly+BWCHyn7G&CGij zYKlj0u6(|Dk&3&9DCG*POJ5OBF@D*y%_#6f1R8rY=FCDIhHJaGaw*Beo(!?gytTj5 zZVf4Kc|WJ(q7+9fu^Z=&RzMD!umD(l$)6DThtH7~FD=oE`NK1P#|p{Vb(V$Kz80*y zsM};ETFKDE!0{qHg15VY_TD1Z>Y?(U>0`n%<4+-j0+x=NC|c*etyStT?xksRQ4tR- zYa&!=Aaw~Bje>wjC|Oi!sm1D$(qTuK{c9Li)nt)KoM@zFpv>eBs{QTNYQ#nUA6$B{-~kjAcM!Sb_oJeUbMmPE#lAkobA5sXGY?i{=Vd#lq8PgyF*sMS+13#F5!n* zbU{!+vGp;Y8ZRK*lgDE+j|W$f9r#Eio^*Ac*FpeG_ldc7`QDq`=Q3cwt;6k8oSBt4 z5z^g-N|ts?Sz-mGwbRTsla#Duch;mcH`IQ15~2E6p1Xn~S@(?Gd8Up@sk-Zp2SQC0 z+pcy69sakl=CTi^=#p3gJ`7QY1wQt0{1JiJb(90otiz|ph-OpKDj*V>BFw+&oFnP9 z1;W`?%`SQ#)MhE107M4jW}Z*yXEhDaroDq_%FYMe<)bJ1GM95UjBp1do^+`^*OnfD zJNVdex-Vr$upN7iU)ClonR`+kJ_a6DD|;};jYhe4VXtNqNv4XRmhB0-gvucZR6fUe zvL$XO+iQI2Fh%jRbxXHe@s;T+AmX0QVU$_umdG|-b~X%@ja0;TT?hZ@yUuPj@&WeH zjYdH4>69M*aPOz|z@M@%t+?7})`j6ZBd?z;j7c}*3`XZ)+@rr8fKrb&%g0oj|^v9xbh7}|Sx9ugrJoKGZRg@Kc-x`el0joZrD@}ql? zEh~Y;xp_Pso-gi&sOR*O5H*$vvC8ueyRd3jl}<9Gp*{Lqg)bmtFYsWuLh4j8CmZbo zE732qVE}!hCn#jPflEl|^K^bA!K2>UQ3ieo^P0GWxrZMAuXFc}pOMV;lh4JSetInputStreamHandler("ImmediateInputStreamHandler"); + if (cc->Inputs().HasTag("FRAME")) + { + cc->Inputs().Tag("FRAME").Set(); + } + if (cc->Inputs().HasTag("OVERLAY")) + { + cc->Inputs().Tag("OVERLAY").Set(); + } + if (cc->Outputs().HasTag("COMBINED")) + { + cc->Outputs().Tag("COMBINED").Set(); + } + + return absl::OkStatus(); + } + + absl::Status Open(CalculatorContext *cc) override + { + cc->SetOffset(::mediapipe::TimestampDiff(0)); + return absl::OkStatus(); + } + + absl::Status Process(CalculatorContext *cc) override + { + if (!cc->Inputs().Tag("OVERLAY").IsEmpty()) + { + cv::Mat overlayed_image; + const auto &IFFrame = cc->Inputs().Tag("FRAME").Get(); + const auto &IFOverlay = cc->Inputs().Tag("OVERLAY").Get(); + cv::Mat frame = formats::MatView(&IFFrame); + cv::Mat overlay = formats::MatView(&IFOverlay); + if (overlay.cols == 0) + { + cc->Outputs() + .Tag("COMBINED") + .AddPacket(cc->Inputs().Tag("FRAME").Value()); + return absl::OkStatus(); + } + cv::cvtColor(frame, frame, cv::COLOR_BGRA2BGR); + cv::cvtColor(overlay, overlay, cv::COLOR_BGRA2BGR); + cv::addWeighted(frame, 1.0, overlay, 1.0, 0, overlayed_image); + std::unique_ptr output_image_frame = + absl::make_unique(ImageFormat::SRGB, overlayed_image.cols, + overlayed_image.rows, ImageFrame::kGlDefaultAlignmentBoundary); + overlayed_image.copyTo(formats::MatView(output_image_frame.get())); + cc->Outputs() + .Tag("COMBINED") + .Add(output_image_frame.release(), cc->InputTimestamp()); + return absl::OkStatus(); + } + if (!cc->Inputs().Tag("FRAME").IsEmpty()) + { + cc->Outputs().Tag("COMBINED").AddPacket(cc->Inputs().Tag("FRAME").Value()); + return absl::OkStatus(); + } + + return absl::OkStatus(); + } + }; + REGISTER_CALCULATOR(FrameJoinerCalculator); +} // namespace mediapipe \ No newline at end of file diff --git a/mediapipe/calculators/image/reranking_calculator.cc b/mediapipe/calculators/image/reranking_calculator.cc index b1c86eb..449d628 100644 --- a/mediapipe/calculators/image/reranking_calculator.cc +++ b/mediapipe/calculators/image/reranking_calculator.cc @@ -94,7 +94,7 @@ namespace mediapipe // Find the index with the maximum match count int max_index = -1; - int max_count = 8; + int max_count = 10; for (int i = 0; i < match_counts.size(); ++i) { if (match_counts[i] > max_count) diff --git a/mediapipe/calculators/image/overlay_calculator.cc b/mediapipe/calculators/image/tracking_calculator.cc similarity index 55% rename from mediapipe/calculators/image/overlay_calculator.cc rename to mediapipe/calculators/image/tracking_calculator.cc index 369d7af..72c86c3 100644 --- a/mediapipe/calculators/image/overlay_calculator.cc +++ b/mediapipe/calculators/image/tracking_calculator.cc @@ -5,30 +5,37 @@ #include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/framework/formats/image_frame_opencv.h" #include "mediapipe/framework/port/opencv_core_inc.h" +#include "mediapipe/framework/port/opencv_video_inc.h" #include "mediapipe/framework/port/opencv_features2d_inc.h" #include "mediapipe/framework/port/opencv_imgproc_inc.h" #include "mediapipe/framework/port/opencv_calib3d_inc.h" #include "mediapipe/framework/tool/options_util.h" +#include "mediapipe/framework/port/threadpool.h" +#include "absl/synchronization/blocking_counter.h" +#include "mediapipe/util/tracking/box_tracker.pb.h" +#include "mediapipe/util/tracking/box_tracker.h" #include "absl/status/status.h" +#include namespace mediapipe { const char kOptionsTag[] = "OPTIONS"; const int kPatchSize = 32; - const int kNumThreads = 16; - + const int kNumThreads = 2; + const double MAX_CHANGE_THRESHOLD = 70.0; // Calculator to find homography and warp a secondary image onto the primary // image. - class ImageOverlayCalculator : public CalculatorBase + class TrackingCalculator : public CalculatorBase { public: static absl::Status GetContract(CalculatorContract *cc) { - // cc->SetInputStreamHandler("ImmediateInputStreamHandler"); - // cc->SetProcessTimestampBounds(true); cc->Inputs().Tag("PRIMARY_IMAGE").Set(); cc->Inputs().Tag("SECONDARY_IMAGE").Set(); + cc->Inputs().Tag("FEEDBACK_I").Set(); cc->Outputs().Tag("OVERLAYED_IMAGE").Set(); + cc->Outputs().Tag("FEEDBACK_O").Set(); + cc->Outputs().Tag("BOXES").Set(); return absl::OkStatus(); } @@ -36,13 +43,19 @@ namespace mediapipe { cc->SetOffset(::mediapipe::TimestampDiff(0)); feature_detector_ = cv::ORB::create( - 200, 1.2, 4, kPatchSize - 1, 0, 2, cv::ORB::FAST_SCORE); + 500, 1.4, 8, kPatchSize - 1, 0, 4, cv::ORB::FAST_SCORE); matcher_ = cv::makePtr(cv::makePtr(20, 10, 2)); + pool_ = absl::make_unique("ThreadPool", kNumThreads); + pool_->StartWorkers(); return absl::OkStatus(); } absl::Status Process(CalculatorContext *cc) override { + if (!cc->Inputs().Tag("FEEDBACK_I").IsEmpty()) + { + return absl::OkStatus(); + } // Handle the first non-empty packet from SECONDARY_IMAGE if (!cc->Inputs().Tag("SECONDARY_IMAGE").IsEmpty()) { @@ -90,7 +103,6 @@ namespace mediapipe { // Process the overlay OverlayImages(cc, primary_image, secondary_image); - ABSL_LOG(INFO) << "ProcessAndCopy done"; return absl::OkStatus(); } @@ -113,29 +125,41 @@ namespace mediapipe cv::Mat secondary_view = formats::MatView(&secondary_image); // Convert to grayscale for feature detection. cv::Mat primary_gray, secondary_gray; - cv::cvtColor(primary_view, primary_gray, cv::COLOR_BGR2GRAY); - cv::cvtColor(secondary_view, secondary_gray, cv::COLOR_BGR2GRAY); // Detect ORB features. std::vector primary_keypoints, secondary_keypoints; cv::Mat primary_descriptors, secondary_descriptors; - - feature_detector_->detectAndCompute( - primary_gray, cv::noArray(), primary_keypoints, primary_descriptors); - feature_detector_->detectAndCompute( - secondary_gray, cv::noArray(), secondary_keypoints, - secondary_descriptors); + absl::BlockingCounter counter(2); + pool_->Schedule( + [this, &primary_view, &primary_gray, &primary_keypoints, &primary_descriptors, &counter] + { + cv::cvtColor(primary_view, primary_gray, cv::COLOR_BGR2GRAY); + feature_detector_->detectAndCompute( + primary_gray, cv::noArray(), primary_keypoints, primary_descriptors); + counter.DecrementCount(); + }); + pool_->Schedule( + [this, &secondary_view, &secondary_gray, &secondary_keypoints, &secondary_descriptors, &counter] + { + cv::cvtColor(secondary_view, secondary_gray, cv::COLOR_BGR2GRAY); + feature_detector_->detectAndCompute( + secondary_gray, cv::noArray(), secondary_keypoints, secondary_descriptors); + counter.DecrementCount(); + }); + counter.Wait(); // Match features using brute force matcher. - ABSL_LOG(INFO) << "Matching ORB"; std::vector> knn_matches; try { if (primary_descriptors.empty() || secondary_descriptors.empty()) { + std::unique_ptr output_image_frame = + absl::make_unique(ImageFormat::SRGB, 0, + 0, ImageFrame::kGlDefaultAlignmentBoundary); cc->Outputs() .Tag("OVERLAYED_IMAGE") - .AddPacket(cc->Inputs().Tag("PRIMARY_IMAGE").Value()); + .Add(output_image_frame.release(), cc->InputTimestamp()); return; } @@ -153,6 +177,8 @@ namespace mediapipe const float ratio_thresh = 0.7f; std::vector good_matches; + +#pragma omp parallel for num_threads(16) for (size_t i = 0; i < knn_matches.size(); i++) { if (knn_matches[i][0].distance < @@ -164,23 +190,44 @@ namespace mediapipe // Find homography matrix. std::vector primary_points, secondary_points; + +#pragma omp parallel for num_threads(16) for (const auto &match : good_matches) { primary_points.push_back(primary_keypoints[match.queryIdx].pt); secondary_points.push_back(secondary_keypoints[match.trainIdx].pt); } + cv::Mat homography; - if (primary_points.size() >= 8 && secondary_points.size() >= 8) + if (primary_points.size() >= 16 && secondary_points.size() >= 16) { + ABSL_LOG(INFO) << "Primary Points :" << primary_points.size(); + ABSL_LOG(INFO) << "Secondary Points" << secondary_points.size(); homography = cv::findHomography(secondary_points, primary_points, cv::RANSAC); + if (!cc->Inputs().Tag("FEEDBACK_I").IsEmpty() && !homography.empty()) + { + const cv::Mat &previousHomography = cc->Inputs().Tag("FEEDBACK_I").Get(); + if (!previousHomography.empty()) + { + cv::Mat diff = homography - previousHomography; + double change = cv::norm(diff); + + if (change > MAX_CHANGE_THRESHOLD) + { + // If change is too large, use a weighted average + double weight = MAX_CHANGE_THRESHOLD / change; + homography = weight * homography + (1 - weight) * previousHomography; + } + } + } } else { - LOG(WARNING) << "Insufficient keypoints for homography. Sending primary" - << " image as is."; - cc->Outputs() - .Tag("OVERLAYED_IMAGE") - .AddPacket(cc->Inputs().Tag("PRIMARY_IMAGE").Value()); + // LOG(WARNING) << "Insufficient keypoints for homography. Sending primary" + // << " image as is."; + // cc->Outputs() + // .Tag("OVERLAYED_IMAGE") + // .AddPacket(cc->Inputs().Tag("PRIMARY_IMAGE").Value()); return; } @@ -188,15 +235,57 @@ namespace mediapipe cv::Mat warped_secondary_image; if (!homography.empty()) { + cv::cvtColor(primary_view, primary_gray, cv::COLOR_BGR2GRAY); cv::warpPerspective(secondary_view, warped_secondary_image, homography, - primary_view.size(), cv::INTER_NEAREST, cv::BORDER_CONSTANT, 0); + primary_view.size(), cv::INTER_LINEAR, cv::BORDER_CONSTANT, 0); + + const int templateWidth = 400; + const int templateHeight = 200; + + // Define template corners (in template image coordinates) + std::vector templateCorners = { + cv::Point2f(0, 0), // Top-left + cv::Point2f(templateWidth, 0), // Top-right + cv::Point2f(templateWidth, templateHeight), // Bottom-right + cv::Point2f(0, templateHeight) // Bottom-left + }; + + std::vector projectedCorners; + cv::perspectiveTransform(templateCorners, projectedCorners, homography); + + // Construct TimedBoxProtoList + TimedBoxProtoList timed_box_proto_list; + auto *box_ptr = timed_box_proto_list.add_box(); + + // Assuming you want to track only one box (the template) + box_ptr->set_id(0); // Assign a unique ID + + // Add vertices to TimedBoxProto in COUNTER-CLOCKWISE order: + box_ptr->mutable_quad()->add_vertices(projectedCorners[0].x / primary_view.cols); + box_ptr->mutable_quad()->add_vertices(projectedCorners[0].y / primary_view.rows); + + box_ptr->mutable_quad()->add_vertices(projectedCorners[3].x / primary_view.cols); + box_ptr->mutable_quad()->add_vertices(projectedCorners[3].y / primary_view.rows); + + box_ptr->mutable_quad()->add_vertices(projectedCorners[2].x / primary_view.cols); + box_ptr->mutable_quad()->add_vertices(projectedCorners[2].y / primary_view.rows); + + box_ptr->mutable_quad()->add_vertices(projectedCorners[1].x / primary_view.cols); + box_ptr->mutable_quad()->add_vertices(projectedCorners[1].y / primary_view.rows); + + std::unique_ptr output_boxes(new TimedBoxProtoList()); + *output_boxes = std::move(timed_box_proto_list); + cc->Outputs().Tag("BOXES").Add(output_boxes.release(), cc->InputTimestamp()); } else { LOG(ERROR) << "Homography matrix is empty. Sending primary image as is."; + std::unique_ptr output_image_frame = + absl::make_unique(ImageFormat::SRGB, 0, + 0, ImageFrame::kGlDefaultAlignmentBoundary); cc->Outputs() .Tag("OVERLAYED_IMAGE") - .AddPacket(cc->Inputs().Tag("PRIMARY_IMAGE").Value()); + .Add(output_image_frame.release(), cc->InputTimestamp()); return; } @@ -204,14 +293,8 @@ namespace mediapipe cv::Mat overlayed_image; if (!warped_secondary_image.empty()) { - overlayed_image = primary_view.clone(); - cv::Mat mask; - cv::cvtColor(warped_secondary_image, mask, cv::COLOR_BGR2GRAY); - cv::threshold(mask, mask, 0, 0, cv::THRESH_BINARY_INV); - cv::Mat inverted_mask; - cv::bitwise_not(mask, inverted_mask); - // Use the mask to overlay the warped secondary image - warped_secondary_image.copyTo(overlayed_image, inverted_mask); + // TODO: OVerlay wrapped image on primary image + overlayed_image = warped_secondary_image.clone(); } else { @@ -222,11 +305,14 @@ namespace mediapipe // Create an ImageFrame for the overlaid image. std::unique_ptr output_image_frame = absl::make_unique(ImageFormat::SRGB, overlayed_image.cols, - overlayed_image.rows, - ImageFrame::kGlDefaultAlignmentBoundary); + overlayed_image.rows, ImageFrame::kGlDefaultAlignmentBoundary); overlayed_image.copyTo(formats::MatView(output_image_frame.get())); // Send the output packet. + auto feedback_homography = absl::make_unique(homography); + cc->Outputs() + .Tag("FEEDBACK_O") + .Add(feedback_homography.release(), cc->InputTimestamp()); cc->Outputs() .Tag("OVERLAYED_IMAGE") .Add(output_image_frame.release(), cc->InputTimestamp()); @@ -241,8 +327,9 @@ namespace mediapipe cv::Ptr feature_detector_; cv::Ptr matcher_; + std::unique_ptr pool_; }; - REGISTER_CALCULATOR(ImageOverlayCalculator); + REGISTER_CALCULATOR(TrackingCalculator); } // namespace mediapipe \ No newline at end of file diff --git a/mediapipe/examples/android/solutions/.gradle/nb-cache/subprojects.ser b/mediapipe/examples/android/solutions/.gradle/nb-cache/subprojects.ser index 77102ac835a2fbbf899f78c9c512554c7c23bd04..70b9a45112bba68c4199efefb97ed855e45952ac 100644 GIT binary patch delta 15 WcmZqRYT%l%mPNwHCvxLPe-;2I`vnvL delta 15 WcmZqRYT%l%mWB89G5(Dk{aFAkh6T6) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/custom/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/custom/BUILD index d6fb7b1..d17fc04 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/custom/BUILD +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/custom/BUILD @@ -67,7 +67,7 @@ android_binary( "inputVideoStreamName": "input_video_gpu", "outputVideoStreamName": "output_video", "flipFramesVertically": "True", - "converterNumBuffers": "8", + "converterNumBuffers": "200", }, multidex = "native", deps = [ diff --git a/mediapipe/framework/port/opencv_features2d_inc.h b/mediapipe/framework/port/opencv_features2d_inc.h index 31ab245..460f8bb 100644 --- a/mediapipe/framework/port/opencv_features2d_inc.h +++ b/mediapipe/framework/port/opencv_features2d_inc.h @@ -17,10 +17,10 @@ #include -#ifdef CV_VERSION_EPOCH // for OpenCV 2.x +#ifdef CV_VERSION_EPOCH // for OpenCV 2.x #include #else #include #endif -#endif // MEDIAPIPE_FRAMEWORK_PORT_OPENCV_FEATURES2D_INC_H_ +#endif // MEDIAPIPE_FRAMEWORK_PORT_OPENCV_FEATURES2D_INC_H_ diff --git a/mediapipe/graphs/custom/BUILD b/mediapipe/graphs/custom/BUILD index a53d72a..869ed15 100644 --- a/mediapipe/graphs/custom/BUILD +++ b/mediapipe/graphs/custom/BUILD @@ -13,14 +13,21 @@ cc_library( deps = [ "//mediapipe/gpu:gpu_buffer_to_image_frame_calculator", + "//mediapipe/calculators/image:frame_joiner_calculator", "//mediapipe/calculators/image:image_localization_calculator", "//mediapipe/calculators/core:pass_through_calculator", - "//mediapipe/calculators/core:gate_calculator", - "//mediapipe/calculators/core:side_packet_to_stream_calculator", - "//mediapipe/calculators/core:stream_to_side_packet_calculator", + "//mediapipe/calculators/core:gate_calculator", + "//mediapipe/calculators/core:side_packet_to_stream_calculator", + "//mediapipe/calculators/core:stream_to_side_packet_calculator", "//mediapipe/graphs/custom/subgraphs:tracking_subgraph", "//mediapipe/graphs/custom/subgraphs:scanning_subgraph", "//mediapipe/gpu:image_frame_to_gpu_buffer_calculator", + "//mediapipe/calculators/core:packet_cloner_calculator", + "//mediapipe/calculators/video:box_tracker_calculator", + "//mediapipe/calculators/video:flow_packager_calculator", + "//mediapipe/calculators/video:motion_analysis_calculator", + "//mediapipe/framework/stream_handler:immediate_input_stream_handler", + "//mediapipe/framework/stream_handler:sync_set_input_stream_handler", ], ) diff --git a/mediapipe/graphs/custom/feature_extraction_desktop copy.pbtxt b/mediapipe/graphs/custom/feature_extraction_desktop copy.pbtxt new file mode 100644 index 0000000..df193b7 --- /dev/null +++ b/mediapipe/graphs/custom/feature_extraction_desktop copy.pbtxt @@ -0,0 +1,178 @@ + +input_stream: "input_video_gpu" +input_stream: "query_feats" +input_stream: "match_image" +input_stream: "enable_scanning" + +output_stream: "output_video" +output_stream: "output_tensor_floats" +output_stream: "rr_index" + +max_queue_size: 200 +num_threads: 16 + +node { + calculator: "GpuBufferToImageFrameCalculator" + input_stream: "input_video_gpu" + output_stream: "input_video_cpu" +} + +node { + calculator: "PassThroughCalculator" + input_stream: "input_video_cpu" + output_stream: "scanning_frame" +} + + +node { + calculator: "StreamToSidePacketCalculator" + input_stream: "enable_scanning" + output_side_packet: "enable_scanning_side_packet" +} + +node { + calculator: "SidePacketToStreamCalculator" + input_stream: "TICK:scanning_frame" + input_side_packet: "enable_scanning_side_packet" + output_stream: "AT_TICK:enable_scanning_packet" +} + +node { + calculator: "GateCalculator" + input_stream: "scanning_frame" + input_stream: "ALLOW:enable_scanning_packet" + output_stream: "scanning_frame_enabled" +} + +node { + calculator: "ScanningSubgraph" + input_stream: "scanning_frame_enabled" + input_stream: "query_feats" + output_stream: "FLOATS:output_tensor_floats" + output_stream: "rr_index" +} + +node { + calculator: "TrackingSubgraph" + input_stream: "input_video_cpu" + input_stream: "match_image" + output_stream: "tracking_output" + output_stream: "box_output" +} + +node: { + calculator: "MotionAnalysisCalculator" + input_stream: "VIDEO:input_video_cpu" + output_stream: "CAMERA:camera_motion" + output_stream: "FLOW:region_flow" + + node_options: { + [type.googleapis.com/mediapipe.MotionAnalysisCalculatorOptions]: { + analysis_options { + analysis_policy: ANALYSIS_POLICY_CAMERA_MOBILE + flow_options { + fast_estimation_min_block_size: 100 + top_inlier_sets: 1 + frac_inlier_error_threshold: 3e-3 + downsample_mode: DOWNSAMPLE_TO_INPUT_SIZE + verification_distance: 5.0 + verify_long_feature_acceleration: true + verify_long_feature_trigger_ratio: 0.1 + tracking_options { + max_features: 500 + adaptive_extraction_levels: 2 + min_eig_val_settings { + adaptive_lowest_quality_level: 2e-4 + } + klt_tracker_implementation: KLT_OPENCV + } + } + } + } + } +} + +# Reads optical flow fields defined in +# mediapipe/framework/formats/motion/optical_flow_field.h, +# returns a VideoFrame with 2 channels (v_x and v_y), each channel is quantized +# to 0-255. +node: { + calculator: "FlowPackagerCalculator" + input_stream: "FLOW:region_flow" + input_stream: "CAMERA:camera_motion" + output_stream: "TRACKING:tracking_data" + + node_options: { + [type.googleapis.com/mediapipe.FlowPackagerCalculatorOptions]: { + flow_packager_options: { + binary_tracking_data_support: false + } + } + } +} + +# Tracks box positions over time. +node: { + calculator: "BoxTrackerCalculator" + input_stream: "TRACKING:tracking_data" + input_stream: "TRACK_TIME:input_video_cpu" + input_stream: "VIDEO:input_video_cpu" + input_stream: "START_POS:box_output" + output_stream: "VIZ:boxes" + + input_stream_handler { + input_stream_handler: "SyncSetInputStreamHandler" + options { + [mediapipe.SyncSetInputStreamHandlerOptions.ext] { + sync_set { + tag_index: "TRACKING" + tag_index: "TRACK_TIME" + } + sync_set { + tag_index: "START_POS" + } + } + } + } + + node_options: { + [type.googleapis.com/mediapipe.BoxTrackerCalculatorOptions]: { + tracker_options: { + track_step_options { + track_object_and_camera: true + tracking_degrees: TRACKING_DEGREE_OBJECT_PERSPECTIVE + inlier_spring_force: 0.0 + static_motion_temporal_ratio: 3e-2 + } + } + visualize_tracking_data: false + streaming_track_data_cache_size: 100 + } + } +} + + +node { + calculator: "PacketClonerCalculator" + input_stream: "TICK:input_video_cpu" + input_stream: "tracking_output" + output_stream: "cloned_tracking_output" + node_options: { + [type.googleapis.com/mediapipe.PacketClonerCalculatorOptions] { + output_packets_only_when_all_inputs_received: true + } + } +} + +node { + calculator: "FrameJoinerCalculator" + input_stream: "FRAME:input_video_cpu" + input_stream: "OVERLAY:cloned_tracking_output" + output_stream: "COMBINED:overlayed_frame" +} + +node: { + calculator: "ImageFrameToGpuBufferCalculator" + input_stream: "overlayed_frame" + output_stream: "output_video" +} diff --git a/mediapipe/graphs/custom/feature_extraction_desktop.pbtxt b/mediapipe/graphs/custom/feature_extraction_desktop.pbtxt index ff7cb41..152d644 100644 --- a/mediapipe/graphs/custom/feature_extraction_desktop.pbtxt +++ b/mediapipe/graphs/custom/feature_extraction_desktop.pbtxt @@ -8,8 +8,8 @@ output_stream: "output_video" output_stream: "output_tensor_floats" output_stream: "rr_index" -max_queue_size: 10 -num_threads: 4 +max_queue_size: 200 +num_threads: 16 node { calculator: "GpuBufferToImageFrameCalculator" @@ -56,11 +56,105 @@ node { calculator: "TrackingSubgraph" input_stream: "input_video_cpu" input_stream: "match_image" - output_stream: "overlayed_frame" + output_stream: "tracking_output" + output_stream: "box_output" } +node: { + calculator: "MotionAnalysisCalculator" + input_stream: "VIDEO:input_video_cpu" + output_stream: "CAMERA:camera_motion" + output_stream: "FLOW:region_flow" + + node_options: { + [type.googleapis.com/mediapipe.MotionAnalysisCalculatorOptions]: { + analysis_options { + analysis_policy: ANALYSIS_POLICY_CAMERA_MOBILE + flow_options { + fast_estimation_min_block_size: 100 + top_inlier_sets: 1 + frac_inlier_error_threshold: 3e-3 + downsample_mode: DOWNSAMPLE_TO_INPUT_SIZE + verification_distance: 5.0 + verify_long_feature_acceleration: true + verify_long_feature_trigger_ratio: 0.1 + tracking_options { + max_features: 500 + adaptive_extraction_levels: 2 + min_eig_val_settings { + adaptive_lowest_quality_level: 2e-4 + } + klt_tracker_implementation: KLT_OPENCV + } + } + } + } + } +} + +# Reads optical flow fields defined in +# mediapipe/framework/formats/motion/optical_flow_field.h, +# returns a VideoFrame with 2 channels (v_x and v_y), each channel is quantized +# to 0-255. +node: { + calculator: "FlowPackagerCalculator" + input_stream: "FLOW:region_flow" + input_stream: "CAMERA:camera_motion" + output_stream: "TRACKING:tracking_data" + + node_options: { + [type.googleapis.com/mediapipe.FlowPackagerCalculatorOptions]: { + flow_packager_options: { + binary_tracking_data_support: false + } + } + } +} + +# Tracks box positions over time. +node: { + calculator: "BoxTrackerCalculator" + input_stream: "TRACKING:tracking_data" + input_stream: "TRACK_TIME:input_video_cpu" + input_stream: "VIDEO:input_video_cpu" + input_stream: "START_POS:box_output" + output_stream: "VIZ:boxes" + + input_stream_handler { + input_stream_handler: "SyncSetInputStreamHandler" + options { + [mediapipe.SyncSetInputStreamHandlerOptions.ext] { + sync_set { + tag_index: "TRACKING" + tag_index: "TRACK_TIME" + tag_index: "VIDEO" + } + sync_set { + tag_index: "START_POS" + } + } + } + } + + node_options: { + [type.googleapis.com/mediapipe.BoxTrackerCalculatorOptions]: { + tracker_options: { + track_step_options { + track_object_and_camera: true + tracking_degrees: TRACKING_DEGREE_OBJECT_PERSPECTIVE + inlier_spring_force: 0.0 + static_motion_temporal_ratio: 3e-2 + } + } + visualize_tracking_data: false + streaming_track_data_cache_size: 100 + } + } +} + + node: { calculator: "ImageFrameToGpuBufferCalculator" - input_stream: "overlayed_frame" + input_stream: "boxes" output_stream: "output_video" } diff --git a/mediapipe/graphs/custom/subgraphs/BUILD b/mediapipe/graphs/custom/subgraphs/BUILD index 4fb90b1..8e0fc61 100644 --- a/mediapipe/graphs/custom/subgraphs/BUILD +++ b/mediapipe/graphs/custom/subgraphs/BUILD @@ -14,10 +14,11 @@ mediapipe_simple_subgraph( deps = [ "//mediapipe/calculators/image:image_transformation_calculator", "//mediapipe/calculators/core:flow_limiter_calculator", - "//mediapipe/calculators/core:stream_to_side_packet_calculator", - "//mediapipe/calculators/core:side_packet_to_stream_calculator", - "//mediapipe/calculators/core:packet_cloner_calculator", - "//mediapipe/calculators/image:overlay_calculator_custom", + "//mediapipe/calculators/core:stream_to_side_packet_calculator", + "//mediapipe/calculators/core:side_packet_to_stream_calculator", + "//mediapipe/calculators/core:packet_cloner_calculator", + "//mediapipe/calculators/core:previous_loopback_calculator", + "//mediapipe/calculators/image:tracking_calculator", ], ) @@ -27,7 +28,6 @@ mediapipe_simple_subgraph( register_as = "ScanningSubgraph", deps = [ "//mediapipe/calculators/core:flow_limiter_calculator", - "//mediapipe/calculators/image:image_localization_calculator", "//mediapipe/calculators/image:image_transformation_calculator", "//mediapipe/calculators/tflite:tflite_converter_calculator", "//mediapipe/calculators/tflite:tflite_inference_calculator", diff --git a/mediapipe/graphs/custom/subgraphs/scanning_subgraph.pbtxt b/mediapipe/graphs/custom/subgraphs/scanning_subgraph.pbtxt index 4e93144..babb316 100644 --- a/mediapipe/graphs/custom/subgraphs/scanning_subgraph.pbtxt +++ b/mediapipe/graphs/custom/subgraphs/scanning_subgraph.pbtxt @@ -6,16 +6,10 @@ input_stream: "query_feats" output_stream: "FLOATS:output_tensor_floats" output_stream: "rr_index" -node { - calculator: "PictureLocalizationCalculator" - input_stream: "IMAGE:input_frame" - output_stream: "IMAGE:localized_frame" -} - node { calculator: "FlowLimiterCalculator" - input_stream: "localized_frame" - input_stream: "FINISHED:feats" + input_stream: "input_frame" + input_stream: "FINISHED:output_tensor_floats" input_stream_info: { tag_index: "FINISHED" back_edge: true @@ -28,11 +22,9 @@ node { output_stream: "throttled_input_video_cpu" } - - node { calculator: "ImageTransformationCalculator" - input_stream: "IMAGE:input_frame" + input_stream: "IMAGE:throttled_input_video_cpu" output_stream: "IMAGE:resized_frame" node_options: { [type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] { @@ -67,7 +59,7 @@ node { node { calculator: "ImageTransformationCalculator" - input_stream: "IMAGE:input_frame" + input_stream: "IMAGE:throttled_input_video_cpu" output_stream: "IMAGE:transformed_input_frame" node_options: { [type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] { diff --git a/mediapipe/graphs/custom/subgraphs/tracking_subgraph.pbtxt b/mediapipe/graphs/custom/subgraphs/tracking_subgraph.pbtxt index 02e1661..0cdb8e6 100644 --- a/mediapipe/graphs/custom/subgraphs/tracking_subgraph.pbtxt +++ b/mediapipe/graphs/custom/subgraphs/tracking_subgraph.pbtxt @@ -4,6 +4,7 @@ type: "TrackingSubgraph" input_stream: "input_frame" input_stream: "match_image" output_stream: "overlayed_frame" +output_stream: "tracked_boxes" node { calculator: "FlowLimiterCalculator" @@ -43,9 +44,23 @@ node { output_stream: "cloned_match_image_packet" } +node { + calculator: "PreviousLoopbackCalculator" + input_stream: "MAIN:cloned_input_frame" + input_stream: "LOOP:homography" + input_stream_info: { + tag_index: "LOOP" + back_edge: true + } + output_stream: "PREV_LOOP:prev_homography" +} + node: { - calculator: "ImageOverlayCalculator" + calculator: "TrackingCalculator" input_stream: "PRIMARY_IMAGE:cloned_input_frame" input_stream : "SECONDARY_IMAGE:cloned_match_image_packet" + input_stream: "FEEDBACK_I:prev_homography" output_stream: "OVERLAYED_IMAGE:overlayed_frame" + output_stream: "FEEDBACK_O:homography" + output_stream: "BOXES:tracked_boxes" } \ No newline at end of file