From 3334995ec9386e47a9a13be3cf21bbac7893f3a1 Mon Sep 17 00:00:00 2001 From: jim Date: Sun, 29 Sep 2019 00:11:18 -0400 Subject: [PATCH] Initial commit. --- .clang-format | 83 + .devcontainer/Dockerfile | 138 + .devcontainer/devcontainer.json | 46 + .dockerignore | 9 + .github/workflows/devcontainer.yml | 68 + .github/workflows/docker-image.yml | 44 + .gitignore | 11 + .gitmodules | 21 + .vscode/launch.json | 32 + .vscode/settings.json | 10 + CMakeLists.txt | 147 + README.md | 155 ++ docker/Dockerfile | 24 + docker/alice.sh | 19 + docker/bob.sh | 19 + docker/build.sh | 5 + docker/run.sh | 14 + extern/ABY | 1 + extern/Catch2 | 1 + extern/emp-ot | 1 + extern/emp-sh2pc | 1 + extern/emp-tool | 1 + extern/opencv | 1 + extern/opencv_contrib | 1 + plotter_deps.sh | 3 + plottertest/3dbarplot.sh | 9 + plottertest/3dbarplot_customized.sh | 16 + plottertest/3dbarplottest.csv | 37 + plottertest/3dsurfplot.sh | 9 + plottertest/3dsurfplot_customized.sh | 16 + plottertest/3dsurfplottest.csv | 16 + plottertest/barboxplot.sh | 9 + plottertest/barboxplot_customized.sh | 16 + plottertest/barboxplottest.csv | 18 + plottertest/barplot.sh | 9 + plottertest/barplottest.csv | 21 + plottertest/boxplot.sh | 9 + plottertest/boxplottest.csv | 18 + plottertest/grouped_bar.csv | 6 + plottertest/grouped_bar.sh | 9 + plottertest/xyplot.sh | 9 + plottertest/xyplot_customized.sh | 16 + plottertest/xyplottest.csv | 13 + scripts/aby_float_benchmark_run.sh | 8 + scripts/clear_fixed_benchmark_plot.sh | 44 + scripts/clear_fixed_benchmark_run.sh | 8 + scripts/emp_float_benchmark_dataobl_run.sh | 10 + scripts/emp_float_benchmark_plot.sh | 40 + scripts/emp_float_benchmark_run.sh | 13 + scripts/emp_float_benchmark_run_latency.sh | 12 + scripts/emp_vs_aby_plot.sh | 51 + scripts/insetplotter.py | 780 ++++++ scripts/loopleak_vs_dataobl_plot.sh | 43 + scripts/mult_add_fixed_float_time_plot.sh | 69 + scripts/mult_add_fixed_float_time_run.sh | 10 + scripts/netio_plot.sh | 36 + scripts/network_setup.sh | 10 + scripts/network_teardown.sh | 6 + scripts/num_arith_ops_plot.sh | 39 + scripts/plotter-do-vs-sil.py | 716 +++++ scripts/plotter-emp-vs-aby.py | 751 +++++ scripts/plotter-mult-add.py | 739 +++++ scripts/plotter-netio.py | 748 +++++ scripts/plotter.py | 696 +++++ src/CMakeLists.txt | 2 + src/aby-float-server/CMakeLists.txt | 11 + .../gaussnewtonlocalization.cpp | 622 +++++ .../gaussnewtonlocalization.h | 24 + src/aby-float-server/invert.cpp | 180 ++ src/aby-float-server/invert.h | 10 + src/aby-float-server/lmlocalization.cpp | 975 +++++++ src/aby-float-server/lmlocalization.h | 26 + src/aby-float-server/matmult.cpp | 76 + src/aby-float-server/matmult.h | 19 + src/aby-float-server/projectpoints.cpp | 63 + src/aby-float-server/projectpoints.h | 19 + src/aby-float-server/rodrigues.cpp | 146 + src/aby-float-server/rodrigues.h | 20 + src/aby-float-server/svd.cpp | 2405 +++++++++++++++++ src/aby-float-server/svd.h | 45 + src/aby-float-server/trigfuncs.cpp | 98 + src/aby-float-server/trigfuncs.h | 15 + src/aby-float-server/twonormsq.cpp | 38 + src/aby-float-server/twonormsq.h | 7 + src/aby-float-server/util.cpp | 81 + src/aby-float-server/util.h | 14 + src/common/fixed_point.h | 1268 +++++++++ src/common/jlog.h | 265 ++ src/common/printutil.h | 56 + src/common/privacyconf.h | 56 + src/emp-float-server/CMakeLists.txt | 22 + src/emp-float-server/fixed_point_emp.h | 798 ++++++ .../gaussnewtonlocalization.cpp | 226 ++ .../gaussnewtonlocalization.h | 15 + src/emp-float-server/invert.cpp | 84 + src/emp-float-server/invert.h | 7 + src/emp-float-server/lmlocalization.cpp | 459 ++++ src/emp-float-server/lmlocalization.h | 21 + src/emp-float-server/matmult.cpp | 55 + src/emp-float-server/matmult.h | 15 + src/emp-float-server/projectpoints.cpp | 72 + src/emp-float-server/projectpoints.h | 15 + src/emp-float-server/rodrigues.cpp | 71 + src/emp-float-server/rodrigues.h | 9 + src/emp-float-server/server-lm.cpp | 76 + src/emp-float-server/svd.cpp | 611 +++++ src/emp-float-server/svd.h | 11 + src/emp-float-server/trigfuncs.cpp | 22 + src/emp-float-server/trigfuncs.h | 15 + src/emp-float-server/twonormsq.cpp | 29 + src/emp-float-server/twonormsq.h | 7 + src/emp-float-server/util.cpp | 63 + src/emp-float-server/util.h | 15 + test/CMakeLists.txt | 5 + test/aby-float/CMakeLists.txt | 26 + test/aby-float/eth3d_bench.cpp | 169 ++ test/aby-float/localize_wrapper.hpp | 163 ++ test/aby-float/test.cpp | 985 +++++++ test/cleartext-fixed/CMakeLists.txt | 42 + test/cleartext-fixed/benchmark.cpp | 568 ++++ test/cleartext-fixed/float_vs_fixed_speed.cpp | 223 ++ test/cleartext-fixed/test.cpp | 387 +++ test/common-test/CMakeLists.txt | 18 + test/common-test/april_snail_features.hpp | 24 + .../cleartext-ref/gaussnewtonlocalization.hpp | 273 ++ test/common-test/cleartext-ref/invert.hpp | 194 ++ .../cleartext-ref/lmlocalization.hpp | 348 +++ test/common-test/cleartext-ref/matmult.hpp | 48 + .../cleartext-ref/projectpoints.hpp | 48 + test/common-test/cleartext-ref/rodrigues.hpp | 33 + test/common-test/cleartext-ref/svd.hpp | 363 +++ test/common-test/cleartext-ref/trigfuncs.hpp | 17 + test/common-test/cleartext-ref/twonormsq.hpp | 12 + test/common-test/eth3d_features.hpp | 280 ++ test/common-test/eth3d_tester.cpp | 50 + test/common-test/hoff_features.hpp | 26 + test/common-test/jlog.h | 279 ++ test/common-test/kitti_tester.cpp | 318 +++ test/common-test/test_harness.hpp | 247 ++ test/common-test/test_params.h | 8 + test/emp-fixed/CMakeLists.txt | 21 + test/emp-fixed/README.md | 3 + test/emp-fixed/test.cpp | 105 + test/emp-float/CMakeLists.txt | 37 + test/emp-float/client.cpp | 172 ++ test/emp-float/eth3d_bench.cpp | 213 ++ test/emp-float/localize_wrapper.hpp | 66 + test/emp-float/test.cpp | 359 +++ test/emp-float/test_utils.hpp | 474 ++++ 149 files changed, 21803 insertions(+) create mode 100644 .clang-format create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/devcontainer.json create mode 100644 .dockerignore create mode 100644 .github/workflows/devcontainer.yml create mode 100644 .github/workflows/docker-image.yml create mode 100644 .gitignore create mode 100644 .gitmodules create mode 100644 .vscode/launch.json create mode 100644 .vscode/settings.json create mode 100644 CMakeLists.txt create mode 100644 README.md create mode 100644 docker/Dockerfile create mode 100755 docker/alice.sh create mode 100755 docker/bob.sh create mode 100755 docker/build.sh create mode 100755 docker/run.sh create mode 160000 extern/ABY create mode 160000 extern/Catch2 create mode 160000 extern/emp-ot create mode 160000 extern/emp-sh2pc create mode 160000 extern/emp-tool create mode 160000 extern/opencv create mode 160000 extern/opencv_contrib create mode 100755 plotter_deps.sh create mode 100755 plottertest/3dbarplot.sh create mode 100755 plottertest/3dbarplot_customized.sh create mode 100644 plottertest/3dbarplottest.csv create mode 100755 plottertest/3dsurfplot.sh create mode 100755 plottertest/3dsurfplot_customized.sh create mode 100644 plottertest/3dsurfplottest.csv create mode 100755 plottertest/barboxplot.sh create mode 100755 plottertest/barboxplot_customized.sh create mode 100644 plottertest/barboxplottest.csv create mode 100755 plottertest/barplot.sh create mode 100644 plottertest/barplottest.csv create mode 100755 plottertest/boxplot.sh create mode 100644 plottertest/boxplottest.csv create mode 100644 plottertest/grouped_bar.csv create mode 100755 plottertest/grouped_bar.sh create mode 100755 plottertest/xyplot.sh create mode 100755 plottertest/xyplot_customized.sh create mode 100644 plottertest/xyplottest.csv create mode 100755 scripts/aby_float_benchmark_run.sh create mode 100755 scripts/clear_fixed_benchmark_plot.sh create mode 100755 scripts/clear_fixed_benchmark_run.sh create mode 100755 scripts/emp_float_benchmark_dataobl_run.sh create mode 100755 scripts/emp_float_benchmark_plot.sh create mode 100755 scripts/emp_float_benchmark_run.sh create mode 100755 scripts/emp_float_benchmark_run_latency.sh create mode 100755 scripts/emp_vs_aby_plot.sh create mode 100644 scripts/insetplotter.py create mode 100755 scripts/loopleak_vs_dataobl_plot.sh create mode 100755 scripts/mult_add_fixed_float_time_plot.sh create mode 100755 scripts/mult_add_fixed_float_time_run.sh create mode 100755 scripts/netio_plot.sh create mode 100644 scripts/network_setup.sh create mode 100644 scripts/network_teardown.sh create mode 100755 scripts/num_arith_ops_plot.sh create mode 100644 scripts/plotter-do-vs-sil.py create mode 100644 scripts/plotter-emp-vs-aby.py create mode 100644 scripts/plotter-mult-add.py create mode 100644 scripts/plotter-netio.py create mode 100644 scripts/plotter.py create mode 100644 src/CMakeLists.txt create mode 100644 src/aby-float-server/CMakeLists.txt create mode 100644 src/aby-float-server/gaussnewtonlocalization.cpp create mode 100644 src/aby-float-server/gaussnewtonlocalization.h create mode 100644 src/aby-float-server/invert.cpp create mode 100644 src/aby-float-server/invert.h create mode 100644 src/aby-float-server/lmlocalization.cpp create mode 100644 src/aby-float-server/lmlocalization.h create mode 100644 src/aby-float-server/matmult.cpp create mode 100644 src/aby-float-server/matmult.h create mode 100644 src/aby-float-server/projectpoints.cpp create mode 100644 src/aby-float-server/projectpoints.h create mode 100644 src/aby-float-server/rodrigues.cpp create mode 100644 src/aby-float-server/rodrigues.h create mode 100644 src/aby-float-server/svd.cpp create mode 100644 src/aby-float-server/svd.h create mode 100644 src/aby-float-server/trigfuncs.cpp create mode 100644 src/aby-float-server/trigfuncs.h create mode 100644 src/aby-float-server/twonormsq.cpp create mode 100644 src/aby-float-server/twonormsq.h create mode 100644 src/aby-float-server/util.cpp create mode 100644 src/aby-float-server/util.h create mode 100644 src/common/fixed_point.h create mode 100644 src/common/jlog.h create mode 100644 src/common/printutil.h create mode 100644 src/common/privacyconf.h create mode 100644 src/emp-float-server/CMakeLists.txt create mode 100644 src/emp-float-server/fixed_point_emp.h create mode 100644 src/emp-float-server/gaussnewtonlocalization.cpp create mode 100644 src/emp-float-server/gaussnewtonlocalization.h create mode 100644 src/emp-float-server/invert.cpp create mode 100644 src/emp-float-server/invert.h create mode 100644 src/emp-float-server/lmlocalization.cpp create mode 100644 src/emp-float-server/lmlocalization.h create mode 100644 src/emp-float-server/matmult.cpp create mode 100644 src/emp-float-server/matmult.h create mode 100644 src/emp-float-server/projectpoints.cpp create mode 100644 src/emp-float-server/projectpoints.h create mode 100644 src/emp-float-server/rodrigues.cpp create mode 100644 src/emp-float-server/rodrigues.h create mode 100644 src/emp-float-server/server-lm.cpp create mode 100644 src/emp-float-server/svd.cpp create mode 100644 src/emp-float-server/svd.h create mode 100644 src/emp-float-server/trigfuncs.cpp create mode 100644 src/emp-float-server/trigfuncs.h create mode 100644 src/emp-float-server/twonormsq.cpp create mode 100644 src/emp-float-server/twonormsq.h create mode 100644 src/emp-float-server/util.cpp create mode 100644 src/emp-float-server/util.h create mode 100644 test/CMakeLists.txt create mode 100644 test/aby-float/CMakeLists.txt create mode 100644 test/aby-float/eth3d_bench.cpp create mode 100644 test/aby-float/localize_wrapper.hpp create mode 100644 test/aby-float/test.cpp create mode 100644 test/cleartext-fixed/CMakeLists.txt create mode 100644 test/cleartext-fixed/benchmark.cpp create mode 100644 test/cleartext-fixed/float_vs_fixed_speed.cpp create mode 100644 test/cleartext-fixed/test.cpp create mode 100644 test/common-test/CMakeLists.txt create mode 100644 test/common-test/april_snail_features.hpp create mode 100644 test/common-test/cleartext-ref/gaussnewtonlocalization.hpp create mode 100644 test/common-test/cleartext-ref/invert.hpp create mode 100644 test/common-test/cleartext-ref/lmlocalization.hpp create mode 100644 test/common-test/cleartext-ref/matmult.hpp create mode 100644 test/common-test/cleartext-ref/projectpoints.hpp create mode 100644 test/common-test/cleartext-ref/rodrigues.hpp create mode 100644 test/common-test/cleartext-ref/svd.hpp create mode 100644 test/common-test/cleartext-ref/trigfuncs.hpp create mode 100644 test/common-test/cleartext-ref/twonormsq.hpp create mode 100644 test/common-test/eth3d_features.hpp create mode 100644 test/common-test/eth3d_tester.cpp create mode 100644 test/common-test/hoff_features.hpp create mode 100644 test/common-test/jlog.h create mode 100644 test/common-test/kitti_tester.cpp create mode 100644 test/common-test/test_harness.hpp create mode 100644 test/common-test/test_params.h create mode 100644 test/emp-fixed/CMakeLists.txt create mode 100644 test/emp-fixed/README.md create mode 100644 test/emp-fixed/test.cpp create mode 100644 test/emp-float/CMakeLists.txt create mode 100644 test/emp-float/client.cpp create mode 100644 test/emp-float/eth3d_bench.cpp create mode 100644 test/emp-float/localize_wrapper.hpp create mode 100644 test/emp-float/test.cpp create mode 100644 test/emp-float/test_utils.hpp diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..a374015 --- /dev/null +++ b/.clang-format @@ -0,0 +1,83 @@ +# Google C/C++ Code Style settings +# https://clang.llvm.org/docs/ClangFormatStyleOptions.html +# Author: Kehan Xue, kehan.xue (at) gmail.com + +Language: Cpp +BasedOnStyle: Google +AccessModifierOffset: -1 +AlignAfterOpenBracket: Align +AlignConsecutiveAssignments: None +AlignOperands: Align +AllowAllArgumentsOnNextLine: true +AllowAllConstructorInitializersOnNextLine: true +AllowAllParametersOfDeclarationOnNextLine: false +AllowShortBlocksOnASingleLine: Empty +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: Inline +AllowShortIfStatementsOnASingleLine: Never # To avoid conflict, set this "Never" and each "if statement" should include brace when coding +AllowShortLambdasOnASingleLine: Inline +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterReturnType: None +AlwaysBreakTemplateDeclarations: Yes +BinPackArguments: true +BreakBeforeBraces: Custom +BraceWrapping: + AfterCaseLabel: false + AfterClass: false + AfterStruct: false + AfterControlStatement: Never + AfterEnum: false + AfterFunction: false + AfterNamespace: false + AfterUnion: false + AfterExternBlock: false + BeforeCatch: false + BeforeElse: false + BeforeLambdaBody: false + IndentBraces: false + SplitEmptyFunction: false + SplitEmptyRecord: false + SplitEmptyNamespace: false +BreakBeforeBinaryOperators: None +BreakBeforeTernaryOperators: true +BreakConstructorInitializers: BeforeColon +BreakInheritanceList: BeforeColon +ColumnLimit: 80 +CompactNamespaces: false +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: true +DerivePointerAlignment: false # Make sure the * or & align on the left +EmptyLineBeforeAccessModifier: LogicalBlock +FixNamespaceComments: true +IncludeBlocks: Preserve +IndentCaseLabels: true +IndentPPDirectives: None +IndentWidth: 2 +KeepEmptyLinesAtTheStartOfBlocks: true +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: None +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: true +PointerAlignment: Left +ReflowComments: false +# SeparateDefinitionBlocks: Always # Only support since clang-format 14 +SpaceAfterCStyleCast: false +SpaceAfterLogicalNot: false +SpaceAfterTemplateKeyword: true +SpaceBeforeAssignmentOperators: true +SpaceBeforeCpp11BracedList: false +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true +SpaceBeforeParens: ControlStatements +SpaceBeforeRangeBasedForLoopColon: true +SpaceBeforeSquareBrackets: false +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 2 +SpacesInAngles: false +SpacesInCStyleCastParentheses: false +SpacesInContainerLiterals: false +SpacesInParentheses: false +SpacesInSquareBrackets: false +Standard: c++17 +TabWidth: 4 +UseTab: Never \ No newline at end of file diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000..e2b5eb6 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,138 @@ +# [Choice] bionic (18.04), focal (20.04) +ARG VARIANT="jammy" +FROM ubuntu:${VARIANT} + +# Restate the variant to use it later on in the llvm and cmake installations +ARG VARIANT + +# Install necessary packages available from standard repos +RUN apt-get update -qq && export DEBIAN_FRONTEND=noninteractive && \ + apt-get install -y --no-install-recommends \ + software-properties-common wget apt-utils file zip \ + openssh-client gpg-agent socat rsync \ + make ninja-build git \ + python3 python3-pip + +# Install conan +RUN python3 -m pip install --upgrade pip setuptools && \ + python3 -m pip install conan && \ + conan --version + +# By default, anything you run in Docker is done as superuser. +# Conan runs some install commands as superuser, and will prepend `sudo` to +# these commands, unless `CONAN_SYSREQUIRES_SUDO=0` is in your env variables. +ENV CONAN_SYSREQUIRES_SUDO 0 +# Some packages request that Conan use the system package manager to install +# a few dependencies. This flag allows Conan to proceed with these installations; +# leaving this flag undefined can cause some installation failures. +ENV CONAN_SYSREQUIRES_MODE enabled + +# Store Conan files at this path so host chose to map as volume for semantic +# completion. Must be same path on host and container for compile_commands.json +# to match. +ENV CONAN_USER_HOME=/etc/conan +RUN mkdir ${CONAN_USER_HOME} && chmod a+rw ${CONAN_USER_HOME} + +# User-settable versions: +# This Dockerfile should support gcc-[7, 8, 9, 10, 11] and clang-[10, 11, 12, 13] +# Earlier versions of clang will require significant modifications to the IWYU section +ARG GCC_VER="11" +# Add gcc-${GCC_VER} +RUN add-apt-repository -y ppa:ubuntu-toolchain-r/test && \ + apt-get update -qq && export DEBIAN_FRONTEND=noninteractive && \ + apt-get install -y --no-install-recommends \ + gcc-${GCC_VER} g++-${GCC_VER} gdb + +# Set gcc-${GCC_VER} as default gcc +RUN update-alternatives --install /usr/bin/gcc gcc $(which gcc-${GCC_VER}) 100 +RUN update-alternatives --install /usr/bin/g++ g++ $(which g++-${GCC_VER}) 100 + +ARG LLVM_VER="14" +# Add clang-${LLVM_VER} +ARG LLVM_URL="http://apt.llvm.org/${VARIANT}/" +ARG LLVM_PKG="llvm-toolchain-${VARIANT}-${LLVM_VER}" +RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - 2>/dev/null && \ + add-apt-repository -y "deb ${LLVM_URL} ${LLVM_PKG} main" && \ + apt-get update -qq && export DEBIAN_FRONTEND=noninteractive && \ + apt-get install -y --no-install-recommends \ + clang-${LLVM_VER} lldb-${LLVM_VER} lld-${LLVM_VER} clangd-${LLVM_VER} \ + llvm-${LLVM_VER}-dev libclang-${LLVM_VER}-dev clang-tidy-${LLVM_VER} clang-format-${LLVM_VER} + +# Set the default clang-tidy, so CMake can find it +RUN update-alternatives --install /usr/bin/clang-tidy clang-tidy $(which clang-tidy-${LLVM_VER}) 1 + +# Set the default clang-format +RUN update-alternatives --install /usr/bin/clang-format clang-format $(which clang-format-${LLVM_VER}) 1 + +# Set clang-${LLVM_VER} as default clang +RUN update-alternatives --install /usr/bin/clang clang $(which clang-${LLVM_VER}) 100 +RUN update-alternatives --install /usr/bin/clang++ clang++ $(which clang++-${LLVM_VER}) 100 + +# Add current cmake/ccmake, from Kitware +ARG CMAKE_URL="https://apt.kitware.com/ubuntu/" +ARG CMAKE_PKG=${VARIANT} +RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null \ + | gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null && \ + apt-add-repository -y "deb ${CMAKE_URL} ${CMAKE_PKG} main" && \ + apt-get update -qq && export DEBIAN_FRONTEND=noninteractive && \ + apt-get install -y --no-install-recommends cmake cmake-curses-gui + +# Install editors +RUN apt-get update -qq && export DEBIAN_FRONTEND=noninteractive && \ + apt-get install -y --no-install-recommends \ + neovim emacs nano + +# Install optional dependecies +RUN apt-get update -qq && export DEBIAN_FRONTEND=noninteractive && \ + apt-get install -y --no-install-recommends \ + doxygen graphviz ccache cppcheck +ENV CCACHE_DIR=/home/ccache +RUN mkdir ${CCACHE_DIR} && chmod a+rw ${CCACHE_DIR} + +# Install include-what-you-use +ENV IWYU /home/iwyu +ENV IWYU_BUILD ${IWYU}/build +ENV IWYU_SRC ${IWYU}/include-what-you-use +RUN mkdir -p ${IWYU_BUILD} && \ + git clone --branch clang_${LLVM_VER} \ + https://github.com/include-what-you-use/include-what-you-use.git \ + ${IWYU_SRC} +RUN CC=clang-${LLVM_VER} CXX=clang++-${LLVM_VER} cmake -S ${IWYU_SRC} \ + -B ${IWYU_BUILD} \ + -G "Unix Makefiles" -DCMAKE_PREFIX_PATH=/usr/lib/llvm-${LLVM_VER} && \ + cmake --build ${IWYU_BUILD} -j && \ + cmake --install ${IWYU_BUILD} + +# Per https://github.com/include-what-you-use/include-what-you-use#how-to-install: +# `You need to copy the Clang include directory to the expected location before +# running (similarly, use include-what-you-use -print-resource-dir to learn +# exactly where IWYU wants the headers).` +RUN mkdir -p $(include-what-you-use -print-resource-dir 2>/dev/null) +RUN ln -s $(readlink -f /usr/lib/clang/${LLVM_VER}/include) \ + $(include-what-you-use -print-resource-dir 2>/dev/null)/include + +## Cleanup cached apt data we don't need anymore +RUN apt-get autoremove -y && apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# Allow the user to set compiler defaults +ARG USE_CLANG +# if --build-arg USE_CLANG=1, set CC to 'clang' or set to null otherwise. +ENV CC=${USE_CLANG:+"clang"} +ENV CXX=${USE_CLANG:+"clang++"} +# if CC is null, set it to 'gcc' (or leave as is otherwise). +ENV CC=${CC:-"gcc"} +ENV CXX=${CXX:-"g++"} + +ARG USERNAME=user +ARG USER_UID=1000 +ARG USER_GID=$USER_UID +RUN groupadd --gid $USER_GID $USERNAME \ + && useradd --uid $USER_UID --gid $USER_GID -m $USERNAME \ + && apt-get update \ + && apt-get install -y sudo \ + && echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \ + && chmod 0440 /etc/sudoers.d/$USERNAME +USER $USERNAME + +CMD ["/bin/bash"] diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..0adf89e --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,46 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: +// https://github.com/microsoft/vscode-dev-containers/tree/v0.205.2/containers/cpp +{ + "name": "C++", + "build": { + "dockerfile": "Dockerfile" + // Update 'VARIANT' to pick an Ubuntu OS version. Options: [bionic, focal]. Default: focal + // Update 'GCC_VER' to pick a gcc and g++ version. Options: [7, 8, 9, 10, 11]. Default: 11 + // Update 'LLVM_VER' to pick clang version. Options: [10, 11, 12, 13]. Default: 13 + // Update 'USE_CLANG' to set clang as the default C and C++ compiler. Options: [1, null]. Default null + // "args": { + // "VARIANT": "focal", + // "GCC_VER": "11", + // "LLVM_VER": "13" + // } + }, + "runArgs": [ + "--cap-add=SYS_PTRACE", + "--security-opt", + "seccomp=unconfined" + ], + // Set *default* container specific settings.json values on container create. + "settings": { + "cmake.configureOnOpen": true, + "editor.formatOnSave": true + }, + // Add the IDs of extensions you want installed when the container is created. + "extensions": [ + "ms-vscode.cpptools-extension-pack", + "cschlosser.doxdocgen", + "ms-python.python", + "ms-python.vscode-pylance", + ], + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + // Use 'postCreateCommand' to run commands after the container is created. + "postCreateCommand": "sudo apt update && sudo apt install -y g++ make cmake libgmp-dev libssl-dev libboost-all-dev software-properties-common cmake git build-essential libssl-dev xxd", + // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. + //"remoteUser": "vscode", + "workspaceMount": "source=${localWorkspaceFolder},target=/workspaces/${localWorkspaceFolderBasename},type=bind,consistency=delegated", + "workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}", + "features": { + "git": "latest", + "git-lfs": "latest" + } +} \ No newline at end of file diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..3f102d8 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +.git/ +build/ +**/build/ +**/CMakeCache.txt +**/.git/ +docker/ +plottertest/ +results/ +results-paper/ diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml new file mode 100644 index 0000000..77e4da5 --- /dev/null +++ b/.github/workflows/devcontainer.yml @@ -0,0 +1,68 @@ +name: devcontainer + +on: + push: + branches: [ "master" ] + pull_request: + branches: [ "master" ] + +env: + # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.) + BUILD_TYPE: Release + +jobs: + build: + runs-on: [self-hosted, linux, x64] + + steps: + - name: Copy Runner Env to Workflow Env + run: | + echo "::add-mask::${LOCAL_REG_URL}" + echo "::add-mask::${LOCAL_REG_USER}" + echo "::add-mask::${LOCAL_REG_PASS}" + echo "LOCAL_REG_URL=${LOCAL_REG_URL}" >> $GITHUB_ENV + echo "LOCAL_REG_USER=${LOCAL_REG_USER}" >> $GITHUB_ENV + echo "LOCAL_REG_PASS=${LOCAL_REG_PASS}" >> $GITHUB_ENV + + - uses: actions/checkout@v4 + with: + submodules: 'recursive' + token: ${{ secrets.SUBMODULE_PAT }} + + - uses: actions/setup-node@v4 + with: + node-version: 'latest' + + - name: Docker Login to Local Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.LOCAL_REG_URL }} + username: ${{ env.LOCAL_REG_USER }} + password: ${{ env.LOCAL_REG_PASS }} + + - name: Install updated Skopeo + # This can be omitted once runner images have a version of Skopeo > 1.4.1 + # See https://github.com/containers/skopeo/issues/1874 + run: | + # sudo apt purge buildah golang-github-containers-common podman skopeo + # sudo apt autoremove --purge + REPO_URL="https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/unstable" + source /etc/os-release + sudo sh -c "echo 'deb ${REPO_URL}/x${NAME}_${VERSION_ID}/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:unstable.list" + sudo wget -qnv https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/x${NAME}_${VERSION_ID}/Release.key -O Release.key + sudo apt-key add Release.key + sudo apt-get update + sudo apt-get install skopeo + + - name: Configure CMake in Cached devcontainer + uses: devcontainers/ci@v0.3 + env: + BUILDX_NO_DEFAULT_ATTESTATIONS: true + with: + imageName: ${{ env.LOCAL_REG_URL }}/secret-snail/localization-server-devcontainer + cacheFrom: ${{ env.LOCAL_REG_URL }}/secret-snail/localization-server-devcontainer + push: always + runCmd: | + cmake -B ./build -S ./ -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} && \ + cd ./build && make -j8 && ctest + diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml new file mode 100644 index 0000000..eb4d7ee --- /dev/null +++ b/.github/workflows/docker-image.yml @@ -0,0 +1,44 @@ +name: container + +on: + push: + branches: [ "master" ] + pull_request: + branches: [ "master" ] + +jobs: + + build: + runs-on: [self-hosted, linux, x64] + steps: + - name: Copy Runner Env to Workflow Env + run: | + echo "::add-mask::${LOCAL_REG_URL}" + echo "::add-mask::${LOCAL_REG_USER}" + echo "::add-mask::${LOCAL_REG_PASS}" + echo "LOCAL_REG_URL=${LOCAL_REG_URL}" >> $GITHUB_ENV + echo "LOCAL_REG_USER=${LOCAL_REG_USER}" >> $GITHUB_ENV + echo "LOCAL_REG_PASS=${LOCAL_REG_PASS}" >> $GITHUB_ENV + + - uses: actions/checkout@v4 + with: + submodules: 'recursive' + token: ${{ secrets.SUBMODULE_PAT }} + + - name: Docker Login to Local Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.LOCAL_REG_URL }} + username: ${{ env.LOCAL_REG_USER }} + password: ${{ env.LOCAL_REG_PASS }} + + - name: Build the Docker image + run: docker build . --file ./docker/Dockerfile --tag ${{ env.LOCAL_REG_URL }}/secret-snail/localization-server:latest + + - name: Run the tests in the container + run: | + docker run --rm ${{ env.LOCAL_REG_URL }}/secret-snail/localization-server:latest \ + bash -c "cd ./build && ctest" + + - name: Push the Docker image + run: docker push ${{ env.LOCAL_REG_URL }}/secret-snail/localization-server:latest diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..abbfdea --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +plottertest/*.pdf +**/build +**/debug +**/tags +**Session.vim +.cache/ +data-eth3d/** +ln +compile_commands.json +results/** +plots/** diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..4975acb --- /dev/null +++ b/.gitmodules @@ -0,0 +1,21 @@ +[submodule "opencv"] + path = extern/opencv + url = https://github.com/opencv/opencv +[submodule "opencv_contrib"] + path = extern/opencv_contrib + url = https://github.com/opencv/opencv_contrib +[submodule "ABY"] + path = extern/ABY + url = ../server-aby +[submodule "extern/emp-tool"] + path = extern/emp-tool + url = ../server-emp-tool +[submodule "extern/emp-sh2pc"] + path = extern/emp-sh2pc + url = ../server-emp-sh2pc +[submodule "extern/emp-ot"] + path = extern/emp-ot + url = https://github.com/emp-toolkit/emp-ot +[submodule "extern/Catch2"] + path = extern/Catch2 + url = https://github.com/catchorg/Catch2 diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..6901981 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,32 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "(gdb) Launch", + "type": "cppdbg", + "request": "launch", + "program": "${workspaceFolder}/build/bin/emp_float_server_tester", + "args": [], + "stopAtEntry": false, + "cwd": "${fileDirname}", + "environment": [], + "externalConsole": false, + "MIMode": "gdb", + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + }, + { + "description": "Set Disassembly Flavor to Intel", + "text": "-gdb-set disassembly-flavor intel", + "ignoreFailures": true + } + ] + } + ] +} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..41be605 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,10 @@ +{ + "cmake.configureOnOpen": false, + "cmake.generator": "Unix Makefiles", + "cmake.parallelJobs": 1, // only needed because tests must run sequentially + "C_Cpp.formatting": "clangFormat", + "files.associations": { + "thread": "cpp", + "cmath": "cpp" + }, +} \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..93ca298 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,147 @@ +cmake_minimum_required(VERSION 3.12) +project(SecureLocal LANGUAGES CXX) + +# YouCompleteMe +set( CMAKE_EXPORT_COMPILE_COMMANDS ON ) +execute_process(COMMAND ln -s ln -s build/compile_commands.json ./ ) + +if (CMAKE_COMPILER_IS_GNUCXX AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8.0) + message(FATAL_ERROR "requires at least g++-8") +endif() + +#set (CMAKE_CXX_STANDARD 11) + +set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -ggdb -fno-omit-frame-pointer") + +# Add sanitizer flags for C++ compiler for "Debug" configuration +set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fsanitize=address,undefined") + +# Set build type to `Release` if non was specified: +# (cf. https://gitlab.kitware.com/cmake/community/wikis/FAQ#how-can-i-change-the-default-build-mode-and-see-it-reflected-in-the-gui) +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release CACHE STRING + "Choose the type of build, options are: None Debug Release RelWithDebInfo MinSizeRel." + FORCE) +endif(NOT CMAKE_BUILD_TYPE) + +# Write built executables and libraries to bin/ and lib/, respectively. +if(NOT CMAKE_RUNTIME_OUTPUT_DIRECTORY) + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}/bin") +endif() +if(NOT CMAKE_LIBRARY_OUTPUT_DIRECTORY) + set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}/lib") +endif() +if(NOT CMAKE_ARCHIVE_OUTPUT_DIRECTORY) + set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}/lib") +endif() + +set(CMAKE_POSITION_INDEPENDENT_CODE ON) +add_subdirectory(extern/ABY) +set(CMAKE_POSITION_INDEPENDENT_CODE OFF) + + + +# To used an opencv version installed in build +# You might need to do this first??? +#set(OpenCV_DIR ${CMAKE_CURRENT_BINARY_DIR}) +#set(BUILD_SHARED_LIBS OFF) +#add_subdirectory(extern/opencv) +#set(INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_BINARY_DIR}/") + +#find_package(OpenCV REQUIRED) +#add_library(opencv INTERFACE) +#include_directories(${OpenCV_INCLUDE_DIRS}) + +# To use the version installed on the system +#find_package(opencv4 QUIET) + +execute_process(COMMAND mkdir build + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/extern/opencv/") +execute_process(COMMAND cmake .. + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/extern/opencv/build") +execute_process(COMMAND make -j16 + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/extern/opencv/build") +execute_process(COMMAND sudo make install + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/extern/opencv/build") +set(OpenCV_DIR "") +find_package(OpenCV REQUIRED) +#include_directories(${OpenCV_INCLUDE_DIRS}) +message(STATUS "cv libs: ${OpenCV_LIBS}") +message(STATUS "include_path: ${OpenCV_INCLUDE_DIRS}") + + + + +foreach(empdir tool ot sh2pc) +# execute_process(COMMAND rm -rf build/ +# WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/extern/emp-${empdir}") +# execute_process(COMMAND rm -rf CMakeCache.txt +# WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/extern/emp-${empdir}") +# execute_process(COMMAND make clean +# WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/extern/emp-${empdir}") + execute_process(COMMAND cmake -DENABLE_FLOAT=On -DTHREADING=On . + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/extern/emp-${empdir}") + execute_process(COMMAND make -j16 + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/extern/emp-${empdir}") + execute_process(COMMAND sudo make install + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/extern/emp-${empdir}") + find_package(emp-tool REQUIRED) + include_directories("${CMAKE_CURRENT_SOURCE_DIR}/extern/emp-${empdir}") +endforeach() +find_package(OpenSSL REQUIRED) +find_package(Boost REQUIRED COMPONENTS system) +find_package(emp-tool REQUIRED) +find_package(emp-ot REQUIRED) +find_package(emp-sh2pc REQUIRED) +#find_package(GMP REQUIRED) +include(${CMAKE_CURRENT_SOURCE_DIR}/extern/emp-sh2pc/cmake/emp-sh2pc-config.cmake) + +# turn on threading for this code b/c it's turned on for emp +ADD_DEFINITIONS(-DTHREADING) + + +add_subdirectory("${CMAKE_CURRENT_SOURCE_DIR}/extern/Catch2") +include(CTest) +include(Catch) + +# additional target to perform clang-format run, requires clang-format +file(GLOB_RECURSE ALL_SOURCE_FILES src/*.cpp src/*.h src/*.hpp test/*.cpp test/*.h test/*.hpp) +#set(CLANG_FORMAT_EXCLUDE "") +#foreach (SOURCE_FILE ${ALL_SOURCE_FILES}) +# string(FIND ${SOURCE_FILE} ${CLANG_FORMAT_EXCLUDE} CLANG_FORMAT_EXCLUDE_FOUND) +# if (NOT ${CLANG_FORMAT_EXCLUDE_FOUND} EQUAL -1) +# list(REMOVE_ITEM ALL_SOURCE_FILES ${SOURCE_FILE}) +# endif () +#endforeach () + +add_custom_target( + clangformat + COMMAND /usr/bin/clang-format + -style=file + -i + ${ALL_SOURCE_FILES} +) + +add_custom_target( + checkclangformat + DEPENDS ${ALL_SOURCE_FILES} + COMMAND /usr/bin/clang-format + -style=file + --dry-run -Werror --ferror-limit=10 + ${ALL_SOURCE_FILES} +) + + +#find_package(RELIC::relic REQUIRED) +#add_subdirectory(extern/relic) + +#find_package(RELIC QUIET) +#find_package(RELIC::relic REQUIRED) +#add_library(RELIC::relic STATIC IMPORTED) + +#find_package(GMP REQUIRED) +#find_package(Threads REQUIRED) +#find_package(Boost 1.66.0 REQUIRED COMPONENTS thread system) + +add_subdirectory(src/) +add_subdirectory(test/) diff --git a/README.md b/README.md new file mode 100644 index 0000000..7656b48 --- /dev/null +++ b/README.md @@ -0,0 +1,155 @@ +# Secure Localization Server +[![devcontainer](https://github.com/secret-snail/localization-server/actions/workflows/devcontainer.yml/badge.svg)](https://github.com/secret-snail/localization-server/actions/workflows/devcontainer.yml) +[![container](https://github.com/secret-snail/localization-server/actions/workflows/docker-image.yml/badge.svg)](https://github.com/secret-snail/localization-server/actions/workflows/docker-image.yml) + +Privacy preserving localization based on secure multiparty computation (MPC). + +## Build via Container +```bash +docker/build.sh +``` + +Run an interactive shell inside the conatiner: +```bash +docker run -it --rm --init \ + --net=host \ + --name snail-server \ + snail-server bash +``` + +## Build From Source +ABY Dependencies: +`sudo apt install g++ make cmake libgmp-dev libssl-dev libboost-all-dev` + +EMP Dependencies: +`sudo apt install software-properties-common cmake git build-essential libssl-dev` + +Build and test the code: +``` +git submodule update --init --recursive +mkdir build && cd build +cmake .. +make +ctest +``` + +ABY testing requires -DCMAKE_BUILD_TYPE=Release, tests fail when the sanitizers +are turned on because there appears to be memory leaks in the ABY library. + +Note - circuits cannot be built on the fly. must be fully specified then executed. +This means if control flow requires some secret data, circuit must be broken and +intermediate ciphertext stored as secret share. +[Developer Guide](https://www.informatik.tu-darmstadt.de/media/encrypto/encrypto_code/abydevguide.pdf) +[Reusing Computation](https://github.com/encryptogroup/ABY/issues/167) + +## Experimental Evaluation +First, download the +[eth3d dataset](https://www.eth3d.net/datasets#high-res-multi-view) with the +following commands. These will specifically download the high-res multi-view +undistorted images and ground truth scan evaluation from eth3d. +```bash +sudo apt-get install p7zip-full +mkdir ./data-eth3d +curl https://www.eth3d.net/data/multi_view_training_dslr_undistorted.7z -o im.7z +7z x im.7z -o./data-eth3d/ +rm im.7z +curl https://www.eth3d.net/data/multi_view_training_dslr_scan_eval.7z -o gt.7z +7z x gt.7z -o./data-eth3d/ +rm gt.7z +``` + +The easiest way to run the experiments is via the provided Docker container. +Build and start the container with the following commands. + +```bash +docker/build.sh + +mkdir -p results + +docker run -it --rm --init \ + --net=host \ + --name snail-tester \ + --volume "$(pwd)/results":/snail/results \ + snail-server bash +``` + +Run the experiments. Note some of the commands must be run outside the container +due to network setup requirements. They are marked with `# outside container`. + +```bash +# ABY and EMP single-iteration localization tests. +sed -i 's/#define PPL_FLOW .*/#define PPL_FLOW PPL_FLOW_SiSL/' src/common/privacyconf.h +(cd build/ && make) +LAT=0msec source scripts/network_setup.sh # outside container +scripts/emp_float_benchmark_run.sh # 2.5 hours +scripts/aby_float_benchmark_run.sh # 7.5 hours +scripts/mult_add_fixed_float_time_run.sh # 30 seconds + +# EMP data-oblivious tests. +sed -i 's/#define PPL_FLOW .*/#define PPL_FLOW PPL_FLOW_DO/' src/common/privacyconf.h +(cd build/ && make) +LAT=0msec source scripts/network_setup.sh # outside container +scripts/emp_float_benchmark_dataobl_run.sh # 38 hours + +# EMP single-iteration localization tests with network latency. +sed -i 's/#define PPL_FLOW .*/#define PPL_FLOW PPL_FLOW_SiSL/' src/common/privacyconf.h +(cd build/ && make) +LAT=5msec source scripts/network_setup.sh # outside container +scripts/emp_float_benchmark_run_latency.sh # 3 hours + +source scripts/network_teardown.sh # outside container +exit # stop the container +``` + +Install dependencies for the plotter scripts (requires python3 and pip3). + +```bash +./plotter_deps.sh +``` + +Plot the results. + +```bash +mkdir plots +scripts/emp_vs_aby_plot.sh +scripts/loopleak_vs_dataobl_plot.sh +scripts/emp_float_benchmark_plot.sh +scripts/netio_plot.sh +scripts/num_arith_ops_plot.sh +scripts/mult_add_fixed_float_time_plot.sh +``` + +## Robotic Snail Demo +*Requires raspberry pi, see [snail repo](https://github.com/secret-snail/snail)* + +Currently, the demo requires `privacy_conf.h` to have this set: +``` +#define PPL_FLOW PPL_FLOW_LOOP_LEAK +``` + +To run the snail demo, first `docker/build.sh` the container. Then run +`docker/alice.sh` and `docker/bob.sh` in two terminals on the server(s). +On the [snail](https://github.com/secret-snail/snail), run +`sudo ./build/bin/visp_snail --secure`. +Make sure the snail can see the marker in the first 10 seconds, otherwise the +snail will need to be restarted. + + +## Running on Separate Machines +Set the machines IP addresses in `test/emp-float/client.cpp` and +`src/emp-float-server/server-lm.cpp` replacing the localhost IP. + +```bash +cd build/bin +./emp_float_client +``` + +```bash +cd build/bin +./lm_emp_float_server 0 +``` + +```bash +cd build/bin +./lm_emp_float_server 1 +``` diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..846dbc6 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,24 @@ +FROM ubuntu:22.04 + +RUN apt update && apt upgrade -y && apt install -y sudo +RUN DEBIAN_FRONTEND=noninteractive apt install -y g++ make cmake libgmp-dev libssl-dev libboost-all-dev + +RUN DEBIAN_FRONTEND=noninteractive apt install -y software-properties-common cmake git build-essential libssl-dev xxd + +WORKDIR /snail + +# build opencv first for caching +COPY ./extern/opencv ./extern/opencv +COPY ./extern/opencv_contrib ./extern/opencv_contrib +RUN cd extern/opencv && \ + mkdir build && \ + cmake -S ./ -B ./build/ && \ + cmake --build ./build/ --parallel 8 && \ + cd build && sudo make install + +COPY ./ ./ + +RUN mkdir build && \ + cd build && \ + cmake -DCMAKE_BUILD_TYPE=Release .. && \ + make -j 8 diff --git a/docker/alice.sh b/docker/alice.sh new file mode 100755 index 0000000..2d8f4f1 --- /dev/null +++ b/docker/alice.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +scriptpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd $scriptpath + +cmd="./build/bin/lm_emp_float_server 0" + +# need --init so ctrl-c signal is passed to proc correctly +# need --net=host if two containers on same machine talking +# through localhost, because you can't open same port (8080) +# for two containers. +# otherwise open the following ports: + #-p 8080:8080 \ + #-p 8097:8097 \ + +docker run -it --rm --init \ + --net=host \ + --name snail-server-alice \ + snail-server $cmd diff --git a/docker/bob.sh b/docker/bob.sh new file mode 100755 index 0000000..1290b0b --- /dev/null +++ b/docker/bob.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +scriptpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd $scriptpath + +cmd="./build/bin/lm_emp_float_server 1" + +# need --init so ctrl-c signal is passed to proc correctly +# need --net=host if two containers on same machine talking +# through localhost, because you can't open same port (8080) +# for two containers. +# otherwise open the following ports: + #-p 8114:8114 \ + +docker run -it --rm --init \ + --net=host \ + --name snail-server-bob \ + snail-server $cmd + diff --git a/docker/build.sh b/docker/build.sh new file mode 100755 index 0000000..3affea8 --- /dev/null +++ b/docker/build.sh @@ -0,0 +1,5 @@ +#!/bin/bash +scriptpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd $scriptpath/.. + +docker build -f ./docker/Dockerfile --tag=snail-server . diff --git a/docker/run.sh b/docker/run.sh new file mode 100755 index 0000000..ae22c62 --- /dev/null +++ b/docker/run.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +scriptpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd $scriptpath + +cmd="./build/bin/lm_emp_float_server 0 & +./build/bin/lm_emp_float_server 1" + +docker run -it --rm \ + --init \ + -p 8097:8097 \ + -p 8114:8114 \ + --name snail-server \ + snail-server bash -c "$cmd" diff --git a/extern/ABY b/extern/ABY new file mode 160000 index 0000000..bfb316f --- /dev/null +++ b/extern/ABY @@ -0,0 +1 @@ +Subproject commit bfb316f65ce49810407f60d355dcefabc33a3de9 diff --git a/extern/Catch2 b/extern/Catch2 new file mode 160000 index 0000000..77f7c01 --- /dev/null +++ b/extern/Catch2 @@ -0,0 +1 @@ +Subproject commit 77f7c0104dbaa43c31e75147cfdbd4142484914b diff --git a/extern/emp-ot b/extern/emp-ot new file mode 160000 index 0000000..7f3d4f0 --- /dev/null +++ b/extern/emp-ot @@ -0,0 +1 @@ +Subproject commit 7f3d4f02d2f44da8c6d355c9ec8159910faf4044 diff --git a/extern/emp-sh2pc b/extern/emp-sh2pc new file mode 160000 index 0000000..2aa4493 --- /dev/null +++ b/extern/emp-sh2pc @@ -0,0 +1 @@ +Subproject commit 2aa44934d5aaf60b4b5da07df1031d6394a6e286 diff --git a/extern/emp-tool b/extern/emp-tool new file mode 160000 index 0000000..3610728 --- /dev/null +++ b/extern/emp-tool @@ -0,0 +1 @@ +Subproject commit 361072844cfff26a07ed3d4d298e21d81afe1726 diff --git a/extern/opencv b/extern/opencv new file mode 160000 index 0000000..69357b1 --- /dev/null +++ b/extern/opencv @@ -0,0 +1 @@ +Subproject commit 69357b1e88680658a07cffde7678a4d697469f03 diff --git a/extern/opencv_contrib b/extern/opencv_contrib new file mode 160000 index 0000000..f5d7f67 --- /dev/null +++ b/extern/opencv_contrib @@ -0,0 +1 @@ +Subproject commit f5d7f6712d4ff229ba4f45cf79dfd11c557d56fd diff --git a/plotter_deps.sh b/plotter_deps.sh new file mode 100755 index 0000000..6d07cf5 --- /dev/null +++ b/plotter_deps.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +pip3 install numpy matplotlib diff --git a/plottertest/3dbarplot.sh b/plottertest/3dbarplot.sh new file mode 100755 index 0000000..df06f23 --- /dev/null +++ b/plottertest/3dbarplot.sh @@ -0,0 +1,9 @@ +#!/bin/bash +scriptpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +outputdir=$scriptpath + +python3 $scriptpath/../plotter.py \ + --csvlog "$scriptpath/3dbarplottest.csv" \ + --graphpath "$outputdir/3dbarplottest.pdf" \ + --show + diff --git a/plottertest/3dbarplot_customized.sh b/plottertest/3dbarplot_customized.sh new file mode 100755 index 0000000..d3b08d2 --- /dev/null +++ b/plottertest/3dbarplot_customized.sh @@ -0,0 +1,16 @@ +#!/bin/bash +scriptpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +outputdir=$scriptpath + +python3 $scriptpath/../plotter.py \ + --csvlog "$scriptpath/3dbarplottest.csv" \ + --graphpath "$outputdir/3dbarplottest.pdf" \ + --only-tags "label-a" "label-b" "label-c" \ + --title "3dbar Test" \ + --xlabel "X-Label" \ + --ylabel "Y-Label" \ + --custom-legend-labels "custom-label-a" "custom-label-b" "custom-label-c" \ + --color-theme "dracula" \ + --color-offset 1 \ + --show + diff --git a/plottertest/3dbarplottest.csv b/plottertest/3dbarplottest.csv new file mode 100644 index 0000000..4f91fa9 --- /dev/null +++ b/plottertest/3dbarplottest.csv @@ -0,0 +1,37 @@ +SeNtInAl,3dbar,func,label-a,xone,yone,0.5 +SeNtInAl,3dbar,func,label-a,xone,yone,1.5 + +SeNtInAl,3dbar,func,label-a,xtwo,ytwo,1.5 +SeNtInAl,3dbar,func,label-a,xtwo,ytwo,2.5 + +SeNtInAl,3dbar,func,label-a,xthree,ythree,2.5 +SeNtInAl,3dbar,func,label-a,xthree,ythree,3.5 + +SeNtInAl,3dbar,func,label-a,xthree,yfour,3.5 +SeNtInAl,3dbar,func,label-a,xthree,yfour,4.5 + + +SeNtInAl,3dbar,func,label-b,xone,yone,0.5 +SeNtInAl,3dbar,func,label-b,xone,yone,1.5 + +SeNtInAl,3dbar,func,label-b,xtwo,ytwo,1.5 +SeNtInAl,3dbar,func,label-b,xtwo,ytwo,2.5 + +SeNtInAl,3dbar,func,label-b,xthree,ythree,2.5 +SeNtInAl,3dbar,func,label-b,xthree,ythree,3.5 + +SeNtInAl,3dbar,func,label-b,xthree,yfour,3.5 +SeNtInAl,3dbar,func,label-b,xthree,yfour,4.5 + + +SeNtInAl,3dbar,func,label-c,xone,yone,0.5 +SeNtInAl,3dbar,func,label-c,xone,yone,1.5 + +SeNtInAl,3dbar,func,label-c,xtwo,ytwo,1.5 +SeNtInAl,3dbar,func,label-c,xtwo,ytwo,2.5 + +SeNtInAl,3dbar,func,label-c,xthree,ythree,2.5 +SeNtInAl,3dbar,func,label-c,xthree,ythree,3.5 + +SeNtInAl,3dbar,func,label-c,xtwo,yfour,0.5 +SeNtInAl,3dbar,func,label-c,xtwo,yfour,1.5 diff --git a/plottertest/3dsurfplot.sh b/plottertest/3dsurfplot.sh new file mode 100755 index 0000000..1db7c4b --- /dev/null +++ b/plottertest/3dsurfplot.sh @@ -0,0 +1,9 @@ +#!/bin/bash +scriptpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +outputdir=$scriptpath + +python3 $scriptpath/../plotter.py \ + --csvlog "$scriptpath/3dsurfplottest.csv" \ + --graphpath "$outputdir/3dsurfplottest.pdf" \ + --show + diff --git a/plottertest/3dsurfplot_customized.sh b/plottertest/3dsurfplot_customized.sh new file mode 100755 index 0000000..82b8ef5 --- /dev/null +++ b/plottertest/3dsurfplot_customized.sh @@ -0,0 +1,16 @@ +#!/bin/bash +scriptpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +outputdir=$scriptpath + +python3 $scriptpath/../plotter.py \ + --csvlog "$scriptpath/3dsurfplottest.csv" \ + --graphpath "$outputdir/3dsurfplottest.pdf" \ + --only-tags "label-a" \ + --title "3dsurf Test" \ + --xlabel "X-Label" \ + --ylabel "Y-Label" \ + --custom-legend-labels "custom-label-a" \ + --color-theme "dracula" \ + --color-offset 1 \ + --show + diff --git a/plottertest/3dsurfplottest.csv b/plottertest/3dsurfplottest.csv new file mode 100644 index 0000000..ef473c0 --- /dev/null +++ b/plottertest/3dsurfplottest.csv @@ -0,0 +1,16 @@ +SeNtInAl,3dsurf,func,label-a,xone,yone,0.5 +SeNtInAl,3dsurf,func,label-a,xone,yone,1.5 + +SeNtInAl,3dsurf,func,label-a,xtwo,ytwo,1.5 +SeNtInAl,3dsurf,func,label-a,xtwo,ytwo,2.5 + +SeNtInAl,3dsurf,func,label-a,xthree,ythree,2.5 +SeNtInAl,3dsurf,func,label-a,xthree,ythree,3.5 + +SeNtInAl,3dsurf,func,label-a,xthree,yfour,3.5 +SeNtInAl,3dsurf,func,label-a,xthree,yfour,4.5 + +SeNtInAl,3dsurf,func,label-a,xtwo,yone,1.5 +SeNtInAl,3dsurf,func,label-a,xthree,yone,2.5 +SeNtInAl,3dsurf,func,label-a,xthree,ytwo,3.5 + diff --git a/plottertest/barboxplot.sh b/plottertest/barboxplot.sh new file mode 100755 index 0000000..03fe611 --- /dev/null +++ b/plottertest/barboxplot.sh @@ -0,0 +1,9 @@ +#!/bin/bash +scriptpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +outputdir=$scriptpath + +python3 $scriptpath/../plotter.py \ + --csvlog "$scriptpath/barboxplottest.csv" \ + --graphpath "$outputdir/barboxplottest.pdf" \ + --show + diff --git a/plottertest/barboxplot_customized.sh b/plottertest/barboxplot_customized.sh new file mode 100755 index 0000000..365cfd6 --- /dev/null +++ b/plottertest/barboxplot_customized.sh @@ -0,0 +1,16 @@ +#!/bin/bash +scriptpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +outputdir=$scriptpath + +python3 $scriptpath/../plotter.py \ + --csvlog "$scriptpath/barboxplottest.csv" \ + --graphpath "$outputdir/barboxplottest.pdf" \ + --only-tags "label-a" "label-b" \ + --title "barbox Test" \ + --xlabel "X-Label" \ + --ylabel "Y-Label" \ + --custom-legend-labels "label-a" "label-b" \ + --color-theme "dracula" \ + --color-offset 1 \ + --show + diff --git a/plottertest/barboxplottest.csv b/plottertest/barboxplottest.csv new file mode 100644 index 0000000..b2febb3 --- /dev/null +++ b/plottertest/barboxplottest.csv @@ -0,0 +1,18 @@ +SeNtInAl,barbox,func,label-a,xone,0.5 +SeNtInAl,barbox,func,label-a,xone,1.5 + +SeNtInAl,barbox,func,label-a,xtwo,1.5 +SeNtInAl,barbox,func,label-a,xtwo,2.5 + +SeNtInAl,barbox,func,label-a,xthree,2.5 +SeNtInAl,barbox,func,label-a,xthree,3.5 + + +SeNtInAl,barbox,func,label-b,xone,0.5 +SeNtInAl,barbox,func,label-b,xone,1.5 + +SeNtInAl,barbox,func,label-b,xtwo,1.5 +SeNtInAl,barbox,func,label-b,xtwo,2.5 + +SeNtInAl,barbox,func,label-b,xthree,2.5 +SeNtInAl,barbox,func,label-b,xthree,3.5 diff --git a/plottertest/barplot.sh b/plottertest/barplot.sh new file mode 100755 index 0000000..7f91c30 --- /dev/null +++ b/plottertest/barplot.sh @@ -0,0 +1,9 @@ +#!/bin/bash +scriptpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +outputdir=$scriptpath + +python3 $scriptpath/../plotter.py \ + --csvlog "$scriptpath/barplottest.csv" \ + --graphpath "$outputdir/barplottest.pdf" \ + --show + diff --git a/plottertest/barplottest.csv b/plottertest/barplottest.csv new file mode 100644 index 0000000..5a378f6 --- /dev/null +++ b/plottertest/barplottest.csv @@ -0,0 +1,21 @@ +SeNtInAl,bar,func,label-a,xone,0.5 +SeNtInAl,bar,func,label-a,xone,1.5 + +SeNtInAl,bar,func,label-a,xtwo,1.5 +SeNtInAl,bar,func,label-a,xtwo,2.5 + +SeNtInAl,bar,func,label-a,xthree,2.5 +SeNtInAl,bar,func,label-a,xthree,3.5 + + +SeNtInAl,bar,func,label-b,xone,0.5 +SeNtInAl,bar,func,label-b,xone,1.5 + +SeNtInAl,bar,func,label-b,xtwo,1.5 +SeNtInAl,bar,func,label-b,xtwo,2.5 + +SeNtInAl,bar,func,label-b,xthree,2.5 +SeNtInAl,bar,func,label-b,xthree,3.5 + +SeNtInAl,bar,func,label-c,xone,.25 + diff --git a/plottertest/boxplot.sh b/plottertest/boxplot.sh new file mode 100755 index 0000000..f5f428e --- /dev/null +++ b/plottertest/boxplot.sh @@ -0,0 +1,9 @@ +#!/bin/bash +scriptpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +outputdir=$scriptpath + +python3 $scriptpath/../plotter.py \ + --csvlog "$scriptpath/boxplottest.csv" \ + --graphpath "$outputdir/boxplottest.pdf" \ + --show + diff --git a/plottertest/boxplottest.csv b/plottertest/boxplottest.csv new file mode 100644 index 0000000..30d572d --- /dev/null +++ b/plottertest/boxplottest.csv @@ -0,0 +1,18 @@ +SeNtInAl,box,func,label-a,xone,0.5 +SeNtInAl,box,func,label-a,xone,1.5 + +SeNtInAl,box,func,label-a,xtwo,1.5 +SeNtInAl,box,func,label-a,xtwo,2.5 + +SeNtInAl,box,func,label-a,xthree,2.5 +SeNtInAl,box,func,label-a,xthree,3.5 + + +SeNtInAl,box,func,label-b,xone,0.5 +SeNtInAl,box,func,label-b,xone,1.5 + +SeNtInAl,box,func,label-b,xtwo,1.5 +SeNtInAl,box,func,label-b,xtwo,2.5 + +SeNtInAl,box,func,label-b,xthree,2.5 +SeNtInAl,box,func,label-b,xthree,3.5 diff --git a/plottertest/grouped_bar.csv b/plottertest/grouped_bar.csv new file mode 100644 index 0000000..60f2214 --- /dev/null +++ b/plottertest/grouped_bar.csv @@ -0,0 +1,6 @@ +SeNtInAl,grouped_bar,hand-config,unscoped,xlab,10 +SeNtInAl,grouped_bar,hand-config,unscoped,x2lab,10 +SeNtInAl,grouped_bar,hand-config,scoped,xlab,8 +SeNtInAl,grouped_bar,hand-config,scoped,x2lab,8 +SeNtInAl,grouped_bar,hand-config,test,xlab,9 +SeNtInAl,grouped_bar,hand-config,test,x2lab,9 diff --git a/plottertest/grouped_bar.sh b/plottertest/grouped_bar.sh new file mode 100755 index 0000000..f2be4a2 --- /dev/null +++ b/plottertest/grouped_bar.sh @@ -0,0 +1,9 @@ +#!/bin/bash +scriptpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +outputdir=$scriptpath + +python3 $scriptpath/../plotter.py \ + --csvlog "$scriptpath/grouped_bar.csv" \ + --graphpath "$outputdir/grouped_bar.pdf" \ + --show + diff --git a/plottertest/xyplot.sh b/plottertest/xyplot.sh new file mode 100755 index 0000000..54a01b3 --- /dev/null +++ b/plottertest/xyplot.sh @@ -0,0 +1,9 @@ +#!/bin/bash +scriptpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +outputdir=$scriptpath + +python3 $scriptpath/../plotter.py \ + --csvlog "$scriptpath/xyplottest.csv" \ + --graphpath "$outputdir/xyplottest.pdf" \ + --show + diff --git a/plottertest/xyplot_customized.sh b/plottertest/xyplot_customized.sh new file mode 100755 index 0000000..3432d03 --- /dev/null +++ b/plottertest/xyplot_customized.sh @@ -0,0 +1,16 @@ +#!/bin/bash +scriptpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +outputdir=$scriptpath + +python3 $scriptpath/../plotter.py \ + --csvlog "$scriptpath/xyplottest.csv" \ + --graphpath "$outputdir/xyplottest.pdf" \ + --only-tags "label-a" "label-b" \ + --title "xy Test" \ + --xlabel "X-Label" \ + --ylabel "Y-Label" \ + --custom-legend-labels "label-a" "label-b" \ + --color-theme "dracula" \ + --color-offset 1 \ + --show + diff --git a/plottertest/xyplottest.csv b/plottertest/xyplottest.csv new file mode 100644 index 0000000..b0e8958 --- /dev/null +++ b/plottertest/xyplottest.csv @@ -0,0 +1,13 @@ +SeNtInAl,xy,func,label-a,1,0.5 +SeNtInAl,xy,func,label-a,2,1.5 +SeNtInAl,xy,func,label-a,3,2.5 + + +SeNtInAl,xy,func,label-b,1,0 +SeNtInAl,xy,func,label-b,1,0.25 + +SeNtInAl,xy,func,label-b,2,0.25 +SeNtInAl,xy,func,label-b,2,1.25 + +SeNtInAl,xy,func,label-b,3,1.25 +SeNtInAl,xy,func,label-b,3,2.25 diff --git a/scripts/aby_float_benchmark_run.sh b/scripts/aby_float_benchmark_run.sh new file mode 100755 index 0000000..0140239 --- /dev/null +++ b/scripts/aby_float_benchmark_run.sh @@ -0,0 +1,8 @@ +#!/bin/bash +scriptpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +NUM_FRAMES=1 +NUM_TRIALS=3 + +$scriptpath/../build/bin/aby_float_eth3d_bench gn $NUM_FRAMES $NUM_TRIALS 12 | tee $scriptpath/../results/gn_aby_float_eth3d_bench_short.log +$scriptpath/../build/bin/aby_float_eth3d_bench lm $NUM_FRAMES $NUM_TRIALS 12 | tee $scriptpath/../results/lm_aby_float_eth3d_bench_short.log \ No newline at end of file diff --git a/scripts/clear_fixed_benchmark_plot.sh b/scripts/clear_fixed_benchmark_plot.sh new file mode 100755 index 0000000..211c806 --- /dev/null +++ b/scripts/clear_fixed_benchmark_plot.sh @@ -0,0 +1,44 @@ +#!/bin/bash +scriptpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +logdir=$scriptpath/../results/ +plotdir=$scriptpath/../plots/ + +#SHOW="--show" +SHOW="" + + +# Plot error +python3 $scriptpath/../scripts/plotter.py \ + --csvlog "$logdir/gn_fixed_eth3d_bench_short.log" \ + --graphpath "$plotdir/gn_fixed_eth3d_bench_error.pdf" \ + --title "GN Error vs Ground Truth (plaintext)" \ + --only-tags "opencv_error_vs_numpts" \ + "gn_float_error_vs_numpts" \ + "gn_baset_error_vs_numpts" \ + --custom-legend-labels "opencv (float)" \ + "float" \ + "fixed (64bit)" \ + --xlabel "Number of Points" \ + --ylabel "Error (2norm)" \ + --color-theme "dracula" \ + --fig-w 5 \ + --fig-h 4 \ + $SHOW + +# Plot number of iterations until convergence +python3 $scriptpath/../scripts/plotter.py \ + --csvlog "$logdir/gn_fixed_eth3d_bench_short.log" \ + --graphpath "$plotdir/gn_fixed_eth3d_bench_iterations.pdf" \ + --title "GN Iterations (plaintext)" \ + --only-tags \ + "gn_float_iterations_vs_numpts" \ + "gn_baset_iterations_vs_numpts" \ + --custom-legend-labels \ + "float" \ + "fixed (64bit)" \ + --xlabel "Number of Points" \ + --ylabel "Iterations" \ + --color-theme "dracula" \ + --fig-w 5 \ + --fig-h 4 \ + $SHOW diff --git a/scripts/clear_fixed_benchmark_run.sh b/scripts/clear_fixed_benchmark_run.sh new file mode 100755 index 0000000..1ee18f2 --- /dev/null +++ b/scripts/clear_fixed_benchmark_run.sh @@ -0,0 +1,8 @@ +#!/bin/bash +scriptpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +killall gn_fixed_server_benchmark +killall lm_fixed_server_benchmark + +$scriptpath/../build/fixed_server_benchmark 0 1 6 > $scriptpath/../results/fixed_server_benchmark_short.log & +$scriptpath/../build/fixed_server_benchmark 1 1 6 diff --git a/scripts/emp_float_benchmark_dataobl_run.sh b/scripts/emp_float_benchmark_dataobl_run.sh new file mode 100755 index 0000000..04c7d02 --- /dev/null +++ b/scripts/emp_float_benchmark_dataobl_run.sh @@ -0,0 +1,10 @@ +#!/bin/bash +scriptpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# only runs the data oblivious implementation. +# dont forget to set privacyconf.h PPL_FLOW to data oblivious. + +# short run +$scriptpath/../build/bin/emp_float_eth3d_bench gn 3 2 12 | tee $scriptpath/../results/gn_emp_float_eth3d_bench_dataobl_short.log +$scriptpath/../build/bin/emp_float_eth3d_bench lm 3 2 12 | tee $scriptpath/../results/lm_emp_float_eth3d_bench_dataobl_short.log + diff --git a/scripts/emp_float_benchmark_plot.sh b/scripts/emp_float_benchmark_plot.sh new file mode 100755 index 0000000..a84a566 --- /dev/null +++ b/scripts/emp_float_benchmark_plot.sh @@ -0,0 +1,40 @@ +#!/bin/bash +scriptpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +logdir=$scriptpath/../results/ +plotdir=$scriptpath/../plots/ + +#SHOW="--show" +SHOW="" + +# relies on: +# emp_float_benchmark_run.sh +# emp_float_benchmark_run_latency.sh + +sed -i 's/,emp_float_gn_time_vs_points_per_loc_itr,/,emp_float_gn_time_vs_points_per_loc_itr_latency,/g' $scriptpath/../results/gn_emp_float_eth3d_bench_long_latency.log +sed -i 's/,emp_float_lm_time_vs_points_per_loc_itr,/,emp_float_lm_time_vs_points_per_loc_itr_latency,/g' $scriptpath/../results/lm_emp_float_eth3d_bench_long_latency.log + +python3 $scriptpath/../scripts/insetplotter.py \ + --csvlog \ + "$logdir/gn_emp_float_eth3d_bench_long.log" \ + "$logdir/lm_emp_float_eth3d_bench_long.log" \ + "$logdir/gn_emp_float_eth3d_bench_long_latency.log" \ + "$logdir/lm_emp_float_eth3d_bench_long_latency.log" \ + --graphpath "$plotdir/emp_float_runtime_long.pdf" \ + --title "Comparing Optimization +Algorithms at Large Input Sizes" \ + --only-tags \ + "emp_float_gn_time_vs_points_per_loc_itr" \ + "emp_float_lm_time_vs_points_per_loc_itr" \ + "emp_float_gn_time_vs_points_per_loc_itr_latency" \ + "emp_float_lm_time_vs_points_per_loc_itr_latency" \ + --custom-legend-labels \ + "GN (0 ms)" \ + "LM (0 ms)" \ + "GN (5 ms)" \ + "LM (5 ms)" \ + --xlabel "Number of Features" \ + --ylabel "Runtime per Iteration (s)" \ + --fig-w 4 \ + --fig-h 3 \ + --color-theme "dracula" \ + $SHOW diff --git a/scripts/emp_float_benchmark_run.sh b/scripts/emp_float_benchmark_run.sh new file mode 100755 index 0000000..fa94384 --- /dev/null +++ b/scripts/emp_float_benchmark_run.sh @@ -0,0 +1,13 @@ +#!/bin/bash +scriptpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +NUM_FRAMES=1 +NUM_TRIALS=3 + +# short run (12 points) +$scriptpath/../build/bin/emp_float_eth3d_bench gn $NUM_FRAMES $NUM_TRIALS 12 | tee $scriptpath/../results/gn_emp_float_eth3d_bench_short.log +$scriptpath/../build/bin/emp_float_eth3d_bench lm $NUM_FRAMES $NUM_TRIALS 12 | tee $scriptpath/../results/lm_emp_float_eth3d_bench_short.log + +# large run (256 points) +$scriptpath/../build/bin/emp_float_eth3d_bench gn $NUM_FRAMES $NUM_TRIALS 256 | tee $scriptpath/../results/gn_emp_float_eth3d_bench_long.log +$scriptpath/../build/bin/emp_float_eth3d_bench lm $NUM_FRAMES $NUM_TRIALS 256 | tee $scriptpath/../results/lm_emp_float_eth3d_bench_long.log \ No newline at end of file diff --git a/scripts/emp_float_benchmark_run_latency.sh b/scripts/emp_float_benchmark_run_latency.sh new file mode 100755 index 0000000..3a7eb28 --- /dev/null +++ b/scripts/emp_float_benchmark_run_latency.sh @@ -0,0 +1,12 @@ +#!/bin/bash +scriptpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +NUM_FRAMES=1 +NUM_TRIALS=3 + +# long run only (256 points) +$scriptpath/../build/bin/emp_float_eth3d_bench gn $NUM_FRAMES $NUM_TRIALS 256 | tee $scriptpath/../results/gn_emp_float_eth3d_bench_long_latency.log +$scriptpath/../build/bin/emp_float_eth3d_bench lm $NUM_FRAMES $NUM_TRIALS 256 | tee $scriptpath/../results/lm_emp_float_eth3d_bench_long_latency.log + +sed -i 's/emp_float_gn_time_vs_points_per_loc_itr/emp_float_gn_time_vs_points_per_loc_itr_latency/g' $scriptpath/../results/gn_emp_float_eth3d_bench_long_latency.log +sed -i 's/emp_float_lm_time_vs_points_per_loc_itr/emp_float_lm_time_vs_points_per_loc_itr_latency/g' $scriptpath/../results/lm_emp_float_eth3d_bench_long_latency.log diff --git a/scripts/emp_vs_aby_plot.sh b/scripts/emp_vs_aby_plot.sh new file mode 100755 index 0000000..2e395b2 --- /dev/null +++ b/scripts/emp_vs_aby_plot.sh @@ -0,0 +1,51 @@ +#!/bin/bash +scriptpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +logdir=$scriptpath/../results/ +plotdir=$scriptpath/../plots/ + +#SHOW="--show" +SHOW="" + +# relies on: +# aby_float_benchmark_run.sh +# emp_float_benchmark_run.sh + +sed -i 's/xy,operator(),emp_float_gn_time_vs_point/grouped_bar,operator(),emp_float_gn_time_vs_point/g' $scriptpath/../results/gn_emp_float_eth3d_bench_short.log +sed -i 's/xy,operator(),emp_float_lm_time_vs_point/grouped_bar,operator(),emp_float_lm_time_vs_point/g' $scriptpath/../results/lm_emp_float_eth3d_bench_short.log +sed -i 's/xy,operator(),aby_bool_float_gn_time_vs_point/grouped_bar,operator(),aby_bool_float_gn_time_vs_point/g' $scriptpath/../results/gn_aby_float_eth3d_bench_short.log +sed -i 's/xy,operator(),aby_yao_float_gn_time_vs_point/grouped_bar,operator(),aby_yao_float_gn_time_vs_point/g' $scriptpath/../results/gn_aby_float_eth3d_bench_short.log +sed -i 's/xy,operator(),aby_bool_float_lm_time_vs_point/grouped_bar,operator(),aby_bool_float_lm_time_vs_point/g' $scriptpath/../results/lm_aby_float_eth3d_bench_short.log +sed -i 's/xy,operator(),aby_yao_float_lm_time_vs_point/grouped_bar,operator(),aby_yao_float_lm_time_vs_point/g' $scriptpath/../results/lm_aby_float_eth3d_bench_short.log + +# may need to change plot tags from xy to grouped_bar +# comment out twelve points :%s/\(.*\),12,/#\1,12,/g +python3 $scriptpath/../scripts/plotter-emp-vs-aby.py \ + --csvlog \ + "$logdir/gn_aby_float_eth3d_bench_short.log" \ + "$logdir/lm_aby_float_eth3d_bench_short.log" \ + "$logdir/gn_emp_float_eth3d_bench_short.log" \ + "$logdir/lm_emp_float_eth3d_bench_short.log" \ + --graphpath "$plotdir/emp_vs_aby.pdf" \ + --title "ABY and EMP Localization Runtime" \ + --only-tags \ + "aby_yao_float_gn_time_vs_points" \ + "aby_bool_float_gn_time_vs_points" \ + "aby_yao_float_lm_time_vs_points" \ + "aby_bool_float_lm_time_vs_points" \ + "emp_float_gn_time_vs_points" \ + "emp_float_lm_time_vs_points" \ + --custom-legend-labels \ + "ABY Yao GN" \ + "ABY Bool GN" \ + "ABY Yao LM" \ + "ABY Bool LM" \ + "EMP GN" \ + "EMP LM" \ + --xlabel "Number of Features" \ + --ylabel "Runtime (s)" \ + --fig-w 4 \ + --fig-h 5 \ + --log-scale \ + --horizontal \ + --color-theme "dracula" \ + $SHOW diff --git a/scripts/insetplotter.py b/scripts/insetplotter.py new file mode 100644 index 0000000..1974b63 --- /dev/null +++ b/scripts/insetplotter.py @@ -0,0 +1,780 @@ +import matplotlib.pyplot as plt +import matplotlib.patheffects as pe +from mpl_toolkits.mplot3d import Axes3D +from matplotlib import cm +import matplotlib.colors +import numpy as np +from operator import add +import os +import argparse +import enum +import sys +import json +import re + +from mpl_toolkits.axes_grid1.inset_locator import zoomed_inset_axes +from mpl_toolkits.axes_grid1.inset_locator import mark_inset + + +sentinalStr = "SeNtInAl" +tmpFile = "plotlines.tmp" +maxColumns = 7 + +# Plotter cheat sheet +# +# XY +# SeNtInAl, tag_type, user_defined, tag_name, x, y +# +# +# BAR | GROUPED_BAR | BARBOX | BOX +# SeNtInAl, tag_type, user_defined, tag_name, x_label, y +# +# tag_names - plotting multiple will create segmented bars with +# a legend. Legend can be overridden with --custom-legend-labels +# x_label - string used to denote X +# y - float value +# +# THREEDBAR +# SeNtInAl, tag_type, z, tag_name, x_label, y_label +# tag_names - plotting multiple will create segmented bars with +# a legend. Legend can be overridden with --custom-legend-labels +# x_label - string used to denote X +# y_label - string used to denote Y +# z - float + + + +class TAG_TYPE(enum.Enum): + UNDEFINED = 0 + XY = 1 + HISTOGRAM = 2 + CDF = 3 + BAR = 4 + BOX = 5 + BARBOX = 6 + GROUPED_BAR = 7 + THREEDBAR = 8 + THREEDSURF = 9 + THREEDPROJECTX = 10 + THREEDPROJECTY = 11 + THREEDUNROLL = 12 + +#class dracula(): +# CYAN = '#8be9fd' +# GREEN = '#50fa7b' +# ORANGE = '#ffb86c' +# PINK = '#ff79c6' +# PURPLE = '#bd93f9' +# RED = '#ff5555' +# YELLOW = '#f1fa8c' +# COLORS = [CYAN, ORANGE, GREEN, PINK, PURPLE, YELLOW, RED] + +class dracula(): + ZERO = '#D291AC' + ONE = '#779E44' + TWO = '#F6B7AA' + THREE = '#367E51' + FOUR = '#5F1333' + FIVE = '#426615' + SIX = '#AD5B4A' + #COLORS = [ZERO, ONE, TWO, THREE, FOUR, FIVE, SIX, ZERO, ONE, TWO, THREE, FOUR] + COLORS = [plt.cm.Pastel1(i) for i in range(9)] + +#cmap1 = matplotlib.colors.LinearSegmentedColormap.from_list("", [dracula.PINK, dracula.PURPLE]) +#cmap2 = matplotlib.colors.LinearSegmentedColormap.from_list("", [dracula.CYAN, dracula.ORANGE]) + +cmap1 = matplotlib.colors.LinearSegmentedColormap.from_list("", [dracula.ZERO, dracula.TWO]) +cmap2 = matplotlib.colors.LinearSegmentedColormap.from_list("", [dracula.FOUR, dracula.SIX]) + +#cms = [cm.inferno, cm.viridis] +#cms = [cm.inferno, cm.cool] +#cms = [cm.copper, cm.summer] +cms = [cmap1, cmap2] + + +class tag: + def __init__(self): + self.tagType = TAG_TYPE.UNDEFINED + self.name = "" + + + +def getTags(csvFilePaths, tagWhitelist): + tags = [] + + printlogwarning = True; + with open(tmpFile, "w") as aggregateFile: + for p in csvFilePaths: + with open(p, "r") as csvIn: + lines = csvIn.readlines() + + for line in lines: + if line[:len(sentinalStr)] == sentinalStr: + tokens = line.split(',') + + if len(tokens) < 6 or len(tokens) > 8: + if printlogwarning: + print("WARNING - Malformed log line") + print(line) + printlogwarning = False + continue + + # strip trailing \n, add right amount of commas, add \n back + line = line.rstrip() + (','*(maxColumns - len(tokens))) + '\n' + aggregateFile.write(line) + + if tokens[3] == "error": + print("WARNING - OVERFLOW ERROR IN LOG FILE") + print(line) + continue + + whitelisted = tagWhitelist == None + if not whitelisted: + for whitelistedTag in tagWhitelist: + whitelisted = whitelisted or re.match('^'+whitelistedTag+'$', tokens[3]) + if not whitelisted: + continue + + newTag = True + for t in tags: + if t.name == tokens[3]: # must be exact match to be old tag + newTag = False + break + + if newTag and whitelisted: + print("found new tag: " + tokens[3]) + t = tag() + if tokens[1] == "xy": + t.tagType = TAG_TYPE.XY + if tokens[1] == "box": + t.tagType = TAG_TYPE.BOX + elif tokens[1] == "histogram": + t.tagType = TAG_TYPE.HISTOGRAM + elif tokens[1] == "cdf": + t.tagType = TAG_TYPE.CDF + elif tokens[1] == "bar": + t.tagType = TAG_TYPE.BAR + elif tokens[1] == "grouped_bar": + t.tagType = TAG_TYPE.GROUPED_BAR + elif tokens[1] == "barbox": + t.tagType = TAG_TYPE.BARBOX + elif tokens[1] == "3dbar": + t.tagType = TAG_TYPE.THREEDBAR + elif tokens[1] == "3dsurf": + t.tagType = TAG_TYPE.THREEDSURF + elif tokens[1] == "3dprojectx": + t.tagType = TAG_TYPE.THREEDPROJECTX + elif tokens[1] == "3dprojecty": + t.tagType = TAG_TYPE.THREEDPROJECTY + elif tokens[1] == "3dunroll": + t.tagType = TAG_TYPE.THREEDUNROLL + t.name = tokens[3] + tags.append(t) + + # make sure tags in same order as tagWhitelist + if tagWhitelist != None: + newTags = [] + for tname in tagWhitelist: + for t in tags: + if re.match('^'+tname+'$', t.name): + newTags.append(t) + continue + return newTags + else: + return tags + + + +def plot(csvFilePaths, options, tags): + data = np.genfromtxt(tmpFile, dtype=None, invalid_raise = False, + delimiter=',', encoding=None, filling_values="0", + names="sentinal, type, func, tag, x, y, z") + + tagIndex = 0; + tagLabels = [] + lastPlotBottom = [] + legendArtistProxyShapes=[] + skipLegend = options.hide_legend + + for t in tags: + tagLabels.append(t.name) + + plt.figure(0, figsize=(options.fig_w, options.fig_h), dpi=800) + + for t in tags: + print("plotting tag: " + t.name) + + if t.tagType == TAG_TYPE.XY: + exes = [] + whys = [] + for row in data: + if row[3] == t.name and (options.xfilter == None or row[4] in options.xfilter): + exes.append(float(row[4])); + whys.append(float(row[5])); + # decorate, sort, then undecorate + exes, whys = (np.array(t) for t in zip(*sorted(zip(exes, whys)))) + # average duplicate values + dedup_exes = np.unique(exes) + dedup_whys = np.empty(dedup_exes.shape) + for i, x in enumerate(dedup_exes): + dedup_whys[i] = np.mean(whys[exes == x]) # / 1e9 + + if options.title == "Network IO per Iteration": + dedup_whys /= 1e9 + + if t.name == 'emp_float_lm_time_vs_points_per_loc_itr_latency': + edgecolor=dracula.COLORS[tagIndex+options.color_offset] + import colorsys + h, l, s = colorsys.rgb_to_hls(*matplotlib.colors.to_rgb(edgecolor)) + edgehighlight = colorsys.hls_to_rgb(h, min(1, l * 0.5), s = s) + + plt.plot(dedup_exes, dedup_whys, + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None), + linestyle = '--', lw=2, path_effects=[pe.Stroke(linewidth=3, foreground=edgehighlight), pe.Normal()]) + else: + plt.plot(dedup_exes, dedup_whys, + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + + elif t.tagType == TAG_TYPE.HISTOGRAM: + print("plotting histogram tag: " + t.name) + print("TODO!") + + elif t.tagType == TAG_TYPE.BAR or t.tagType == TAG_TYPE.GROUPED_BAR or t.tagType == TAG_TYPE.BARBOX or t.tagType == TAG_TYPE.BOX: + labels = [] # vector of x-axis label strings + dataByLabel = [] # vector of all y values, indexed by label + offsetDataByLabel = [] # dataByLable, stacked + whys = [] # average y value indexed by label + width = 0.5 # width of a group of bars for grouped bar graph + + # collect labels for all tags + for row in data: + if len(row) < 6: + print("bar|grouped_bar|box|barbox data bad format") + exit() + for tagLabel in tagLabels: + if row[3] == tagLabel: + if row[4] not in labels and (options.xfilter == None or row[4] in options.xfilter): + labels.append(row[4]) + dataByLabel.append([]) + offsetDataByLabel.append([]) + + if len(lastPlotBottom) == 0: + lastPlotBottom = [0]*len(labels) + + # collect data for this tag + for row in data: + if row[3] == t.name and (options.xfilter == None or row[4] in options.xfilter): + try: + i = labels.index(row[4]) + dataByLabel[i].append(float(row[5])) + offsetDataByLabel[i].append(float(row[5]) + lastPlotBottom[i]) + except: + continue + for row in dataByLabel: + if len(row) != 0: + whys.append(np.average(row)) + else: + whys.append(0) + + if t.tagType == TAG_TYPE.BOX: + # no label on box plots + plt.boxplot(dataByLabel, positions=range(len(labels)))#, showmeans=True) + + if t.tagType == TAG_TYPE.BAR: + if options.horizontal: + plt.barh(range(len(whys)), whys, bottom=lastPlotBottom, + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + else: + plt.bar(range(len(whys)), whys, bottom=lastPlotBottom, + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + if len(tags) > 1 and not skipLegend: + plt.legend() # bar always gets legend, must be after plotting + lastPlotBottom = list( map(add, lastPlotBottom, whys) )# for stacking using average + + if t.tagType == TAG_TYPE.GROUPED_BAR: + offset=len(tags)/2*width/5 + if options.horizontal: + rects = plt.barh(np.arange(len(whys)) - offset + (width*tagIndex/len(tags)), whys, width/len(tags), + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None), + edgecolor='black', linewidth=1, hatch=options.hatching) + else: + # special case for snail paper + if t.name == "EMP_MUL": + rects = plt.bar(np.arange(len(whys)) - offset + (width*tagIndex/len(tags)), whys, width/len(tags), + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color="#2ca02c", + edgecolor='black', linewidth=1, hatch=options.hatching) + else: + rects = plt.bar(np.arange(len(whys)) - offset + (width*tagIndex/len(tags)), whys, width/len(tags), + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None), + edgecolor='black', linewidth=1, hatch=options.hatching) + if len(tags) > 1 and not skipLegend: + plt.legend() # bar always gets legend, must be after plotting + lastPlotBottom = list( map(add, lastPlotBottom, whys) )# for stacking using average + + # label rectangles + if options.bar_label_dict and t.name in options.bar_label_dict: + if len(options.bar_label_dict[t.name]) != len(whys): + print('Error - wrong number of bar labels - %d (should be %d)\n\n\n' % (len(options.bar_label_dict[t.name]), len(whys)) ) + return + rectnum=0 + for rect in rects: + height = rect.get_height() + plt.text(rect.get_x() + rect.get_width()/2., 1.05*height, options.bar_label_dict[t.name][rectnum], + ha='center', va='bottom', + bbox=dict(facecolor='white', edgecolor='black', boxstyle='round,pad=0.2')) + rectnum=rectnum+1 + + + if t.tagType == TAG_TYPE.BARBOX: + # stack boxplot data - shift upwards by last avg + meanpointprops = dict(marker='o', markeredgecolor='black', markerfacecolor='black') + r = plt.boxplot(offsetDataByLabel, positions=range(len(labels)), showmeans=True, meanprops=meanpointprops) + + # need to parse returned means because they dont include outliers + whysExcludingOutliers = [] + for i in range(len(labels)): + whysExcludingOutliers.append(r['means'][i].get_ydata()[0] - lastPlotBottom[i]) + + plt.bar(range(len(whysExcludingOutliers)), whysExcludingOutliers, width, bottom=lastPlotBottom, + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None), + edgecolor='black', linewidth=1, hatch=options.hatching) + lastPlotBottom = list( map(add, lastPlotBottom, whys) )# for stacking using average + if len(tags) > 1 and not skipLegend: + plt.legend() + + if options.print_raw: + print(labels) + np.set_printoptions(threshold=sys.maxsize) + print(whys) + + if options.rotate_x_labels: + plt.xticks(range(len(labels)), labels, rotation=45) + else: + if options.horizontal: + plt.yticks(range(len(labels)), labels) + else: + plt.xticks(range(len(labels)), labels) + + + elif t.tagType == TAG_TYPE.THREEDBAR or t.tagType == TAG_TYPE.THREEDSURF or t.tagType == TAG_TYPE.THREEDPROJECTX or t.tagType == TAG_TYPE.THREEDPROJECTY or t.tagType == TAG_TYPE.THREEDUNROLL: + if t.tagType == TAG_TYPE.THREEDBAR or t.tagType == TAG_TYPE.THREEDSURF: + ax = plt.gca(projection = '3d') + + #plt.rcParams['legend.fontsize'] = 10 + xlabels = [] # vector of x-axis label strings + ylabels = [] # vector of y-axis label strings + + for row in data: + if len(row) < 7: + print("3d bar data bad format") + exit() + if row[3] == t.name: + if (row[4] not in xlabels and (options.xfilter == None or row[4] in options.xfilter)): + xlabels.append(row[4]) + if (row[5] not in ylabels): + ylabels.append(row[5]) + + if len(lastPlotBottom) == 0: + lastPlotBottom = [0]*len(xlabels)*len(ylabels) + elif len(lastPlotBottom) != len(xlabels)*len(ylabels): + print("ERROR - tags have different number of labels") + print(len(lastPlotBottom)) + print(xlabels) + print(ylabels) + exit() + + zDataByLabel = [[[] for x in range(len(ylabels))] for y in range(len(xlabels))] # vector of all z values, indexed by x/y label + zees = [[0 for x in range(len(ylabels))] for y in range(len(xlabels))] # average z value indexed by x/y labels + + for row in data: + if row[3] == t.name and (options.xfilter == None or row[4] in options.xfilter): + xi = xlabels.index(row[4]) + yi = ylabels.index(row[5]) + zDataByLabel[xi][yi].append(float(row[6])) + + for i in range(len(xlabels)): + for j in range(len(ylabels)): + zRow = zDataByLabel[i][j] + if len(zRow) != 0: + zees[i][j] = np.average(zRow) + + x = [] + y = [] + dz = [] + for i in range(len(xlabels)): + for j in range(len(ylabels)): + x.append(i); + y.append(j); + dz.append(zees[i][j]); + dx = np.ones(len(x)) + dy = np.ones(len(y)) + + if options.print_raw: + print(xlabels) + print(ylabels) + np.set_printoptions(threshold=sys.maxsize) + print(np.resize(dz, [len(xlabels), len(ylabels)])) + + ### Note - this should work: + ### surf = ax.bar3d(x, y, lastPlotBottom, dx, dy, dz, label=t.name) + ### But it looks terrible! + ### Plotting the 0 height bars breaks the rendering. + ### Now lets remove them carefully so we don't break lastPlotBottom + + filteredx = [] + filteredy = [] + filteredlpb = [] + filtereddz = [] + for i in range(len(xlabels)): + for j in range(len(ylabels)): + if zees[i][j] != 0: + filteredx.append(i) + filteredy.append(j) + filtereddz.append(zees[i][j]) + filteredlpb.append(lastPlotBottom[i*len(ylabels)+j]) + filtereddx = np.ones(len(filtereddz)) + filtereddy = np.ones(len(filtereddz)) + + if t.tagType == TAG_TYPE.THREEDBAR: + surf = ax.bar3d(filteredx, filteredy, filteredlpb, filtereddx, filtereddy, filtereddz, + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + #surf = ax.bar3d(x, y, lastPlotBottom, dx, dy, dz, label=t.name) + + # Default legend looks terrible + #surf._facecolors2d=surf._facecolors3d + #surf._edgecolors2d=surf._edgecolors3d + #plt.legend() + # Use this instead: https://stackoverflow.com/questions/5803015/how-to-create-a-legend-for-3d-bar-in-matplotlib + legendArtistProxyShapes.append(plt.Rectangle((0,0), 1, 1, + fc=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else plt.get_cmap("tab10")(tagIndex)))) + ax.set_zlabel(options.zlabel) + if options.azimuth: + ax.azim = options.azimuth + if options.elevation: + ax.elev = options.elevation + + plt.xticks(range(len(xlabels)), xlabels)#, rotation=45) + plt.yticks(range(len(ylabels)), ylabels)#, rotation=45) + + elif t.tagType == TAG_TYPE.THREEDSURF: + surf = ax.plot_trisurf(x, y, dz, cmap=cms[tagIndex+options.color_offset], linewidth=0, + antialiased=False, alpha=(tagIndex+1)/len(tags)) + clb = plt.colorbar(surf, shrink=0.5, aspect=14, + pad=(.16 if (tagIndex == len(tags)-1) else .04)) + clb.ax.set_title(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name) + # set_xlabel('hi') to put at bottom + ax.set_zlabel(options.zlabel) + plt.xticks(range(len(xlabels)), xlabels)#, rotation=45) + plt.yticks(range(len(ylabels)), ylabels)#, rotation=45) + plt.tight_layout()# dont move + skipLegend = True + + elif t.tagType == TAG_TYPE.THREEDPROJECTX: + if options.projection == None: + print("need projection command line option") + return + else: + sliced = [] + slicedLPB = [] + for i in range(len(xlabels)): + sliced.append(zees[i][options.projection]) + slicedLPB.append(lastPlotBottom[i * len(ylabels) + options.projection]) + + # Plot a line + plt.plot([float(i) for i in xlabels], sliced, + label=options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name, + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + + # Stacked Bars + #plt.bar([i for i in xlabels], sliced, bottom=slicedLPB, + # label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + # color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + + plt.xticks([float(i) for i in xlabels], xlabels)#, rotation=45) + + elif t.tagType == TAG_TYPE.THREEDPROJECTY: + if options.projection == None: + print("need projection command line option") + return + else: + # Plot a line + plt.plot([float(i) for i in ylabels], zees[options.projection], + label=options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name, + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + + # Stacked Bars + #plt.bar([i for i in xlabels], zees[options.projection], bottom=lastPlotBottom[options.projection*len(ylabels) : options.projection*len(ylabels)+len(ylabels)], # THIS MAY NOT BE RIGHT + # label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + # color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + + plt.xticks([float(i) for i in ylabels], ylabels)#, rotation=45) + + elif t.tagType == TAG_TYPE.THREEDUNROLL: + width = .3 # width of bar + numX = len(xlabels) + + + # insert spaces + spacedLPB = lastPlotBottom.copy() + spacedDz = dz.copy() + for i in range(len(ylabels)): + if i != 0 and i != len(ylabels): + spacedDz.insert((i*len(ylabels)) + i - 1, 0) + spacedLPB.insert((i * len(ylabels)) + i - 1, 0) + + plt.bar(range(len(spacedDz)), spacedDz, bottom=spacedLPB, + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + if len(tags) > 1 and not skipLegend: + plt.legend() # bar always gets legend, must be after plotting + + xylabels = [] + for i in range(len(xlabels)): + if i != 0: + xylabels.append('') + for j in range(len(ylabels)): + xylabels.append(str(xlabels[i]) + ' - ' + str(int(ylabels[j]))) + plt.xticks(range(len(xylabels)), xylabels, rotation=45) + + lastPlotBottom = list( map(add, lastPlotBottom, dz) )# for stacking using average + + + tagIndex += 1; + + if len(legendArtistProxyShapes) > 1 and not skipLegend: # 3dbar plot = special legend + plt.legend(legendArtistProxyShapes, + options.custom_legend_labels if options.custom_legend_labels != None else tagLabels) + elif options.custom_legend_labels != None and len(options.custom_legend_labels) > 1 and not skipLegend: + if options.horizontal: + handles, labels = plt.gca().get_legend_handles_labels() + plt.legend(reversed(handles), reversed(labels)) + else: + plt.legend(loc = 'upper right') + + if options.horizontal: + plt.ylabel(options.xlabel) + plt.xlabel(options.ylabel) + else: + plt.xlabel(options.xlabel) + plt.ylabel(options.ylabel) + plt.title(options.title) + + + + + + + myax = plt.gca() + #mysubax = zoomed_inset_axes(myax, 3, loc=2, bbox_to_anchor=(60,100,10,10), borderpad=2) # zoom = 6 + mysubax = zoomed_inset_axes(myax, 12, bbox_to_anchor=(70,160), loc=2, borderpad=0) # zoom = 6 + if options.title == "Network IO per Iteration": + mysubax = zoomed_inset_axes(myax, 12, bbox_to_anchor=(64,138), loc=2, borderpad=0) # zoom = 6 + + for tagIndex, t in enumerate(tags): + print("plotting tag: " + t.name) + + if t.tagType == TAG_TYPE.XY: + exes = [] + whys = [] + for row in data: + if row[3] == t.name and (options.xfilter == None or row[4] in options.xfilter): + exes.append(float(row[4])); + whys.append(float(row[5])); + exes, whys = (np.array(t) for t in zip(*sorted(zip(exes, whys)))) + dedup_exes = np.unique(exes) + dedup_whys = np.empty(dedup_exes.shape) + for i, x in enumerate(dedup_exes): + dedup_whys[i] = np.mean(whys[exes == x]) # / 1e9 + if options.title == "Network IO per Iteration": + dedup_whys /= 1e9 + + if t.name == 'emp_float_lm_time_vs_points_per_loc_itr_latency': + edgecolor=dracula.COLORS[tagIndex+options.color_offset] + import colorsys + h, l, s = colorsys.rgb_to_hls(*matplotlib.colors.to_rgb(edgecolor)) + edgehighlight = colorsys.hls_to_rgb(h, min(1, l * 0.5), s = s) + + plt.plot(dedup_exes, dedup_whys, + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None), + linestyle = '--', lw=2, path_effects=[pe.Stroke(linewidth=3, foreground=edgehighlight), pe.Normal()]) + else: + plt.plot(dedup_exes, dedup_whys, + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + + mysubax.set_xlim(6, 12) + mysubax.set_xticks([6, 9, 12]) + plt.xticks(visible=True) + plt.yticks(visible=True) + if options.title == "Network IO per Iteration": + mysubax.set_ylim(0, .000003) + #mysubax.set_yticks([1, 2]) + else: + mysubax.set_ylim(1, 2) + mysubax.set_yticks([1, 2]) + #mark_inset(myax, mysubax, loc1=2, loc2=3, fc="none", ec="0.5") + mark_inset(myax, mysubax, loc1=2, loc2=3, fc="none", ec="0.7") + + from matplotlib.ticker import StrMethodFormatter + plt.gca().yaxis.set_major_formatter(StrMethodFormatter('{x:,.0f}')) # No decimal places + + + + + + + + + plt.tight_layout() + if options.show: + plt.show() + if options.log_scale: + if options.horizontal: + plt.gca().set_xscale('log') + else: + plt.gca().set_yscale('log') + plt.savefig(options.graphpath) + print("saved figure to " + options.graphpath) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="JLog plotter") + parser.add_argument('--csvlog', + required=True, + nargs="+", + help=("Path to log file.")) + parser.add_argument('--graphpath', + required=True, + help=("Path to store output graphs.")) + parser.add_argument('--title', + required=False, + default="", + help=("Graph Title.")) + parser.add_argument('--xlabel', + required=False, + default="", + help=("Label for plot x axis.")) + parser.add_argument('--ylabel', + required=False, + default="", + help=("Label for plot y axis.")) + parser.add_argument('--zlabel', + required=False, + default="", + help=("Label for plot z axis.")) + parser.add_argument('--only-tags', + required=False, + nargs="+", + help=("Only plot specific (space seperated) tags in csv log files. Default plots all tags.")) + parser.add_argument('--custom-legend-labels', + required=False, + nargs="+", + help=("Attach a custom legend label to each tag's plot.")) + parser.add_argument('--color-theme', + required=False, + help=("Use a custom color theme - dracula suported")) + parser.add_argument('--color-offset', + required=False, + type=int, + default=0, + help=("Offset starting color in color theme")) + parser.add_argument('--projection', + required=False, + type=int, + help=("Index of x/y projection for 3d to 2d projections")) + parser.add_argument('--azimuth', + required=False, + type=int, + help=("Azimuth sets view for 3D plot")) + parser.add_argument('--elevation', + required=False, + type=int, + help=("Elevation sets view for 3D plot")) + parser.add_argument('--fig-w', + required=False, + type=float, + default=5, + help=("Override figure width")) + parser.add_argument('--fig-h', + required=False, + type=float, + default=4, + help=("Override figure height")) + parser.add_argument('--xfilter', + required=False, + nargs="+", + help=("Only use data which match one of these x values.")) + parser.add_argument('--rotate_x_labels', + required=False, + action="store_true", + help=("Rotate x axis labels.")) + parser.add_argument('--hide-legend', + required=False, + action="store_true", + help=("Hide legend.")) + parser.add_argument('--bar-label-dict', + required=False, + type=json.loads, + help=("Add labels to bars e.g. '{\"tagName\": [\"l1\", \"l2\", \"l3\"]}' ")) + parser.add_argument('--hatching', + required=False, + help=("Add hatching to bars e.g. \"||||\"")) + parser.add_argument('--show', + required=False, + action="store_true", + help=("Show plots as they are generated.")) + parser.add_argument('--print-raw', + required=False, + action="store_true", + help=("Print raw plotted data to command line.")) + parser.add_argument('--log-scale', + required=False, + action="store_true", + help=("Log scale.")) + parser.add_argument('--horizontal', + required=False, + action="store_true", + help=("Plot bar graphs horizontally.")) + options = parser.parse_args() + + try: + csvFilePaths = options.csvlog + for p in csvFilePaths: + with open(p, "r") as f: + print("found file " + p) + except FileNotFoundError: + print("file not found " + p) + exit() + + if options.only_tags != None: + print("Only plotting specific tags: ") + print(options.only_tags) + + tags = getTags(csvFilePaths, options.only_tags) + + if tags == None or len(tags) == 0: + print("ERROR - No tags found - Nothing to plot") + exit() + + if options.custom_legend_labels != None: + if len(options.custom_legend_labels) == len(tags): + print("Custom Tag Labels Enabled") + else: + print("Wrong Number of Custom Tag Labels") + print(options.custom_legend_labels) + print(tags) + exit() + + plot(csvFilePaths, options, tags) + + if os.path.exists(tmpFile): + os.remove(tmpFile) diff --git a/scripts/loopleak_vs_dataobl_plot.sh b/scripts/loopleak_vs_dataobl_plot.sh new file mode 100755 index 0000000..5b90576 --- /dev/null +++ b/scripts/loopleak_vs_dataobl_plot.sh @@ -0,0 +1,43 @@ +#!/bin/bash +scriptpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +logdir=$scriptpath/../results/ +plotdir=$scriptpath/../plots/ + +#SHOW="--show" +SHOW="" + +# relies on: +# emp_float_benchmark_run.sh +# emp_float_benchmark_dataobl_run.sh + +sed -i 's/grouped_bar,operator(),emp_float_gn_time_vs_point/xy,operator(),emp_float_gn_time_vs_point/g' $scriptpath/../results/gn_emp_float_eth3d_bench_short.log +sed -i 's/grouped_bar,operator(),emp_float_lm_time_vs_point/xy,operator(),emp_float_lm_time_vs_point/g' $scriptpath/../results/lm_emp_float_eth3d_bench_short.log +sed -i 's/grouped_bar,operator(),emp_float_gn_time_vs_point_dataobl/xy,operator(),emp_float_gn_time_vs_point_dataobl/g' $scriptpath/../results/gn_emp_float_eth3d_bench_dataobl_short.log +sed -i 's/grouped_bar,operator(),emp_float_lm_time_vs_point_dataobl/xy,operator(),emp_float_lm_time_vs_point_dataobl/g' $scriptpath/../results/lm_emp_float_eth3d_bench_dataobl_short.log + +python3 $scriptpath/../scripts/plotter-do-vs-sil.py \ + --csvlog \ + "$logdir/gn_emp_float_eth3d_bench_short.log" \ + "$logdir/lm_emp_float_eth3d_bench_short.log" \ + "$logdir/gn_emp_float_eth3d_bench_dataobl_short.log" \ + "$logdir/lm_emp_float_eth3d_bench_dataobl_short.log" \ + --graphpath "$plotdir/loopleak_vs_dataobl.pdf" \ + --title "Comparing Data Oblivious and +Single Iteration Localization Runtime" \ + --only-tags \ + "emp_float_gn_time_vs_points_dataobl" \ + "emp_float_lm_time_vs_points_dataobl" \ + "emp_float_gn_time_vs_points" \ + "emp_float_lm_time_vs_points" \ + --custom-legend-labels \ + "GN DO" \ + "LM DO" \ + "GN SIL" \ + "LM SIL" \ + --xlabel "Number of Features" \ + --ylabel "Runtime (s)" \ + --fig-w 4 \ + --fig-h 3 \ + --log-scale \ + --color-theme "dracula" \ + $SHOW diff --git a/scripts/mult_add_fixed_float_time_plot.sh b/scripts/mult_add_fixed_float_time_plot.sh new file mode 100755 index 0000000..c2addd5 --- /dev/null +++ b/scripts/mult_add_fixed_float_time_plot.sh @@ -0,0 +1,69 @@ +#!/bin/bash +scriptpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +logdir=$scriptpath/../results/ +plotdir=$scriptpath/../plots/ + +#SHOW="--show" +SHOW="" + + +# To plot the add multiply together +python3 $scriptpath/../scripts/plotter-mult-add.py \ + --csvlog \ + "$logdir/emp_float_vs_fixed_benchmark_add.log" \ + "$logdir/emp_float_vs_fixed_benchmark_mul.log" \ + --graphpath "$plotdir/emp_float_vs_fixed_benchmark_add_mul.pdf" \ + --title "Comparing Arithmetic Operations +Across Data Representations" \ + --only-tags \ + "EMP_ADD" \ + "EMP_MUL" \ + --custom-legend-labels \ + "Add" \ + "Multiply" \ + --xlabel "" \ + --ylabel "" \ + --fig-w 4 \ + --fig-h 3 \ + --rotate_x_labels \ + --color-theme "dracula" \ + $SHOW + #--log-scale \ + #--only-tags "opencv_error_vs_numpts" \ + #--custom-legend-labels "opencv (float)" \ + # "float" \ + # "fixed (64bit)" \ + + + +# To plot the add multiply graphs separately +#python3 $scriptpath/../scripts/plotter.py \ +# --csvlog \ +# "$logdir/emp_float_vs_fixed_benchmark_add.log" \ +# --graphpath "$plotdir/emp_float_vs_fixed_benchmark_add.pdf" \ +# --title "Float vs Fixed Operation Speed (10k Operations)" \ +# --xlabel "" \ +# --ylabel "Time (s)" \ +# --color-theme "dracula" \ +# --fig-w 5 \ +# --fig-h 2.5 \ +# --rotate_x_labels \ +# $SHOW +# #--only-tags "opencv_error_vs_numpts" \ +# #--custom-legend-labels "opencv (float)" \ +# # "float" \ +# # "fixed (64bit)" \ +# +#python3 $scriptpath/../scripts/plotter.py \ +# --csvlog \ +# "$logdir/emp_float_vs_fixed_benchmark_mul.log" \ +# --graphpath "$plotdir/emp_float_vs_fixed_benchmark_mul.pdf" \ +# --xlabel "" \ +# --ylabel "Time (s)" \ +# --color-theme "dracula" \ +# --fig-w 5 \ +# --fig-h 2.5 \ +# --rotate_x_labels \ +# --color-offset 1 \ +# $SHOW +# #--title "Float vs Fixed Operation Speed" \ diff --git a/scripts/mult_add_fixed_float_time_run.sh b/scripts/mult_add_fixed_float_time_run.sh new file mode 100755 index 0000000..37b0fde --- /dev/null +++ b/scripts/mult_add_fixed_float_time_run.sh @@ -0,0 +1,10 @@ +#!/bin/bash +scriptpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +killall emp_float_vs_fixed_benchmark_* +$scriptpath/../build/bin/emp_float_vs_fixed_benchmark_add 0 > $scriptpath/../results/emp_float_vs_fixed_benchmark_add.log & +$scriptpath/../build/bin/emp_float_vs_fixed_benchmark_add 1 + +killall emp_float_vs_fixed_benchmark_* +$scriptpath/../build/bin/emp_float_vs_fixed_benchmark_mul 0 > $scriptpath/../results/emp_float_vs_fixed_benchmark_mul.log & +$scriptpath/../build/bin/emp_float_vs_fixed_benchmark_mul 1 diff --git a/scripts/netio_plot.sh b/scripts/netio_plot.sh new file mode 100755 index 0000000..3d50b4a --- /dev/null +++ b/scripts/netio_plot.sh @@ -0,0 +1,36 @@ +#!/bin/bash +scriptpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +logdir=$scriptpath/../results/ +plotdir=$scriptpath/../plots/ + +#SHOW="--show" +SHOW="" + +# relies on: +# emp_float_benchmark_run.sh + +# NOTE: For GB you need to modify plotter script + +python3 $scriptpath/../scripts/plotter-netio.py \ + --csvlog \ + "$logdir/gn_emp_float_eth3d_bench_long.log" \ + "$logdir/lm_emp_float_eth3d_bench_long.log" \ + --graphpath "$plotdir/netio.pdf" \ + --title "Comparing Network IO +at Large Input Sizes" \ + --only-tags \ + "gn_bytes_rx_per_itr" \ + "lm_bytes_rx_per_itr" \ + "gn_bytes_rx_per_itr_per_feat" \ + "lm_bytes_rx_per_itr_per_feat" \ + --custom-legend-labels \ + "GN" \ + "LM" \ + "GN per Feature" \ + "LM per Feature" \ + --xlabel "Number of Features" \ + --ylabel "Bytes per Iteration" \ + --fig-w 4 \ + --fig-h 3 \ + --color-theme 'dracula' \ + $SHOW diff --git a/scripts/network_setup.sh b/scripts/network_setup.sh new file mode 100644 index 0000000..0a90ab4 --- /dev/null +++ b/scripts/network_setup.sh @@ -0,0 +1,10 @@ +export DEV="lo" +export SPEED="2.5Gbit" +export BURST="100000" +LAT="${LAT:="0msec"}" + +echo "Setting ${DEV} to ${SPEED} with latency ${LAT}" + +sudo tc qdisc del dev $DEV root &>/dev/null +sudo tc qdisc add dev $DEV root handle 1: tbf rate ${SPEED} burst ${BURST} limit ${BURST} +sudo tc qdisc add dev $DEV parent 1:1 handle 10: netem delay ${LAT} diff --git a/scripts/network_teardown.sh b/scripts/network_teardown.sh new file mode 100644 index 0000000..4698753 --- /dev/null +++ b/scripts/network_teardown.sh @@ -0,0 +1,6 @@ +sudo tc qdisc del dev $DEV root + +unset DEV +unset SPEED +unset BURST +unset LAT \ No newline at end of file diff --git a/scripts/num_arith_ops_plot.sh b/scripts/num_arith_ops_plot.sh new file mode 100755 index 0000000..fe58f3c --- /dev/null +++ b/scripts/num_arith_ops_plot.sh @@ -0,0 +1,39 @@ +#!/bin/bash +scriptpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +logdir=$scriptpath/../results/ +plotdir=$scriptpath/../plots/ + +#SHOW="--show" +SHOW="" + +python3 $scriptpath/../scripts/plotter.py \ + --csvlog \ + `#"$logdir/gn_emp_float_eth3d_bench_short.log"` \ + "$logdir/lm_emp_float_eth3d_bench_short.log" \ + --graphpath "$plotdir/emp_arith_ops.pdf" \ + --title "Counting Arithmetic Operations" \ + --only-tags \ + `#"gn_additions"` \ + `#"gn_subtractions"` \ + `#"gn_multiplications"` \ + `#"gn_divisions"` \ + "lm_additions" \ + "lm_subtractions" \ + "lm_multiplications" \ + "lm_divisions" \ + --custom-legend-labels \ + "Add" \ + "Subract" \ + "Multiply" \ + "Divide" \ + --xlabel "Number of Features" \ + --ylabel "Number of Operations" \ + --fig-w 4 \ + --fig-h 4 \ + --horizontal \ + --color-theme "dracula" \ + $SHOW + #--only-tags "opencv_error_vs_numpts" \ + #--custom-legend-labels "opencv (float)" \ + # "float" \ + # "fixed (64bit)" \ diff --git a/scripts/plotter-do-vs-sil.py b/scripts/plotter-do-vs-sil.py new file mode 100644 index 0000000..9a548c6 --- /dev/null +++ b/scripts/plotter-do-vs-sil.py @@ -0,0 +1,716 @@ +import matplotlib.pyplot as plt +from mpl_toolkits.mplot3d import Axes3D +from matplotlib import cm +import matplotlib.colors +import numpy as np +from operator import add +import os +import argparse +import enum +import sys +import json +import re + +sentinalStr = "SeNtInAl" +tmpFile = "plotlines.tmp" +maxColumns = 7 + +# Plotter cheat sheet +# +# XY +# SeNtInAl, tag_type, user_defined, tag_name, x, y +# +# +# BAR | GROUPED_BAR | BARBOX | BOX +# SeNtInAl, tag_type, user_defined, tag_name, x_label, y +# +# tag_names - plotting multiple will create segmented bars with +# a legend. Legend can be overridden with --custom-legend-labels +# x_label - string used to denote X +# y - float value +# +# THREEDBAR +# SeNtInAl, tag_type, z, tag_name, x_label, y_label +# tag_names - plotting multiple will create segmented bars with +# a legend. Legend can be overridden with --custom-legend-labels +# x_label - string used to denote X +# y_label - string used to denote Y +# z - float + + + +class TAG_TYPE(enum.Enum): + UNDEFINED = 0 + XY = 1 + HISTOGRAM = 2 + CDF = 3 + BAR = 4 + BOX = 5 + BARBOX = 6 + GROUPED_BAR = 7 + THREEDBAR = 8 + THREEDSURF = 9 + THREEDPROJECTX = 10 + THREEDPROJECTY = 11 + THREEDUNROLL = 12 + +#class dracula(): +# CYAN = '#8be9fd' +# GREEN = '#50fa7b' +# ORANGE = '#ffb86c' +# PINK = '#ff79c6' +# PURPLE = '#bd93f9' +# RED = '#ff5555' +# YELLOW = '#f1fa8c' +# COLORS = [CYAN, ORANGE, GREEN, PINK, PURPLE, YELLOW, RED] + +class dracula(): + ZERO = '#D291AC' + ONE = '#779E44' + TWO = '#F6B7AA' + THREE = '#367E51' + FOUR = '#5F1333' + FIVE = '#426615' + SIX = '#AD5B4A' + #COLORS = [ZERO, ONE, TWO, THREE, FOUR, FIVE, SIX, ZERO, ONE, TWO, THREE, FOUR] + COLORS = [plt.cm.Pastel1(i) for i in range(9)] + + + +#cmap1 = matplotlib.colors.LinearSegmentedColormap.from_list("", [dracula.PINK, dracula.PURPLE]) +#cmap2 = matplotlib.colors.LinearSegmentedColormap.from_list("", [dracula.CYAN, dracula.ORANGE]) + +cmap1 = matplotlib.colors.LinearSegmentedColormap.from_list("", [dracula.ZERO, dracula.TWO]) +cmap2 = matplotlib.colors.LinearSegmentedColormap.from_list("", [dracula.FOUR, dracula.SIX]) + +#cms = [cm.inferno, cm.viridis] +#cms = [cm.inferno, cm.cool] +#cms = [cm.copper, cm.summer] +cms = [cmap1, cmap2] + + +class tag: + def __init__(self): + self.tagType = TAG_TYPE.UNDEFINED + self.name = "" + + + +def getTags(csvFilePaths, tagWhitelist): + tags = [] + + printlogwarning = True; + with open(tmpFile, "w") as aggregateFile: + for p in csvFilePaths: + with open(p, "r") as csvIn: + lines = csvIn.readlines() + + for line in lines: + if line[:len(sentinalStr)] == sentinalStr: + tokens = line.split(',') + + if len(tokens) < 6 or len(tokens) > 8: + if printlogwarning: + print("WARNING - Malformed log line") + print(line) + printlogwarning = False + continue + + # strip trailing \n, add right amount of commas, add \n back + line = line.rstrip() + (','*(maxColumns - len(tokens))) + '\n' + aggregateFile.write(line) + + if tokens[3] == "error": + print("WARNING - OVERFLOW ERROR IN LOG FILE") + print(line) + continue + + whitelisted = tagWhitelist == None + if not whitelisted: + for whitelistedTag in tagWhitelist: + whitelisted = whitelisted or re.match('^'+whitelistedTag+'$', tokens[3]) + if not whitelisted: + continue + + newTag = True + for t in tags: + if t.name == tokens[3]: # must be exact match to be old tag + newTag = False + break + + if newTag and whitelisted: + print("found new tag: " + tokens[3]) + t = tag() + if tokens[1] == "xy": + t.tagType = TAG_TYPE.XY + if tokens[1] == "box": + t.tagType = TAG_TYPE.BOX + elif tokens[1] == "histogram": + t.tagType = TAG_TYPE.HISTOGRAM + elif tokens[1] == "cdf": + t.tagType = TAG_TYPE.CDF + elif tokens[1] == "bar": + t.tagType = TAG_TYPE.BAR + elif tokens[1] == "grouped_bar": + t.tagType = TAG_TYPE.GROUPED_BAR + elif tokens[1] == "barbox": + t.tagType = TAG_TYPE.BARBOX + elif tokens[1] == "3dbar": + t.tagType = TAG_TYPE.THREEDBAR + elif tokens[1] == "3dsurf": + t.tagType = TAG_TYPE.THREEDSURF + elif tokens[1] == "3dprojectx": + t.tagType = TAG_TYPE.THREEDPROJECTX + elif tokens[1] == "3dprojecty": + t.tagType = TAG_TYPE.THREEDPROJECTY + elif tokens[1] == "3dunroll": + t.tagType = TAG_TYPE.THREEDUNROLL + t.name = tokens[3] + tags.append(t) + + # make sure tags in same order as tagWhitelist + if tagWhitelist != None: + newTags = [] + for tname in tagWhitelist: + for t in tags: + if re.match('^'+tname+'$', t.name): + newTags.append(t) + continue + return newTags + else: + return tags + + + +def plot(csvFilePaths, options, tags): + data = np.genfromtxt(tmpFile, dtype=None, invalid_raise = False, + delimiter=',', encoding=None, filling_values="0", + names="sentinal, type, func, tag, x, y, z") + + tagIndex = 0; + tagLabels = [] + lastPlotBottom = [] + legendArtistProxyShapes=[] + skipLegend = options.hide_legend + + for t in tags: + tagLabels.append(t.name) + + for t in tags: + print("plotting tag: " + t.name) + + plt.figure(0, figsize=(options.fig_w, options.fig_h), dpi=800) + if t.tagType == TAG_TYPE.XY: + exes = [] + whys = [] + for row in data: + if row[3] == t.name and (options.xfilter == None or row[4] in options.xfilter): + exes.append(float(row[4])); + whys.append(float(row[5])); + # decorate, sort, then undecorate + exes, whys = (np.array(t) for t in zip(*sorted(zip(exes, whys)))) + # average duplicate values + dedup_exes = np.unique(exes) + dedup_whys = np.empty(dedup_exes.shape) + array_whys = [[] for _ in range(len(dedup_exes))] + for i, x in enumerate(dedup_exes): + dedup_whys[i] = np.mean(whys[exes == x]) + array_whys[i] = whys[exes == x].astype(float).tolist() + + plt.plot(dedup_exes, dedup_whys, + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + + if "obl" not in t.name: + if "lm" in t.name: + vys = array_whys[1::2] + vxs = dedup_exes[1::2] + else: + vys = array_whys[::2] + vxs = dedup_exes[::2] + violin_parts = plt.violinplot(vys, positions=vxs, vert=True) + + #edgecolor = matplotlib.colors.colorConverter.to_rgba('black', alpha=0.1) + for pc in violin_parts['bodies']: + pc.set_color(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None) + # pc.set_facecolor(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None) + for partname in ('cbars','cmins','cmaxes'): + vp = violin_parts[partname] + vp.set_edgecolor(dracula.COLORS[tagIndex+options.color_offset]) + vp.set_linewidth(1) + + elif t.tagType == TAG_TYPE.HISTOGRAM: + print("plotting histogram tag: " + t.name) + print("TODO!") + + elif t.tagType == TAG_TYPE.BAR or t.tagType == TAG_TYPE.GROUPED_BAR or t.tagType == TAG_TYPE.BARBOX or t.tagType == TAG_TYPE.BOX: + labels = [] # vector of x-axis label strings + dataByLabel = [] # vector of all y values, indexed by label + offsetDataByLabel = [] # dataByLable, stacked + whys = [] # average y value indexed by label + width = 0.7 # width of a group of bars for grouped bar graph + + # collect labels for all tags + for row in data: + if len(row) < 6: + print("bar|grouped_bar|box|barbox data bad format") + exit() + for tagLabel in tagLabels: + if row[3] == tagLabel: + if row[4] not in labels and (options.xfilter == None or row[4] in options.xfilter): + labels.append(row[4]) + dataByLabel.append([]) + offsetDataByLabel.append([]) + + if len(lastPlotBottom) == 0: + lastPlotBottom = [0]*len(labels) + + # collect data for this tag + for row in data: + if row[3] == t.name and (options.xfilter == None or row[4] in options.xfilter): + try: + i = labels.index(row[4]) + dataByLabel[i].append(float(row[5])) + offsetDataByLabel[i].append(float(row[5]) + lastPlotBottom[i]) + except: + continue + for row in dataByLabel: + if len(row) != 0: + whys.append(np.average(row)) + else: + whys.append(0) + + if t.tagType == TAG_TYPE.BOX: + # no label on box plots + plt.boxplot(dataByLabel, positions=range(len(labels)))#, showmeans=True) + + edgecolor = matplotlib.colors.colorConverter.to_rgba('black', alpha=0.1) + + if t.tagType == TAG_TYPE.BAR: + if options.horizontal: + plt.barh(range(len(whys)), whys, bottom=lastPlotBottom, + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + else: + plt.bar(range(len(whys)), whys, bottom=lastPlotBottom, + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + if len(tags) > 1 and not skipLegend: + plt.legend() # bar always gets legend, must be after plotting + lastPlotBottom = list( map(add, lastPlotBottom, whys) )# for stacking using average + + if t.tagType == TAG_TYPE.GROUPED_BAR: + offset=len(tags)/2*width/5 + if options.horizontal: + rects = plt.barh(np.arange(len(whys)) - offset + (width*tagIndex/len(tags)), whys, width/len(tags), + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None), + edgecolor=edgecolor, linewidth=1, hatch=options.hatching) + else: + rects = plt.bar(np.arange(len(whys)) - offset + (width*tagIndex/len(tags)), whys, width/len(tags), + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None), + edgecolor=edgecolor, linewidth=1, hatch=options.hatching) + if len(tags) > 1 and not skipLegend: + plt.legend() # bar always gets legend, must be after plotting + lastPlotBottom = list( map(add, lastPlotBottom, whys) )# for stacking using average + + # label rectangles + if options.bar_label_dict and t.name in options.bar_label_dict: + if len(options.bar_label_dict[t.name]) != len(whys): + print('Error - wrong number of bar labels - %d (should be %d)\n\n\n' % (len(options.bar_label_dict[t.name]), len(whys)) ) + return + rectnum=0 + for rect in rects: + height = rect.get_height() + plt.text(rect.get_x() + rect.get_width()/2., 1.05*height, options.bar_label_dict[t.name][rectnum], + ha='center', va='bottom', + bbox=dict(facecolor='white', edgecolor=edgecolor, boxstyle='round,pad=0.2')) + rectnum=rectnum+1 + + + if t.tagType == TAG_TYPE.BARBOX: + # stack boxplot data - shift upwards by last avg + meanpointprops = dict(marker='o', markeredgecolor='black', markerfacecolor='black') + r = plt.boxplot(offsetDataByLabel, positions=range(len(labels)), showmeans=True, meanprops=meanpointprops) + + # need to parse returned means because they dont include outliers + whysExcludingOutliers = [] + for i in range(len(labels)): + whysExcludingOutliers.append(r['means'][i].get_ydata()[0] - lastPlotBottom[i]) + + plt.bar(range(len(whysExcludingOutliers)), whysExcludingOutliers, width, bottom=lastPlotBottom, + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None), + edgecolor='black', linewidth=1, hatch=options.hatching) + lastPlotBottom = list( map(add, lastPlotBottom, whys) )# for stacking using average + if len(tags) > 1 and not skipLegend: + plt.legend() + + if options.print_raw: + print(labels) + np.set_printoptions(threshold=sys.maxsize) + print(whys) + + if options.rotate_x_labels: + plt.xticks(range(len(labels)), labels, rotation=45) + else: + if options.horizontal: + plt.yticks(range(len(labels)), labels) + else: + plt.xticks(range(len(labels)), labels) + + + elif t.tagType == TAG_TYPE.THREEDBAR or t.tagType == TAG_TYPE.THREEDSURF or t.tagType == TAG_TYPE.THREEDPROJECTX or t.tagType == TAG_TYPE.THREEDPROJECTY or t.tagType == TAG_TYPE.THREEDUNROLL: + if t.tagType == TAG_TYPE.THREEDBAR or t.tagType == TAG_TYPE.THREEDSURF: + ax = plt.gca(projection = '3d') + + #plt.rcParams['legend.fontsize'] = 10 + xlabels = [] # vector of x-axis label strings + ylabels = [] # vector of y-axis label strings + + for row in data: + if len(row) < 7: + print("3d bar data bad format") + exit() + if row[3] == t.name: + if (row[4] not in xlabels and (options.xfilter == None or row[4] in options.xfilter)): + xlabels.append(row[4]) + if (row[5] not in ylabels): + ylabels.append(row[5]) + + if len(lastPlotBottom) == 0: + lastPlotBottom = [0]*len(xlabels)*len(ylabels) + elif len(lastPlotBottom) != len(xlabels)*len(ylabels): + print("ERROR - tags have different number of labels") + print(len(lastPlotBottom)) + print(xlabels) + print(ylabels) + exit() + + zDataByLabel = [[[] for x in range(len(ylabels))] for y in range(len(xlabels))] # vector of all z values, indexed by x/y label + zees = [[0 for x in range(len(ylabels))] for y in range(len(xlabels))] # average z value indexed by x/y labels + + for row in data: + if row[3] == t.name and (options.xfilter == None or row[4] in options.xfilter): + xi = xlabels.index(row[4]) + yi = ylabels.index(row[5]) + zDataByLabel[xi][yi].append(float(row[6])) + + for i in range(len(xlabels)): + for j in range(len(ylabels)): + zRow = zDataByLabel[i][j] + if len(zRow) != 0: + zees[i][j] = np.average(zRow) + + x = [] + y = [] + dz = [] + for i in range(len(xlabels)): + for j in range(len(ylabels)): + x.append(i); + y.append(j); + dz.append(zees[i][j]); + dx = np.ones(len(x)) + dy = np.ones(len(y)) + + if options.print_raw: + print(xlabels) + print(ylabels) + np.set_printoptions(threshold=sys.maxsize) + print(np.resize(dz, [len(xlabels), len(ylabels)])) + + ### Note - this should work: + ### surf = ax.bar3d(x, y, lastPlotBottom, dx, dy, dz, label=t.name) + ### But it looks terrible! + ### Plotting the 0 height bars breaks the rendering. + ### Now lets remove them carefully so we don't break lastPlotBottom + + filteredx = [] + filteredy = [] + filteredlpb = [] + filtereddz = [] + for i in range(len(xlabels)): + for j in range(len(ylabels)): + if zees[i][j] != 0: + filteredx.append(i) + filteredy.append(j) + filtereddz.append(zees[i][j]) + filteredlpb.append(lastPlotBottom[i*len(ylabels)+j]) + filtereddx = np.ones(len(filtereddz)) + filtereddy = np.ones(len(filtereddz)) + + if t.tagType == TAG_TYPE.THREEDBAR: + surf = ax.bar3d(filteredx, filteredy, filteredlpb, filtereddx, filtereddy, filtereddz, + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + #surf = ax.bar3d(x, y, lastPlotBottom, dx, dy, dz, label=t.name) + + # Default legend looks terrible + #surf._facecolors2d=surf._facecolors3d + #surf._edgecolors2d=surf._edgecolors3d + #plt.legend() + # Use this instead: https://stackoverflow.com/questions/5803015/how-to-create-a-legend-for-3d-bar-in-matplotlib + legendArtistProxyShapes.append(plt.Rectangle((0,0), 1, 1, + fc=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else plt.get_cmap("tab10")(tagIndex)))) + ax.set_zlabel(options.zlabel) + if options.azimuth: + ax.azim = options.azimuth + if options.elevation: + ax.elev = options.elevation + + plt.xticks(range(len(xlabels)), xlabels)#, rotation=45) + plt.yticks(range(len(ylabels)), ylabels)#, rotation=45) + + elif t.tagType == TAG_TYPE.THREEDSURF: + surf = ax.plot_trisurf(x, y, dz, cmap=cms[tagIndex+options.color_offset], linewidth=0, + antialiased=False, alpha=(tagIndex+1)/len(tags)) + clb = plt.colorbar(surf, shrink=0.5, aspect=14, + pad=(.16 if (tagIndex == len(tags)-1) else .04)) + clb.ax.set_title(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name) + # set_xlabel('hi') to put at bottom + ax.set_zlabel(options.zlabel) + plt.xticks(range(len(xlabels)), xlabels)#, rotation=45) + plt.yticks(range(len(ylabels)), ylabels)#, rotation=45) + plt.tight_layout()# dont move + skipLegend = True + + elif t.tagType == TAG_TYPE.THREEDPROJECTX: + if options.projection == None: + print("need projection command line option") + return + else: + sliced = [] + slicedLPB = [] + for i in range(len(xlabels)): + sliced.append(zees[i][options.projection]) + slicedLPB.append(lastPlotBottom[i * len(ylabels) + options.projection]) + + # Plot a line + plt.plot([float(i) for i in xlabels], sliced, + label=options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name, + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + + # Stacked Bars + #plt.bar([i for i in xlabels], sliced, bottom=slicedLPB, + # label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + # color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + + plt.xticks([float(i) for i in xlabels], xlabels)#, rotation=45) + + elif t.tagType == TAG_TYPE.THREEDPROJECTY: + if options.projection == None: + print("need projection command line option") + return + else: + # Plot a line + plt.plot([float(i) for i in ylabels], zees[options.projection], + label=options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name, + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + + # Stacked Bars + #plt.bar([i for i in xlabels], zees[options.projection], bottom=lastPlotBottom[options.projection*len(ylabels) : options.projection*len(ylabels)+len(ylabels)], # THIS MAY NOT BE RIGHT + # label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + # color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + + plt.xticks([float(i) for i in ylabels], ylabels)#, rotation=45) + + elif t.tagType == TAG_TYPE.THREEDUNROLL: + width = .3 # width of bar + numX = len(xlabels) + + + # insert spaces + spacedLPB = lastPlotBottom.copy() + spacedDz = dz.copy() + for i in range(len(ylabels)): + if i != 0 and i != len(ylabels): + spacedDz.insert((i*len(ylabels)) + i - 1, 0) + spacedLPB.insert((i * len(ylabels)) + i - 1, 0) + + plt.bar(range(len(spacedDz)), spacedDz, bottom=spacedLPB, + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + if len(tags) > 1 and not skipLegend: + plt.legend() # bar always gets legend, must be after plotting + + xylabels = [] + for i in range(len(xlabels)): + if i != 0: + xylabels.append('') + for j in range(len(ylabels)): + xylabels.append(str(xlabels[i]) + ' - ' + str(int(ylabels[j]))) + plt.xticks(range(len(xylabels)), xylabels, rotation=45) + + lastPlotBottom = list( map(add, lastPlotBottom, dz) )# for stacking using average + + + tagIndex += 1; + + if len(legendArtistProxyShapes) > 1 and not skipLegend: # 3dbar plot = special legend + plt.legend(legendArtistProxyShapes, + options.custom_legend_labels if options.custom_legend_labels != None else tagLabels) + elif options.custom_legend_labels != None and len(options.custom_legend_labels) > 1 and not skipLegend: + if options.horizontal: + handles, labels = plt.gca().get_legend_handles_labels() + plt.legend(reversed(handles), reversed(labels)) + else: + plt.legend() + + if options.horizontal: + plt.ylabel(options.xlabel) + plt.xlabel(options.ylabel) + else: + plt.xlabel(options.xlabel) + plt.ylabel(options.ylabel) + plt.title(options.title) + plt.tight_layout() + + # hack for snail + if "EMP_MUL" in tagLabels: + plt.subplots_adjust(left=0.18) + + + if options.show: + plt.show() + if options.log_scale: + if options.horizontal: + plt.gca().set_xscale('log') + else: + plt.gca().set_yscale('log') + plt.savefig(options.graphpath) + print("saved figure to " + options.graphpath) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="JLog plotter") + parser.add_argument('--csvlog', + required=True, + nargs="+", + help=("Path to log file.")) + parser.add_argument('--graphpath', + required=True, + help=("Path to store output graphs.")) + parser.add_argument('--title', + required=False, + default="", + help=("Graph Title.")) + parser.add_argument('--xlabel', + required=False, + default="", + help=("Label for plot x axis.")) + parser.add_argument('--ylabel', + required=False, + default="", + help=("Label for plot y axis.")) + parser.add_argument('--zlabel', + required=False, + default="", + help=("Label for plot z axis.")) + parser.add_argument('--only-tags', + required=False, + nargs="+", + help=("Only plot specific (space seperated) tags in csv log files. Default plots all tags.")) + parser.add_argument('--custom-legend-labels', + required=False, + nargs="+", + help=("Attach a custom legend label to each tag's plot.")) + parser.add_argument('--color-theme', + required=False, + help=("Use a custom color theme - dracula suported")) + parser.add_argument('--color-offset', + required=False, + type=int, + default=0, + help=("Offset starting color in color theme")) + parser.add_argument('--projection', + required=False, + type=int, + help=("Index of x/y projection for 3d to 2d projections")) + parser.add_argument('--azimuth', + required=False, + type=int, + help=("Azimuth sets view for 3D plot")) + parser.add_argument('--elevation', + required=False, + type=int, + help=("Elevation sets view for 3D plot")) + parser.add_argument('--fig-w', + required=False, + type=float, + default=5, + help=("Override figure width")) + parser.add_argument('--fig-h', + required=False, + type=float, + default=4, + help=("Override figure height")) + parser.add_argument('--xfilter', + required=False, + nargs="+", + help=("Only use data which match one of these x values.")) + parser.add_argument('--rotate_x_labels', + required=False, + action="store_true", + help=("Rotate x axis labels.")) + parser.add_argument('--hide-legend', + required=False, + action="store_true", + help=("Hide legend.")) + parser.add_argument('--bar-label-dict', + required=False, + type=json.loads, + help=("Add labels to bars e.g. '{\"tagName\": [\"l1\", \"l2\", \"l3\"]}' ")) + parser.add_argument('--hatching', + required=False, + help=("Add hatching to bars e.g. \"||||\"")) + parser.add_argument('--show', + required=False, + action="store_true", + help=("Show plots as they are generated.")) + parser.add_argument('--print-raw', + required=False, + action="store_true", + help=("Print raw plotted data to command line.")) + parser.add_argument('--log-scale', + required=False, + action="store_true", + help=("Log scale.")) + parser.add_argument('--horizontal', + required=False, + action="store_true", + help=("Plot bar graphs horizontally.")) + options = parser.parse_args() + + try: + csvFilePaths = options.csvlog + for p in csvFilePaths: + with open(p, "r") as f: + print("found file " + p) + except FileNotFoundError: + print("file not found " + p) + exit() + + if options.only_tags != None: + print("Only plotting specific tags: ") + print(options.only_tags) + + tags = getTags(csvFilePaths, options.only_tags) + + if tags == None or len(tags) == 0: + print("ERROR - No tags found - Nothing to plot") + exit() + + if options.custom_legend_labels != None: + if len(options.custom_legend_labels) == len(tags): + print("Custom Tag Labels Enabled") + else: + print("Wrong Number of Custom Tag Labels") + print(options.custom_legend_labels) + print(tags) + exit() + + plot(csvFilePaths, options, tags) + + if os.path.exists(tmpFile): + os.remove(tmpFile) diff --git a/scripts/plotter-emp-vs-aby.py b/scripts/plotter-emp-vs-aby.py new file mode 100644 index 0000000..458510c --- /dev/null +++ b/scripts/plotter-emp-vs-aby.py @@ -0,0 +1,751 @@ +import matplotlib.pyplot as plt +from mpl_toolkits.mplot3d import Axes3D +from matplotlib import cm +import matplotlib.colors +import numpy as np +from operator import add +import os +import argparse +import enum +import sys +import json +import re + +sentinalStr = "SeNtInAl" +tmpFile = "plotlines.tmp" +maxColumns = 7 + +# Plotter cheat sheet +# +# XY +# SeNtInAl, tag_type, user_defined, tag_name, x, y +# +# +# BAR | GROUPED_BAR | BARBOX | BOX +# SeNtInAl, tag_type, user_defined, tag_name, x_label, y +# +# tag_names - plotting multiple will create segmented bars with +# a legend. Legend can be overridden with --custom-legend-labels +# x_label - string used to denote X +# y - float value +# +# THREEDBAR +# SeNtInAl, tag_type, z, tag_name, x_label, y_label +# tag_names - plotting multiple will create segmented bars with +# a legend. Legend can be overridden with --custom-legend-labels +# x_label - string used to denote X +# y_label - string used to denote Y +# z - float + + + +class TAG_TYPE(enum.Enum): + UNDEFINED = 0 + XY = 1 + HISTOGRAM = 2 + CDF = 3 + BAR = 4 + BOX = 5 + BARBOX = 6 + GROUPED_BAR = 7 + THREEDBAR = 8 + THREEDSURF = 9 + THREEDPROJECTX = 10 + THREEDPROJECTY = 11 + THREEDUNROLL = 12 + +#class dracula(): +# CYAN = '#8be9fd' +# GREEN = '#50fa7b' +# ORANGE = '#ffb86c' +# PINK = '#ff79c6' +# PURPLE = '#bd93f9' +# RED = '#ff5555' +# YELLOW = '#f1fa8c' +# COLORS = [CYAN, ORANGE, GREEN, PINK, PURPLE, YELLOW, RED] + +class dracula(): + ZERO = '#D291AC' + ONE = '#779E44' + TWO = '#F6B7AA' + THREE = '#367E51' + FOUR = '#5F1333' + FIVE = '#426615' + SIX = '#AD5B4A' + #COLORS = [ZERO, ONE, TWO, THREE, FOUR, FIVE, SIX, ZERO, ONE, TWO, THREE, FOUR] + COLORS = [plt.cm.Pastel1(i) for i in range(9)] + +#cmap1 = matplotlib.colors.LinearSegmentedColormap.from_list("", [dracula.PINK, dracula.PURPLE]) +#cmap2 = matplotlib.colors.LinearSegmentedColormap.from_list("", [dracula.CYAN, dracula.ORANGE]) + +cmap1 = matplotlib.colors.LinearSegmentedColormap.from_list("", [dracula.ZERO, dracula.TWO]) +cmap2 = matplotlib.colors.LinearSegmentedColormap.from_list("", [dracula.FOUR, dracula.SIX]) + +#cms = [cm.inferno, cm.viridis] +#cms = [cm.inferno, cm.cool] +#cms = [cm.copper, cm.summer] +cms = [cmap1, cmap2] + + +class tag: + def __init__(self): + self.tagType = TAG_TYPE.UNDEFINED + self.name = "" + + + +def getTags(csvFilePaths, tagWhitelist): + tags = [] + + printlogwarning = True; + with open(tmpFile, "w") as aggregateFile: + for p in csvFilePaths: + with open(p, "r") as csvIn: + lines = csvIn.readlines() + + for line in lines: + if line[:len(sentinalStr)] == sentinalStr: + tokens = line.split(',') + + if len(tokens) < 6 or len(tokens) > 8: + if printlogwarning: + print("WARNING - Malformed log line") + print(line) + printlogwarning = False + continue + + # strip trailing \n, add right amount of commas, add \n back + line = line.rstrip() + (','*(maxColumns - len(tokens))) + '\n' + aggregateFile.write(line) + + if tokens[3] == "error": + print("WARNING - OVERFLOW ERROR IN LOG FILE") + print(line) + continue + + whitelisted = tagWhitelist == None + if not whitelisted: + for whitelistedTag in tagWhitelist: + whitelisted = whitelisted or re.match('^'+whitelistedTag+'$', tokens[3]) + if not whitelisted: + continue + + newTag = True + for t in tags: + if t.name == tokens[3]: # must be exact match to be old tag + newTag = False + break + + if newTag and whitelisted: + print("found new tag: " + tokens[3]) + t = tag() + if tokens[1] == "xy": + t.tagType = TAG_TYPE.XY + if tokens[1] == "box": + t.tagType = TAG_TYPE.BOX + elif tokens[1] == "histogram": + t.tagType = TAG_TYPE.HISTOGRAM + elif tokens[1] == "cdf": + t.tagType = TAG_TYPE.CDF + elif tokens[1] == "bar": + t.tagType = TAG_TYPE.BAR + elif tokens[1] == "grouped_bar": + t.tagType = TAG_TYPE.GROUPED_BAR + elif tokens[1] == "barbox": + t.tagType = TAG_TYPE.BARBOX + elif tokens[1] == "3dbar": + t.tagType = TAG_TYPE.THREEDBAR + elif tokens[1] == "3dsurf": + t.tagType = TAG_TYPE.THREEDSURF + elif tokens[1] == "3dprojectx": + t.tagType = TAG_TYPE.THREEDPROJECTX + elif tokens[1] == "3dprojecty": + t.tagType = TAG_TYPE.THREEDPROJECTY + elif tokens[1] == "3dunroll": + t.tagType = TAG_TYPE.THREEDUNROLL + t.name = tokens[3] + tags.append(t) + + # make sure tags in same order as tagWhitelist + if tagWhitelist != None: + newTags = [] + for tname in tagWhitelist: + for t in tags: + if re.match('^'+tname+'$', t.name): + newTags.append(t) + continue + return newTags + else: + return tags + + + +def plot(csvFilePaths, options, tags): + data = np.genfromtxt(tmpFile, dtype=None, invalid_raise = False, + delimiter=',', encoding=None, filling_values="0", + names="sentinal, type, func, tag, x, y, z") + + tagIndex = 0; + tagLabels = [] + lastPlotBottom = [] + legendArtistProxyShapes=[] + skipLegend = options.hide_legend + + maxEmp = [] + minAby = [] + + for t in tags: + tagLabels.append(t.name) + + for t in tags: + print("plotting tag: " + t.name) + + plt.figure(0, figsize=(options.fig_w, options.fig_h), dpi=800) + if t.tagType == TAG_TYPE.XY: + exes = [] + whys = [] + for row in data: + if row[3] == t.name and (options.xfilter == None or row[4] in options.xfilter): + exes.append(float(row[4])); + whys.append(float(row[5])); + # decorate, sort, then undecorate + exes, whys = (np.array(t) for t in zip(*sorted(zip(exes, whys)))) + # average duplicate values + dedup_exes = np.unique(exes) + dedup_whys = np.empty(dedup_exes.shape) + for i, x in enumerate(dedup_exes): + dedup_whys[i] = np.mean(whys[exes == x]) # / 1e9 + + plt.plot(dedup_exes, dedup_whys, + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + + elif t.tagType == TAG_TYPE.HISTOGRAM: + print("plotting histogram tag: " + t.name) + print("TODO!") + + elif t.tagType == TAG_TYPE.BAR or t.tagType == TAG_TYPE.GROUPED_BAR or t.tagType == TAG_TYPE.BARBOX or t.tagType == TAG_TYPE.BOX: + labels = [] # vector of x-axis label strings + dataByLabel = [] # vector of all y values, indexed by label + offsetDataByLabel = [] # dataByLable, stacked + whys = [] # average y value indexed by label + width = 0.8 # width of a group of bars for grouped bar graph + + # collect labels for all tags + for row in data: + if len(row) < 6: + print("bar|grouped_bar|box|barbox data bad format") + exit() + for tagLabel in tagLabels: + if row[3] == tagLabel: + if row[4] not in labels and (options.xfilter == None or row[4] in options.xfilter): + labels.append(row[4]) + dataByLabel.append([]) + offsetDataByLabel.append([]) + + if len(lastPlotBottom) == 0: + lastPlotBottom = [0]*len(labels) + + # collect data for this tag + for row in data: + if row[3] == t.name and (options.xfilter == None or row[4] in options.xfilter): + try: + i = labels.index(row[4]) + dataByLabel[i].append(float(row[5])) + offsetDataByLabel[i].append(float(row[5]) + lastPlotBottom[i]) + except: + continue + + for row in dataByLabel: + if len(row) != 0: + whys.append(np.average(row)) + else: + whys.append(0) + + + + + for j in range(len(whys)): + if "emp" in t.name: + if len(maxEmp) <= j: + maxEmp.append(0) + maxEmp[j] = max(maxEmp[j], whys[j]) + + elif "aby" in t.name: + if len(minAby) <= j: + minAby.append(float('inf')) + minAby[j] = min(minAby[j], whys[j]) + + + + + + if t.tagType == TAG_TYPE.BOX: + # no label on box plots + plt.boxplot(dataByLabel, positions=range(len(labels)))#, showmeans=True) + + if t.tagType == TAG_TYPE.BAR: + if options.horizontal: + plt.barh(range(len(whys)), whys, bottom=lastPlotBottom, + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + else: + plt.bar(range(len(whys)), whys, bottom=lastPlotBottom, + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + if len(tags) > 1 and not skipLegend: + plt.legend() # bar always gets legend, must be after plotting + lastPlotBottom = list( map(add, lastPlotBottom, whys) )# for stacking using average + + if t.tagType == TAG_TYPE.GROUPED_BAR: + edgecolor = matplotlib.colors.colorConverter.to_rgba('black', alpha=0.1) + + offset = width/2 - (width/len(tags)/2) + if options.horizontal: + rects = plt.barh(np.arange(len(whys)) - offset + (width*tagIndex/len(tags)), whys, width/len(tags), + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None), + edgecolor=edgecolor, linewidth=1, hatch=options.hatching) + else: + rects = plt.bar(np.arange(len(whys)) - offset + (width*tagIndex/len(tags)), whys, width/len(tags), + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None), + edgecolor=edgecolor, linewidth=1, hatch=options.hatching) + if len(tags) > 1 and not skipLegend: + plt.legend() # bar always gets legend, must be after plotting + lastPlotBottom = list( map(add, lastPlotBottom, whys) )# for stacking using average + + # label rectangles + if options.bar_label_dict and t.name in options.bar_label_dict: + if len(options.bar_label_dict[t.name]) != len(whys): + print('Error - wrong number of bar labels - %d (should be %d)\n\n\n' % (len(options.bar_label_dict[t.name]), len(whys)) ) + return + rectnum=0 + for rect in rects: + height = rect.get_height() + plt.text(rect.get_x() + rect.get_width()/2., 1.05*height, options.bar_label_dict[t.name][rectnum], + ha='center', va='bottom', + bbox=dict(facecolor='white', edgecolor='black', boxstyle='round,pad=0.2')) + rectnum=rectnum+1 + + + if t.tagType == TAG_TYPE.BARBOX: + # stack boxplot data - shift upwards by last avg + meanpointprops = dict(marker='o', markeredgecolor='black', markerfacecolor='black') + r = plt.boxplot(offsetDataByLabel, positions=range(len(labels)), showmeans=True, meanprops=meanpointprops) + + # need to parse returned means because they dont include outliers + whysExcludingOutliers = [] + for i in range(len(labels)): + whysExcludingOutliers.append(r['means'][i].get_ydata()[0] - lastPlotBottom[i]) + + plt.bar(range(len(whysExcludingOutliers)), whysExcludingOutliers, width, bottom=lastPlotBottom, + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None), + edgecolor='black', linewidth=1, hatch=options.hatching) + lastPlotBottom = list( map(add, lastPlotBottom, whys) )# for stacking using average + if len(tags) > 1 and not skipLegend: + plt.legend() + + if options.print_raw: + print(labels) + np.set_printoptions(threshold=sys.maxsize) + print(whys) + + if options.rotate_x_labels: + plt.xticks(range(len(labels)), labels, rotation=45) + else: + if options.horizontal: + plt.yticks(range(len(labels)), labels) + else: + plt.xticks(range(len(labels)), labels) + + + elif t.tagType == TAG_TYPE.THREEDBAR or t.tagType == TAG_TYPE.THREEDSURF or t.tagType == TAG_TYPE.THREEDPROJECTX or t.tagType == TAG_TYPE.THREEDPROJECTY or t.tagType == TAG_TYPE.THREEDUNROLL: + if t.tagType == TAG_TYPE.THREEDBAR or t.tagType == TAG_TYPE.THREEDSURF: + ax = plt.gca(projection = '3d') + + #plt.rcParams['legend.fontsize'] = 10 + xlabels = [] # vector of x-axis label strings + ylabels = [] # vector of y-axis label strings + + for row in data: + if len(row) < 7: + print("3d bar data bad format") + exit() + if row[3] == t.name: + if (row[4] not in xlabels and (options.xfilter == None or row[4] in options.xfilter)): + xlabels.append(row[4]) + if (row[5] not in ylabels): + ylabels.append(row[5]) + + if len(lastPlotBottom) == 0: + lastPlotBottom = [0]*len(xlabels)*len(ylabels) + elif len(lastPlotBottom) != len(xlabels)*len(ylabels): + print("ERROR - tags have different number of labels") + print(len(lastPlotBottom)) + print(xlabels) + print(ylabels) + exit() + + zDataByLabel = [[[] for x in range(len(ylabels))] for y in range(len(xlabels))] # vector of all z values, indexed by x/y label + zees = [[0 for x in range(len(ylabels))] for y in range(len(xlabels))] # average z value indexed by x/y labels + + for row in data: + if row[3] == t.name and (options.xfilter == None or row[4] in options.xfilter): + xi = xlabels.index(row[4]) + yi = ylabels.index(row[5]) + zDataByLabel[xi][yi].append(float(row[6])) + + for i in range(len(xlabels)): + for j in range(len(ylabels)): + zRow = zDataByLabel[i][j] + if len(zRow) != 0: + zees[i][j] = np.average(zRow) + + x = [] + y = [] + dz = [] + for i in range(len(xlabels)): + for j in range(len(ylabels)): + x.append(i); + y.append(j); + dz.append(zees[i][j]); + dx = np.ones(len(x)) + dy = np.ones(len(y)) + + if options.print_raw: + print(xlabels) + print(ylabels) + np.set_printoptions(threshold=sys.maxsize) + print(np.resize(dz, [len(xlabels), len(ylabels)])) + + ### Note - this should work: + ### surf = ax.bar3d(x, y, lastPlotBottom, dx, dy, dz, label=t.name) + ### But it looks terrible! + ### Plotting the 0 height bars breaks the rendering. + ### Now lets remove them carefully so we don't break lastPlotBottom + + filteredx = [] + filteredy = [] + filteredlpb = [] + filtereddz = [] + for i in range(len(xlabels)): + for j in range(len(ylabels)): + if zees[i][j] != 0: + filteredx.append(i) + filteredy.append(j) + filtereddz.append(zees[i][j]) + filteredlpb.append(lastPlotBottom[i*len(ylabels)+j]) + filtereddx = np.ones(len(filtereddz)) + filtereddy = np.ones(len(filtereddz)) + + if t.tagType == TAG_TYPE.THREEDBAR: + surf = ax.bar3d(filteredx, filteredy, filteredlpb, filtereddx, filtereddy, filtereddz, + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + #surf = ax.bar3d(x, y, lastPlotBottom, dx, dy, dz, label=t.name) + + # Default legend looks terrible + #surf._facecolors2d=surf._facecolors3d + #surf._edgecolors2d=surf._edgecolors3d + #plt.legend() + # Use this instead: https://stackoverflow.com/questions/5803015/how-to-create-a-legend-for-3d-bar-in-matplotlib + legendArtistProxyShapes.append(plt.Rectangle((0,0), 1, 1, + fc=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else plt.get_cmap("tab10")(tagIndex)))) + ax.set_zlabel(options.zlabel) + if options.azimuth: + ax.azim = options.azimuth + if options.elevation: + ax.elev = options.elevation + + plt.xticks(range(len(xlabels)), xlabels)#, rotation=45) + plt.yticks(range(len(ylabels)), ylabels)#, rotation=45) + + elif t.tagType == TAG_TYPE.THREEDSURF: + surf = ax.plot_trisurf(x, y, dz, cmap=cms[tagIndex+options.color_offset], linewidth=0, + antialiased=False, alpha=(tagIndex+1)/len(tags)) + clb = plt.colorbar(surf, shrink=0.5, aspect=14, + pad=(.16 if (tagIndex == len(tags)-1) else .04)) + clb.ax.set_title(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name) + # set_xlabel('hi') to put at bottom + ax.set_zlabel(options.zlabel) + plt.xticks(range(len(xlabels)), xlabels)#, rotation=45) + plt.yticks(range(len(ylabels)), ylabels)#, rotation=45) + plt.tight_layout()# dont move + skipLegend = True + + elif t.tagType == TAG_TYPE.THREEDPROJECTX: + if options.projection == None: + print("need projection command line option") + return + else: + sliced = [] + slicedLPB = [] + for i in range(len(xlabels)): + sliced.append(zees[i][options.projection]) + slicedLPB.append(lastPlotBottom[i * len(ylabels) + options.projection]) + + # Plot a line + plt.plot([float(i) for i in xlabels], sliced, + label=options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name, + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + + # Stacked Bars + #plt.bar([i for i in xlabels], sliced, bottom=slicedLPB, + # label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + # color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + + plt.xticks([float(i) for i in xlabels], xlabels)#, rotation=45) + + elif t.tagType == TAG_TYPE.THREEDPROJECTY: + if options.projection == None: + print("need projection command line option") + return + else: + # Plot a line + plt.plot([float(i) for i in ylabels], zees[options.projection], + label=options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name, + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + + # Stacked Bars + #plt.bar([i for i in xlabels], zees[options.projection], bottom=lastPlotBottom[options.projection*len(ylabels) : options.projection*len(ylabels)+len(ylabels)], # THIS MAY NOT BE RIGHT + # label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + # color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + + plt.xticks([float(i) for i in ylabels], ylabels)#, rotation=45) + + elif t.tagType == TAG_TYPE.THREEDUNROLL: + width = .3 # width of bar + numX = len(xlabels) + + + # insert spaces + spacedLPB = lastPlotBottom.copy() + spacedDz = dz.copy() + for i in range(len(ylabels)): + if i != 0 and i != len(ylabels): + spacedDz.insert((i*len(ylabels)) + i - 1, 0) + spacedLPB.insert((i * len(ylabels)) + i - 1, 0) + + plt.bar(range(len(spacedDz)), spacedDz, bottom=spacedLPB, + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + if len(tags) > 1 and not skipLegend: + plt.legend() # bar always gets legend, must be after plotting + + xylabels = [] + for i in range(len(xlabels)): + if i != 0: + xylabels.append('') + for j in range(len(ylabels)): + xylabels.append(str(xlabels[i]) + ' - ' + str(int(ylabels[j]))) + plt.xticks(range(len(xylabels)), xylabels, rotation=45) + + lastPlotBottom = list( map(add, lastPlotBottom, dz) )# for stacking using average + + + tagIndex += 1; + + if len(legendArtistProxyShapes) > 1 and not skipLegend: # 3dbar plot = special legend + plt.legend(legendArtistProxyShapes, + options.custom_legend_labels if options.custom_legend_labels != None else tagLabels) + elif options.custom_legend_labels != None and len(options.custom_legend_labels) > 1 and not skipLegend: + if options.horizontal: + handles, labels = plt.gca().get_legend_handles_labels() + plt.legend(reversed(handles), reversed(labels)) + else: + plt.legend() + + if options.horizontal: + plt.ylabel(options.xlabel) + plt.xlabel(options.ylabel) + else: + plt.xlabel(options.xlabel) + plt.ylabel(options.ylabel) + + + + + print("# Calculating the trendline") + x = np.arange(len(maxEmp)) + + empCoefficients = np.polyfit(x, maxEmp, 2) + empTrendline = np.polyval(empCoefficients, x) + + abyCoefficients = np.polyfit(x, minAby, 2) + abyTrendline = np.polyval(abyCoefficients, x) + + plt.plot(empTrendline, x, color='grey', zorder=-1) + plt.plot(abyTrendline, x, color='black', zorder=-1) + + plt.fill_betweenx(x, empTrendline, abyTrendline, color='lightgrey', alpha=0.5, zorder=-1) + + # zorder = line.get_zorder() + # alpha = line.get_alpha() + # alpha = 1.0 if alpha is None else alpha + + # z = np.empty((100, 1, 4), dtype=float) + # rgb = mcolors.colorConverter.to_rgb(fill_color) + # z[:,:,:3] = rgb + # z[:,:,-1] = np.linspace(0, alpha, 100)[:,None] + + # xmin, xmax, ymin, ymax = x.min(), x.max(), y.min(), y.max() + # im = ax.imshow(z, aspect='auto', extent=[xmin, xmax, ymin, ymax], + # origin='lower', zorder=zorder) + + # xy = np.column_stack([x, y]) + # xy = np.vstack([[xmin, ymin], xy, [xmax, ymin], [xmin, ymin]]) + # clip_path = Polygon(xy, facecolor='none', edgecolor='none', closed=True) + # ax.add_patch(clip_path) + # im.set_clip_path(clip_path) + + + + + + plt.title(options.title) + plt.tight_layout() + if options.show: + plt.show() + if options.log_scale: + if options.horizontal: + plt.gca().set_xscale('log') + else: + plt.gca().set_yscale('log') + plt.savefig(options.graphpath) + print("saved figure to " + options.graphpath) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="JLog plotter") + parser.add_argument('--csvlog', + required=True, + nargs="+", + help=("Path to log file.")) + parser.add_argument('--graphpath', + required=True, + help=("Path to store output graphs.")) + parser.add_argument('--title', + required=False, + default="", + help=("Graph Title.")) + parser.add_argument('--xlabel', + required=False, + default="", + help=("Label for plot x axis.")) + parser.add_argument('--ylabel', + required=False, + default="", + help=("Label for plot y axis.")) + parser.add_argument('--zlabel', + required=False, + default="", + help=("Label for plot z axis.")) + parser.add_argument('--only-tags', + required=False, + nargs="+", + help=("Only plot specific (space seperated) tags in csv log files. Default plots all tags.")) + parser.add_argument('--custom-legend-labels', + required=False, + nargs="+", + help=("Attach a custom legend label to each tag's plot.")) + parser.add_argument('--color-theme', + required=False, + help=("Use a custom color theme - dracula suported")) + parser.add_argument('--color-offset', + required=False, + type=int, + default=0, + help=("Offset starting color in color theme")) + parser.add_argument('--projection', + required=False, + type=int, + help=("Index of x/y projection for 3d to 2d projections")) + parser.add_argument('--azimuth', + required=False, + type=int, + help=("Azimuth sets view for 3D plot")) + parser.add_argument('--elevation', + required=False, + type=int, + help=("Elevation sets view for 3D plot")) + parser.add_argument('--fig-w', + required=False, + type=float, + default=5, + help=("Override figure width")) + parser.add_argument('--fig-h', + required=False, + type=float, + default=4, + help=("Override figure height")) + parser.add_argument('--xfilter', + required=False, + nargs="+", + help=("Only use data which match one of these x values.")) + parser.add_argument('--rotate_x_labels', + required=False, + action="store_true", + help=("Rotate x axis labels.")) + parser.add_argument('--hide-legend', + required=False, + action="store_true", + help=("Hide legend.")) + parser.add_argument('--bar-label-dict', + required=False, + type=json.loads, + help=("Add labels to bars e.g. '{\"tagName\": [\"l1\", \"l2\", \"l3\"]}' ")) + parser.add_argument('--hatching', + required=False, + help=("Add hatching to bars e.g. \"||||\"")) + parser.add_argument('--show', + required=False, + action="store_true", + help=("Show plots as they are generated.")) + parser.add_argument('--print-raw', + required=False, + action="store_true", + help=("Print raw plotted data to command line.")) + parser.add_argument('--log-scale', + required=False, + action="store_true", + help=("Log scale.")) + parser.add_argument('--horizontal', + required=False, + action="store_true", + help=("Plot bar graphs horizontally.")) + options = parser.parse_args() + + try: + csvFilePaths = options.csvlog + for p in csvFilePaths: + with open(p, "r") as f: + print("found file " + p) + except FileNotFoundError: + print("file not found " + p) + exit() + + if options.only_tags != None: + print("Only plotting specific tags: ") + print(options.only_tags) + + tags = getTags(csvFilePaths, options.only_tags) + + if tags == None or len(tags) == 0: + print("ERROR - No tags found - Nothing to plot") + exit() + + if options.custom_legend_labels != None: + if len(options.custom_legend_labels) == len(tags): + print("Custom Tag Labels Enabled") + else: + print("Wrong Number of Custom Tag Labels") + print(options.custom_legend_labels) + print(tags) + exit() + + plot(csvFilePaths, options, tags) + + if os.path.exists(tmpFile): + os.remove(tmpFile) diff --git a/scripts/plotter-mult-add.py b/scripts/plotter-mult-add.py new file mode 100644 index 0000000..231b0ea --- /dev/null +++ b/scripts/plotter-mult-add.py @@ -0,0 +1,739 @@ +import matplotlib.pyplot as plt +from mpl_toolkits.mplot3d import Axes3D +from matplotlib import cm +import matplotlib.colors +import numpy as np +from operator import add +import os +import argparse +import enum +import sys +import json +import re + +sentinalStr = "SeNtInAl" +tmpFile = "plotlines.tmp" +maxColumns = 7 + +# Plotter cheat sheet +# +# XY +# SeNtInAl, tag_type, user_defined, tag_name, x, y +# +# +# BAR | GROUPED_BAR | BARBOX | BOX +# SeNtInAl, tag_type, user_defined, tag_name, x_label, y +# +# tag_names - plotting multiple will create segmented bars with +# a legend. Legend can be overridden with --custom-legend-labels +# x_label - string used to denote X +# y - float value +# +# THREEDBAR +# SeNtInAl, tag_type, z, tag_name, x_label, y_label +# tag_names - plotting multiple will create segmented bars with +# a legend. Legend can be overridden with --custom-legend-labels +# x_label - string used to denote X +# y_label - string used to denote Y +# z - float + + + +class TAG_TYPE(enum.Enum): + UNDEFINED = 0 + XY = 1 + HISTOGRAM = 2 + CDF = 3 + BAR = 4 + BOX = 5 + BARBOX = 6 + GROUPED_BAR = 7 + THREEDBAR = 8 + THREEDSURF = 9 + THREEDPROJECTX = 10 + THREEDPROJECTY = 11 + THREEDUNROLL = 12 + +#class dracula(): +# CYAN = '#8be9fd' +# GREEN = '#50fa7b' +# ORANGE = '#ffb86c' +# PINK = '#ff79c6' +# PURPLE = '#bd93f9' +# RED = '#ff5555' +# YELLOW = '#f1fa8c' +# COLORS = [CYAN, ORANGE, GREEN, PINK, PURPLE, YELLOW, RED] + +class dracula(): + ZERO = '#D291AC' + ONE = '#779E44' + TWO = '#F6B7AA' + THREE = '#367E51' + FOUR = '#5F1333' + FIVE = '#426615' + SIX = '#AD5B4A' + #COLORS = [ZERO, ONE, TWO, THREE, FOUR, FIVE, SIX, ZERO, ONE, TWO, THREE, FOUR] + COLORS = [plt.cm.Pastel1(i) for i in range(9)] + + + +#cmap1 = matplotlib.colors.LinearSegmentedColormap.from_list("", [dracula.PINK, dracula.PURPLE]) +#cmap2 = matplotlib.colors.LinearSegmentedColormap.from_list("", [dracula.CYAN, dracula.ORANGE]) + +cmap1 = matplotlib.colors.LinearSegmentedColormap.from_list("", [dracula.ZERO, dracula.TWO]) +cmap2 = matplotlib.colors.LinearSegmentedColormap.from_list("", [dracula.FOUR, dracula.SIX]) + +#cms = [cm.inferno, cm.viridis] +#cms = [cm.inferno, cm.cool] +#cms = [cm.copper, cm.summer] +cms = [cmap1, cmap2] + + +class tag: + def __init__(self): + self.tagType = TAG_TYPE.UNDEFINED + self.name = "" + + + +def getTags(csvFilePaths, tagWhitelist): + tags = [] + + printlogwarning = True; + with open(tmpFile, "w") as aggregateFile: + for p in csvFilePaths: + with open(p, "r") as csvIn: + lines = csvIn.readlines() + + for line in lines: + if line[:len(sentinalStr)] == sentinalStr: + tokens = line.split(',') + + if len(tokens) < 6 or len(tokens) > 8: + if printlogwarning: + print("WARNING - Malformed log line") + print(line) + printlogwarning = False + continue + + # strip trailing \n, add right amount of commas, add \n back + line = line.rstrip() + (','*(maxColumns - len(tokens))) + '\n' + aggregateFile.write(line) + + if tokens[3] == "error": + print("WARNING - OVERFLOW ERROR IN LOG FILE") + print(line) + continue + + whitelisted = tagWhitelist == None + if not whitelisted: + for whitelistedTag in tagWhitelist: + whitelisted = whitelisted or re.match('^'+whitelistedTag+'$', tokens[3]) + if not whitelisted: + continue + + newTag = True + for t in tags: + if t.name == tokens[3]: # must be exact match to be old tag + newTag = False + break + + if newTag and whitelisted: + print("found new tag: " + tokens[3]) + t = tag() + if tokens[1] == "xy": + t.tagType = TAG_TYPE.XY + if tokens[1] == "box": + t.tagType = TAG_TYPE.BOX + elif tokens[1] == "histogram": + t.tagType = TAG_TYPE.HISTOGRAM + elif tokens[1] == "cdf": + t.tagType = TAG_TYPE.CDF + elif tokens[1] == "bar": + t.tagType = TAG_TYPE.BAR + elif tokens[1] == "grouped_bar": + t.tagType = TAG_TYPE.GROUPED_BAR + elif tokens[1] == "barbox": + t.tagType = TAG_TYPE.BARBOX + elif tokens[1] == "3dbar": + t.tagType = TAG_TYPE.THREEDBAR + elif tokens[1] == "3dsurf": + t.tagType = TAG_TYPE.THREEDSURF + elif tokens[1] == "3dprojectx": + t.tagType = TAG_TYPE.THREEDPROJECTX + elif tokens[1] == "3dprojecty": + t.tagType = TAG_TYPE.THREEDPROJECTY + elif tokens[1] == "3dunroll": + t.tagType = TAG_TYPE.THREEDUNROLL + t.name = tokens[3] + tags.append(t) + + # make sure tags in same order as tagWhitelist + if tagWhitelist != None: + newTags = [] + for tname in tagWhitelist: + for t in tags: + if re.match('^'+tname+'$', t.name): + newTags.append(t) + continue + return newTags + else: + return tags + + + +def plot(csvFilePaths, options, tags): + data = np.genfromtxt(tmpFile, dtype=None, invalid_raise = False, + delimiter=',', encoding=None, filling_values="0", + names="sentinal, type, func, tag, x, y, z") + + tagIndex = 0; + tagLabels = [] + lastPlotBottom = [] + legendArtistProxyShapes=[] + skipLegend = options.hide_legend + + for t in tags: + tagLabels.append(t.name) + + for t in tags: + print("plotting tag: " + t.name) + + plt.figure(0, figsize=(options.fig_w, options.fig_h), dpi=800) + if t.tagType == TAG_TYPE.XY: + exes = [] + whys = [] + for row in data: + if row[3] == t.name and (options.xfilter == None or row[4] in options.xfilter): + exes.append(float(row[4])); + whys.append(float(row[5])); + # decorate, sort, then undecorate + exes, whys = (np.array(t) for t in zip(*sorted(zip(exes, whys)))) + # average duplicate values + dedup_exes = np.unique(exes) + dedup_whys = np.empty(dedup_exes.shape) + for i, x in enumerate(dedup_exes): + dedup_whys[i] = np.mean(whys[exes == x]) # / 1e9 + + plt.plot(dedup_exes, dedup_whys, + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + + elif t.tagType == TAG_TYPE.HISTOGRAM: + print("plotting histogram tag: " + t.name) + print("TODO!") + + elif t.tagType == TAG_TYPE.BAR or t.tagType == TAG_TYPE.GROUPED_BAR or t.tagType == TAG_TYPE.BARBOX or t.tagType == TAG_TYPE.BOX: + labels = [] # vector of x-axis label strings + dataByLabel = [] # vector of all y values, indexed by label + offsetDataByLabel = [] # dataByLable, stacked + whys = [] # average y value indexed by label + width = 0.5 # width of a group of bars for grouped bar graph + + # collect labels for all tags + for row in data: + if len(row) < 6: + print("bar|grouped_bar|box|barbox data bad format") + exit() + for tagLabel in tagLabels: + if row[3] == tagLabel: + if row[4] not in labels and (options.xfilter == None or row[4] in options.xfilter): + labels.append(row[4]) + dataByLabel.append([]) + offsetDataByLabel.append([]) + + if len(lastPlotBottom) == 0: + lastPlotBottom = [0]*len(labels) + + # collect data for this tag + for row in data: + if row[3] == t.name and (options.xfilter == None or row[4] in options.xfilter): + try: + i = labels.index(row[4]) + dataByLabel[i].append(float(row[5])) + offsetDataByLabel[i].append(float(row[5]) + lastPlotBottom[i]) + except: + continue + for row in dataByLabel: + if len(row) != 0: + whys.append(np.average(row)) + else: + whys.append(0) + + if t.tagType == TAG_TYPE.BOX: + # no label on box plots + plt.boxplot(dataByLabel, positions=range(len(labels)))#, showmeans=True) + + edgecolor = matplotlib.colors.colorConverter.to_rgba('black', alpha=0.1) + + if t.tagType == TAG_TYPE.BAR: + if options.horizontal: + plt.barh(range(len(whys)), whys, bottom=lastPlotBottom, + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + else: + plt.bar(range(len(whys)), whys, bottom=lastPlotBottom, + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + if len(tags) > 1 and not skipLegend: + plt.legend() # bar always gets legend, must be after plotting + lastPlotBottom = list( map(add, lastPlotBottom, whys) )# for stacking using average + + if t.tagType == TAG_TYPE.GROUPED_BAR: + offset=len(tags)/2*width/5 + + + + # We save data for 10k operations but let's plot per op + whys = [y / 10000 * 1000 for y in whys] + + + + if options.horizontal: + rects = plt.barh(np.arange(len(whys)) - offset + (width*tagIndex/len(tags)), whys, width/len(tags), + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None), + edgecolor=edgecolor, linewidth=1, hatch=options.hatching) + else: + if "EMP_MUL" in t.name: + ax2 = plt.gca().twinx() + rects = ax2.bar(np.arange(len(whys)) - offset + (width*tagIndex/len(tags)), whys, width/len(tags), + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None), + edgecolor=edgecolor, linewidth=1, hatch=options.hatching) + plt.ylabel("Multiplication (ms)") + + else: + rects = plt.bar(np.arange(len(whys)) - offset + (width*tagIndex/len(tags)), whys, width/len(tags), + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None), + edgecolor=edgecolor, linewidth=1, hatch=options.hatching) + plt.ylabel("Addition (ms)") + # if len(tags) > 1 and not skipLegend: + # plt.legend() # bar always gets legend, must be after plotting + lastPlotBottom = list( map(add, lastPlotBottom, whys) )# for stacking using average + + # label rectangles + if options.bar_label_dict and t.name in options.bar_label_dict: + if len(options.bar_label_dict[t.name]) != len(whys): + print('Error - wrong number of bar labels - %d (should be %d)\n\n\n' % (len(options.bar_label_dict[t.name]), len(whys)) ) + return + rectnum=0 + for rect in rects: + height = rect.get_height() + plt.text(rect.get_x() + rect.get_width()/2., 1.05*height, options.bar_label_dict[t.name][rectnum], + ha='center', va='bottom', + bbox=dict(facecolor='white', edgecolor=edgecolor, boxstyle='round,pad=0.2')) + rectnum=rectnum+1 + + + if t.tagType == TAG_TYPE.BARBOX: + # stack boxplot data - shift upwards by last avg + meanpointprops = dict(marker='o', markeredgecolor='black', markerfacecolor='black') + r = plt.boxplot(offsetDataByLabel, positions=range(len(labels)), showmeans=True, meanprops=meanpointprops) + + # need to parse returned means because they dont include outliers + whysExcludingOutliers = [] + for i in range(len(labels)): + whysExcludingOutliers.append(r['means'][i].get_ydata()[0] - lastPlotBottom[i]) + + plt.bar(range(len(whysExcludingOutliers)), whysExcludingOutliers, width, bottom=lastPlotBottom, + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None), + edgecolor='black', linewidth=1, hatch=options.hatching) + lastPlotBottom = list( map(add, lastPlotBottom, whys) )# for stacking using average + if len(tags) > 1 and not skipLegend: + plt.legend() + + if options.print_raw: + print(labels) + np.set_printoptions(threshold=sys.maxsize) + print(whys) + + if options.rotate_x_labels: + plt.xticks(range(len(labels)), labels, rotation=45) + else: + if options.horizontal: + plt.yticks(range(len(labels)), labels) + else: + plt.xticks(range(len(labels)), labels) + + + elif t.tagType == TAG_TYPE.THREEDBAR or t.tagType == TAG_TYPE.THREEDSURF or t.tagType == TAG_TYPE.THREEDPROJECTX or t.tagType == TAG_TYPE.THREEDPROJECTY or t.tagType == TAG_TYPE.THREEDUNROLL: + if t.tagType == TAG_TYPE.THREEDBAR or t.tagType == TAG_TYPE.THREEDSURF: + ax = plt.gca(projection = '3d') + + #plt.rcParams['legend.fontsize'] = 10 + xlabels = [] # vector of x-axis label strings + ylabels = [] # vector of y-axis label strings + + for row in data: + if len(row) < 7: + print("3d bar data bad format") + exit() + if row[3] == t.name: + if (row[4] not in xlabels and (options.xfilter == None or row[4] in options.xfilter)): + xlabels.append(row[4]) + if (row[5] not in ylabels): + ylabels.append(row[5]) + + if len(lastPlotBottom) == 0: + lastPlotBottom = [0]*len(xlabels)*len(ylabels) + elif len(lastPlotBottom) != len(xlabels)*len(ylabels): + print("ERROR - tags have different number of labels") + print(len(lastPlotBottom)) + print(xlabels) + print(ylabels) + exit() + + zDataByLabel = [[[] for x in range(len(ylabels))] for y in range(len(xlabels))] # vector of all z values, indexed by x/y label + zees = [[0 for x in range(len(ylabels))] for y in range(len(xlabels))] # average z value indexed by x/y labels + + for row in data: + if row[3] == t.name and (options.xfilter == None or row[4] in options.xfilter): + xi = xlabels.index(row[4]) + yi = ylabels.index(row[5]) + zDataByLabel[xi][yi].append(float(row[6])) + + for i in range(len(xlabels)): + for j in range(len(ylabels)): + zRow = zDataByLabel[i][j] + if len(zRow) != 0: + zees[i][j] = np.average(zRow) + + x = [] + y = [] + dz = [] + for i in range(len(xlabels)): + for j in range(len(ylabels)): + x.append(i); + y.append(j); + dz.append(zees[i][j]); + dx = np.ones(len(x)) + dy = np.ones(len(y)) + + if options.print_raw: + print(xlabels) + print(ylabels) + np.set_printoptions(threshold=sys.maxsize) + print(np.resize(dz, [len(xlabels), len(ylabels)])) + + ### Note - this should work: + ### surf = ax.bar3d(x, y, lastPlotBottom, dx, dy, dz, label=t.name) + ### But it looks terrible! + ### Plotting the 0 height bars breaks the rendering. + ### Now lets remove them carefully so we don't break lastPlotBottom + + filteredx = [] + filteredy = [] + filteredlpb = [] + filtereddz = [] + for i in range(len(xlabels)): + for j in range(len(ylabels)): + if zees[i][j] != 0: + filteredx.append(i) + filteredy.append(j) + filtereddz.append(zees[i][j]) + filteredlpb.append(lastPlotBottom[i*len(ylabels)+j]) + filtereddx = np.ones(len(filtereddz)) + filtereddy = np.ones(len(filtereddz)) + + if t.tagType == TAG_TYPE.THREEDBAR: + surf = ax.bar3d(filteredx, filteredy, filteredlpb, filtereddx, filtereddy, filtereddz, + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + #surf = ax.bar3d(x, y, lastPlotBottom, dx, dy, dz, label=t.name) + + # Default legend looks terrible + #surf._facecolors2d=surf._facecolors3d + #surf._edgecolors2d=surf._edgecolors3d + #plt.legend() + # Use this instead: https://stackoverflow.com/questions/5803015/how-to-create-a-legend-for-3d-bar-in-matplotlib + legendArtistProxyShapes.append(plt.Rectangle((0,0), 1, 1, + fc=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else plt.get_cmap("tab10")(tagIndex)))) + ax.set_zlabel(options.zlabel) + if options.azimuth: + ax.azim = options.azimuth + if options.elevation: + ax.elev = options.elevation + + plt.xticks(range(len(xlabels)), xlabels)#, rotation=45) + plt.yticks(range(len(ylabels)), ylabels)#, rotation=45) + + elif t.tagType == TAG_TYPE.THREEDSURF: + surf = ax.plot_trisurf(x, y, dz, cmap=cms[tagIndex+options.color_offset], linewidth=0, + antialiased=False, alpha=(tagIndex+1)/len(tags)) + clb = plt.colorbar(surf, shrink=0.5, aspect=14, + pad=(.16 if (tagIndex == len(tags)-1) else .04)) + clb.ax.set_title(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name) + # set_xlabel('hi') to put at bottom + ax.set_zlabel(options.zlabel) + plt.xticks(range(len(xlabels)), xlabels)#, rotation=45) + plt.yticks(range(len(ylabels)), ylabels)#, rotation=45) + plt.tight_layout()# dont move + skipLegend = True + + elif t.tagType == TAG_TYPE.THREEDPROJECTX: + if options.projection == None: + print("need projection command line option") + return + else: + sliced = [] + slicedLPB = [] + for i in range(len(xlabels)): + sliced.append(zees[i][options.projection]) + slicedLPB.append(lastPlotBottom[i * len(ylabels) + options.projection]) + + # Plot a line + plt.plot([float(i) for i in xlabels], sliced, + label=options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name, + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + + # Stacked Bars + #plt.bar([i for i in xlabels], sliced, bottom=slicedLPB, + # label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + # color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + + plt.xticks([float(i) for i in xlabels], xlabels)#, rotation=45) + + elif t.tagType == TAG_TYPE.THREEDPROJECTY: + if options.projection == None: + print("need projection command line option") + return + else: + # Plot a line + plt.plot([float(i) for i in ylabels], zees[options.projection], + label=options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name, + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + + # Stacked Bars + #plt.bar([i for i in xlabels], zees[options.projection], bottom=lastPlotBottom[options.projection*len(ylabels) : options.projection*len(ylabels)+len(ylabels)], # THIS MAY NOT BE RIGHT + # label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + # color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + + plt.xticks([float(i) for i in ylabels], ylabels)#, rotation=45) + + elif t.tagType == TAG_TYPE.THREEDUNROLL: + width = .3 # width of bar + numX = len(xlabels) + + + # insert spaces + spacedLPB = lastPlotBottom.copy() + spacedDz = dz.copy() + for i in range(len(ylabels)): + if i != 0 and i != len(ylabels): + spacedDz.insert((i*len(ylabels)) + i - 1, 0) + spacedLPB.insert((i * len(ylabels)) + i - 1, 0) + + plt.bar(range(len(spacedDz)), spacedDz, bottom=spacedLPB, + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + if len(tags) > 1 and not skipLegend: + plt.legend() # bar always gets legend, must be after plotting + + xylabels = [] + for i in range(len(xlabels)): + if i != 0: + xylabels.append('') + for j in range(len(ylabels)): + xylabels.append(str(xlabels[i]) + ' - ' + str(int(ylabels[j]))) + plt.xticks(range(len(xylabels)), xylabels, rotation=45) + + lastPlotBottom = list( map(add, lastPlotBottom, dz) )# for stacking using average + + + tagIndex += 1; + + if len(legendArtistProxyShapes) > 1 and not skipLegend: # 3dbar plot = special legend + plt.legend(legendArtistProxyShapes, + options.custom_legend_labels if options.custom_legend_labels != None else tagLabels) + elif options.custom_legend_labels != None and len(options.custom_legend_labels) > 1 and not skipLegend: + if options.horizontal: + handles, labels = plt.gca().get_legend_handles_labels() + plt.legend(reversed(handles), reversed(labels)) + else: + + # combine two ax into single legend + all_lines = [] + all_labels = [] + for a in plt.gcf().get_axes(): + lines, labels = a.get_legend_handles_labels() + all_lines += lines + all_labels += labels + + plt.legend(all_lines, all_labels, loc = 9) + + #plt.gca().spines.values()[0].set_edgecolor('green') + side=0 + for spine in plt.gca().spines.values(): + if side == 0: + spine.set_edgecolor(dracula.COLORS[0]) + if side == 1: + spine.set_edgecolor(dracula.COLORS[1]) + side += 1 + + + + if options.horizontal: + plt.ylabel(options.xlabel) + plt.xlabel(options.ylabel) + else: + plt.xlabel(options.xlabel) + #plt.ylabel(options.ylabel) + plt.title(options.title) + plt.tight_layout() + + # hack for snail + # if "EMP_MUL" in tagLabels: + # plt.subplots_adjust(left=0.18) + # ax = plt.gca() + # ax2 = ax.twinx() + # for r in ax.patches[len(df):]: + # r.set_transform(ax2.transData) + # ax2.set_ylim(0, 2); + + + if options.show: + plt.show() + if options.log_scale: + if options.horizontal: + plt.gca().set_xscale('log') + else: + plt.gca().set_yscale('log') + plt.savefig(options.graphpath) + print("saved figure to " + options.graphpath) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="JLog plotter") + parser.add_argument('--csvlog', + required=True, + nargs="+", + help=("Path to log file.")) + parser.add_argument('--graphpath', + required=True, + help=("Path to store output graphs.")) + parser.add_argument('--title', + required=False, + default="", + help=("Graph Title.")) + parser.add_argument('--xlabel', + required=False, + default="", + help=("Label for plot x axis.")) + parser.add_argument('--ylabel', + required=False, + default="", + help=("Label for plot y axis.")) + parser.add_argument('--zlabel', + required=False, + default="", + help=("Label for plot z axis.")) + parser.add_argument('--only-tags', + required=False, + nargs="+", + help=("Only plot specific (space seperated) tags in csv log files. Default plots all tags.")) + parser.add_argument('--custom-legend-labels', + required=False, + nargs="+", + help=("Attach a custom legend label to each tag's plot.")) + parser.add_argument('--color-theme', + required=False, + help=("Use a custom color theme - dracula suported")) + parser.add_argument('--color-offset', + required=False, + type=int, + default=0, + help=("Offset starting color in color theme")) + parser.add_argument('--projection', + required=False, + type=int, + help=("Index of x/y projection for 3d to 2d projections")) + parser.add_argument('--azimuth', + required=False, + type=int, + help=("Azimuth sets view for 3D plot")) + parser.add_argument('--elevation', + required=False, + type=int, + help=("Elevation sets view for 3D plot")) + parser.add_argument('--fig-w', + required=False, + type=float, + default=5, + help=("Override figure width")) + parser.add_argument('--fig-h', + required=False, + type=float, + default=4, + help=("Override figure height")) + parser.add_argument('--xfilter', + required=False, + nargs="+", + help=("Only use data which match one of these x values.")) + parser.add_argument('--rotate_x_labels', + required=False, + action="store_true", + help=("Rotate x axis labels.")) + parser.add_argument('--hide-legend', + required=False, + action="store_true", + help=("Hide legend.")) + parser.add_argument('--bar-label-dict', + required=False, + type=json.loads, + help=("Add labels to bars e.g. '{\"tagName\": [\"l1\", \"l2\", \"l3\"]}' ")) + parser.add_argument('--hatching', + required=False, + help=("Add hatching to bars e.g. \"||||\"")) + parser.add_argument('--show', + required=False, + action="store_true", + help=("Show plots as they are generated.")) + parser.add_argument('--print-raw', + required=False, + action="store_true", + help=("Print raw plotted data to command line.")) + parser.add_argument('--log-scale', + required=False, + action="store_true", + help=("Log scale.")) + parser.add_argument('--horizontal', + required=False, + action="store_true", + help=("Plot bar graphs horizontally.")) + options = parser.parse_args() + + try: + csvFilePaths = options.csvlog + for p in csvFilePaths: + with open(p, "r") as f: + print("found file " + p) + except FileNotFoundError: + print("file not found " + p) + exit() + + if options.only_tags != None: + print("Only plotting specific tags: ") + print(options.only_tags) + + tags = getTags(csvFilePaths, options.only_tags) + + if tags == None or len(tags) == 0: + print("ERROR - No tags found - Nothing to plot") + exit() + + if options.custom_legend_labels != None: + if len(options.custom_legend_labels) == len(tags): + print("Custom Tag Labels Enabled") + else: + print("Wrong Number of Custom Tag Labels") + print(options.custom_legend_labels) + print(tags) + exit() + + plot(csvFilePaths, options, tags) + + if os.path.exists(tmpFile): + os.remove(tmpFile) diff --git a/scripts/plotter-netio.py b/scripts/plotter-netio.py new file mode 100644 index 0000000..4a9c21b --- /dev/null +++ b/scripts/plotter-netio.py @@ -0,0 +1,748 @@ +import matplotlib.pyplot as plt +from mpl_toolkits.mplot3d import Axes3D +from matplotlib import cm +import matplotlib.colors +import numpy as np +from operator import add +import os +import argparse +import enum +import sys +import json +import re + +sentinalStr = "SeNtInAl" +tmpFile = "plotlines.tmp" +maxColumns = 7 + +# Plotter cheat sheet +# +# XY +# SeNtInAl, tag_type, user_defined, tag_name, x, y +# +# +# BAR | GROUPED_BAR | BARBOX | BOX +# SeNtInAl, tag_type, user_defined, tag_name, x_label, y +# +# tag_names - plotting multiple will create segmented bars with +# a legend. Legend can be overridden with --custom-legend-labels +# x_label - string used to denote X +# y - float value +# +# THREEDBAR +# SeNtInAl, tag_type, z, tag_name, x_label, y_label +# tag_names - plotting multiple will create segmented bars with +# a legend. Legend can be overridden with --custom-legend-labels +# x_label - string used to denote X +# y_label - string used to denote Y +# z - float + + + +class TAG_TYPE(enum.Enum): + UNDEFINED = 0 + XY = 1 + HISTOGRAM = 2 + CDF = 3 + BAR = 4 + BOX = 5 + BARBOX = 6 + GROUPED_BAR = 7 + THREEDBAR = 8 + THREEDSURF = 9 + THREEDPROJECTX = 10 + THREEDPROJECTY = 11 + THREEDUNROLL = 12 + +#class dracula(): +# CYAN = '#8be9fd' +# GREEN = '#50fa7b' +# ORANGE = '#ffb86c' +# PINK = '#ff79c6' +# PURPLE = '#bd93f9' +# RED = '#ff5555' +# YELLOW = '#f1fa8c' +# COLORS = [CYAN, ORANGE, GREEN, PINK, PURPLE, YELLOW, RED] + +class dracula(): + ZERO = '#D291AC' + ONE = '#779E44' + TWO = '#F6B7AA' + THREE = '#367E51' + FOUR = '#5F1333' + FIVE = '#426615' + SIX = '#AD5B4A' + #COLORS = [ZERO, ONE, TWO, THREE, FOUR, FIVE, SIX, ZERO, ONE, TWO, THREE, FOUR] + COLORS = [plt.cm.Pastel1(i) for i in range(9)] + + + +#cmap1 = matplotlib.colors.LinearSegmentedColormap.from_list("", [dracula.PINK, dracula.PURPLE]) +#cmap2 = matplotlib.colors.LinearSegmentedColormap.from_list("", [dracula.CYAN, dracula.ORANGE]) + +cmap1 = matplotlib.colors.LinearSegmentedColormap.from_list("", [dracula.ZERO, dracula.TWO]) +cmap2 = matplotlib.colors.LinearSegmentedColormap.from_list("", [dracula.FOUR, dracula.SIX]) + +#cms = [cm.inferno, cm.viridis] +#cms = [cm.inferno, cm.cool] +#cms = [cm.copper, cm.summer] +cms = [cmap1, cmap2] + + +class tag: + def __init__(self): + self.tagType = TAG_TYPE.UNDEFINED + self.name = "" + + + +def getTags(csvFilePaths, tagWhitelist): + tags = [] + + printlogwarning = True; + with open(tmpFile, "w") as aggregateFile: + for p in csvFilePaths: + with open(p, "r") as csvIn: + lines = csvIn.readlines() + + for line in lines: + if line[:len(sentinalStr)] == sentinalStr: + tokens = line.split(',') + + if len(tokens) < 6 or len(tokens) > 8: + if printlogwarning: + print("WARNING - Malformed log line") + print(line) + printlogwarning = False + continue + + # strip trailing \n, add right amount of commas, add \n back + line = line.rstrip() + (','*(maxColumns - len(tokens))) + '\n' + aggregateFile.write(line) + + if tokens[3] == "error": + print("WARNING - OVERFLOW ERROR IN LOG FILE") + print(line) + continue + + whitelisted = tagWhitelist == None + if not whitelisted: + for whitelistedTag in tagWhitelist: + whitelisted = whitelisted or re.match('^'+whitelistedTag+'$', tokens[3]) + if not whitelisted: + continue + + newTag = True + for t in tags: + if t.name == tokens[3]: # must be exact match to be old tag + newTag = False + break + + if newTag and whitelisted: + print("found new tag: " + tokens[3]) + t = tag() + if tokens[1] == "xy": + t.tagType = TAG_TYPE.XY + if tokens[1] == "box": + t.tagType = TAG_TYPE.BOX + elif tokens[1] == "histogram": + t.tagType = TAG_TYPE.HISTOGRAM + elif tokens[1] == "cdf": + t.tagType = TAG_TYPE.CDF + elif tokens[1] == "bar": + t.tagType = TAG_TYPE.BAR + elif tokens[1] == "grouped_bar": + t.tagType = TAG_TYPE.GROUPED_BAR + elif tokens[1] == "barbox": + t.tagType = TAG_TYPE.BARBOX + elif tokens[1] == "3dbar": + t.tagType = TAG_TYPE.THREEDBAR + elif tokens[1] == "3dsurf": + t.tagType = TAG_TYPE.THREEDSURF + elif tokens[1] == "3dprojectx": + t.tagType = TAG_TYPE.THREEDPROJECTX + elif tokens[1] == "3dprojecty": + t.tagType = TAG_TYPE.THREEDPROJECTY + elif tokens[1] == "3dunroll": + t.tagType = TAG_TYPE.THREEDUNROLL + t.name = tokens[3] + tags.append(t) + + # make sure tags in same order as tagWhitelist + if tagWhitelist != None: + newTags = [] + for tname in tagWhitelist: + for t in tags: + if re.match('^'+tname+'$', t.name): + newTags.append(t) + continue + return newTags + else: + return tags + + + +def plot(csvFilePaths, options, tags): + data = np.genfromtxt(tmpFile, dtype=None, invalid_raise = False, + delimiter=',', encoding=None, filling_values="0", + names="sentinal, type, func, tag, x, y, z") + + tagIndex = 0; + tagLabels = [] + lastPlotBottom = [] + legendArtistProxyShapes=[] + skipLegend = options.hide_legend + + for t in tags: + tagLabels.append(t.name) + + for t in tags: + print("plotting tag: " + t.name) + + plt.figure(0, figsize=(options.fig_w, options.fig_h), dpi=800) + if t.tagType == TAG_TYPE.XY: + exes = [] + whys = [] + for row in data: + if row[3] == t.name and (options.xfilter == None or row[4] in options.xfilter): + exes.append(float(row[4])); + whys.append(float(row[5])); + # decorate, sort, then undecorate + exes, whys = (np.array(t) for t in zip(*sorted(zip(exes, whys)))) + # average duplicate values + dedup_exes = np.unique(exes) + dedup_whys = np.empty(dedup_exes.shape) + for i, x in enumerate(dedup_exes): + dedup_whys[i] = np.mean(whys[exes == x]) # / 1e9 + + dedup_whys /= 1e9 + + if "per_feat" in t.name: + try: + ax2 + except NameError: + ax2 = plt.gca().twinx() + ax2.plot(dedup_exes, dedup_whys, + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + plt.ylabel("GB per Iteration per Feature") + else: + plt.plot(dedup_exes, dedup_whys, + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + plt.ylabel("GB per Iteration") + + elif t.tagType == TAG_TYPE.HISTOGRAM: + print("plotting histogram tag: " + t.name) + print("TODO!") + + elif t.tagType == TAG_TYPE.BAR or t.tagType == TAG_TYPE.GROUPED_BAR or t.tagType == TAG_TYPE.BARBOX or t.tagType == TAG_TYPE.BOX: + labels = [] # vector of x-axis label strings + dataByLabel = [] # vector of all y values, indexed by label + offsetDataByLabel = [] # dataByLable, stacked + whys = [] # average y value indexed by label + width = 0.7 # width of a group of bars for grouped bar graph + + # collect labels for all tags + for row in data: + if len(row) < 6: + print("bar|grouped_bar|box|barbox data bad format") + exit() + for tagLabel in tagLabels: + if row[3] == tagLabel: + if row[4] not in labels and (options.xfilter == None or row[4] in options.xfilter): + labels.append(row[4]) + dataByLabel.append([]) + offsetDataByLabel.append([]) + + if len(lastPlotBottom) == 0: + lastPlotBottom = [0]*len(labels) + + # collect data for this tag + for row in data: + if row[3] == t.name and (options.xfilter == None or row[4] in options.xfilter): + try: + i = labels.index(row[4]) + dataByLabel[i].append(float(row[5])) + offsetDataByLabel[i].append(float(row[5]) + lastPlotBottom[i]) + except: + continue + for row in dataByLabel: + if len(row) != 0: + whys.append(np.average(row)) + else: + whys.append(0) + + if t.tagType == TAG_TYPE.BOX: + # no label on box plots + plt.boxplot(dataByLabel, positions=range(len(labels)))#, showmeans=True) + + edgecolor = matplotlib.colors.colorConverter.to_rgba('black', alpha=0.1) + + if t.tagType == TAG_TYPE.BAR: + if options.horizontal: + plt.barh(range(len(whys)), whys, bottom=lastPlotBottom, + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + else: + plt.bar(range(len(whys)), whys, bottom=lastPlotBottom, + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + if len(tags) > 1 and not skipLegend: + plt.legend() # bar always gets legend, must be after plotting + lastPlotBottom = list( map(add, lastPlotBottom, whys) )# for stacking using average + + if t.tagType == TAG_TYPE.GROUPED_BAR: + offset=len(tags)/2*width/5 + if options.horizontal: + rects = plt.barh(np.arange(len(whys)) - offset + (width*tagIndex/len(tags)), whys, width/len(tags), + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None), + edgecolor=edgecolor, linewidth=1, hatch=options.hatching) + else: + rects = plt.bar(np.arange(len(whys)) - offset + (width*tagIndex/len(tags)), whys, width/len(tags), + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None), + edgecolor=edgecolor, linewidth=1, hatch=options.hatching) + if len(tags) > 1 and not skipLegend: + plt.legend() # bar always gets legend, must be after plotting + lastPlotBottom = list( map(add, lastPlotBottom, whys) )# for stacking using average + + # label rectangles + if options.bar_label_dict and t.name in options.bar_label_dict: + if len(options.bar_label_dict[t.name]) != len(whys): + print('Error - wrong number of bar labels - %d (should be %d)\n\n\n' % (len(options.bar_label_dict[t.name]), len(whys)) ) + return + rectnum=0 + for rect in rects: + height = rect.get_height() + plt.text(rect.get_x() + rect.get_width()/2., 1.05*height, options.bar_label_dict[t.name][rectnum], + ha='center', va='bottom', + bbox=dict(facecolor='white', edgecolor=edgecolor, boxstyle='round,pad=0.2')) + rectnum=rectnum+1 + + + if t.tagType == TAG_TYPE.BARBOX: + # stack boxplot data - shift upwards by last avg + meanpointprops = dict(marker='o', markeredgecolor='black', markerfacecolor='black') + r = plt.boxplot(offsetDataByLabel, positions=range(len(labels)), showmeans=True, meanprops=meanpointprops) + + # need to parse returned means because they dont include outliers + whysExcludingOutliers = [] + for i in range(len(labels)): + whysExcludingOutliers.append(r['means'][i].get_ydata()[0] - lastPlotBottom[i]) + + plt.bar(range(len(whysExcludingOutliers)), whysExcludingOutliers, width, bottom=lastPlotBottom, + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None), + edgecolor='black', linewidth=1, hatch=options.hatching) + lastPlotBottom = list( map(add, lastPlotBottom, whys) )# for stacking using average + if len(tags) > 1 and not skipLegend: + plt.legend() + + if options.print_raw: + print(labels) + np.set_printoptions(threshold=sys.maxsize) + print(whys) + + if options.rotate_x_labels: + plt.xticks(range(len(labels)), labels, rotation=45) + else: + if options.horizontal: + plt.yticks(range(len(labels)), labels) + else: + plt.xticks(range(len(labels)), labels) + + + elif t.tagType == TAG_TYPE.THREEDBAR or t.tagType == TAG_TYPE.THREEDSURF or t.tagType == TAG_TYPE.THREEDPROJECTX or t.tagType == TAG_TYPE.THREEDPROJECTY or t.tagType == TAG_TYPE.THREEDUNROLL: + if t.tagType == TAG_TYPE.THREEDBAR or t.tagType == TAG_TYPE.THREEDSURF: + ax = plt.gca(projection = '3d') + + #plt.rcParams['legend.fontsize'] = 10 + xlabels = [] # vector of x-axis label strings + ylabels = [] # vector of y-axis label strings + + for row in data: + if len(row) < 7: + print("3d bar data bad format") + exit() + if row[3] == t.name: + if (row[4] not in xlabels and (options.xfilter == None or row[4] in options.xfilter)): + xlabels.append(row[4]) + if (row[5] not in ylabels): + ylabels.append(row[5]) + + if len(lastPlotBottom) == 0: + lastPlotBottom = [0]*len(xlabels)*len(ylabels) + elif len(lastPlotBottom) != len(xlabels)*len(ylabels): + print("ERROR - tags have different number of labels") + print(len(lastPlotBottom)) + print(xlabels) + print(ylabels) + exit() + + zDataByLabel = [[[] for x in range(len(ylabels))] for y in range(len(xlabels))] # vector of all z values, indexed by x/y label + zees = [[0 for x in range(len(ylabels))] for y in range(len(xlabels))] # average z value indexed by x/y labels + + for row in data: + if row[3] == t.name and (options.xfilter == None or row[4] in options.xfilter): + xi = xlabels.index(row[4]) + yi = ylabels.index(row[5]) + zDataByLabel[xi][yi].append(float(row[6])) + + for i in range(len(xlabels)): + for j in range(len(ylabels)): + zRow = zDataByLabel[i][j] + if len(zRow) != 0: + zees[i][j] = np.average(zRow) + + x = [] + y = [] + dz = [] + for i in range(len(xlabels)): + for j in range(len(ylabels)): + x.append(i); + y.append(j); + dz.append(zees[i][j]); + dx = np.ones(len(x)) + dy = np.ones(len(y)) + + if options.print_raw: + print(xlabels) + print(ylabels) + np.set_printoptions(threshold=sys.maxsize) + print(np.resize(dz, [len(xlabels), len(ylabels)])) + + ### Note - this should work: + ### surf = ax.bar3d(x, y, lastPlotBottom, dx, dy, dz, label=t.name) + ### But it looks terrible! + ### Plotting the 0 height bars breaks the rendering. + ### Now lets remove them carefully so we don't break lastPlotBottom + + filteredx = [] + filteredy = [] + filteredlpb = [] + filtereddz = [] + for i in range(len(xlabels)): + for j in range(len(ylabels)): + if zees[i][j] != 0: + filteredx.append(i) + filteredy.append(j) + filtereddz.append(zees[i][j]) + filteredlpb.append(lastPlotBottom[i*len(ylabels)+j]) + filtereddx = np.ones(len(filtereddz)) + filtereddy = np.ones(len(filtereddz)) + + if t.tagType == TAG_TYPE.THREEDBAR: + surf = ax.bar3d(filteredx, filteredy, filteredlpb, filtereddx, filtereddy, filtereddz, + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + #surf = ax.bar3d(x, y, lastPlotBottom, dx, dy, dz, label=t.name) + + # Default legend looks terrible + #surf._facecolors2d=surf._facecolors3d + #surf._edgecolors2d=surf._edgecolors3d + #plt.legend() + # Use this instead: https://stackoverflow.com/questions/5803015/how-to-create-a-legend-for-3d-bar-in-matplotlib + legendArtistProxyShapes.append(plt.Rectangle((0,0), 1, 1, + fc=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else plt.get_cmap("tab10")(tagIndex)))) + ax.set_zlabel(options.zlabel) + if options.azimuth: + ax.azim = options.azimuth + if options.elevation: + ax.elev = options.elevation + + plt.xticks(range(len(xlabels)), xlabels)#, rotation=45) + plt.yticks(range(len(ylabels)), ylabels)#, rotation=45) + + elif t.tagType == TAG_TYPE.THREEDSURF: + surf = ax.plot_trisurf(x, y, dz, cmap=cms[tagIndex+options.color_offset], linewidth=0, + antialiased=False, alpha=(tagIndex+1)/len(tags)) + clb = plt.colorbar(surf, shrink=0.5, aspect=14, + pad=(.16 if (tagIndex == len(tags)-1) else .04)) + clb.ax.set_title(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name) + # set_xlabel('hi') to put at bottom + ax.set_zlabel(options.zlabel) + plt.xticks(range(len(xlabels)), xlabels)#, rotation=45) + plt.yticks(range(len(ylabels)), ylabels)#, rotation=45) + plt.tight_layout()# dont move + skipLegend = True + + elif t.tagType == TAG_TYPE.THREEDPROJECTX: + if options.projection == None: + print("need projection command line option") + return + else: + sliced = [] + slicedLPB = [] + for i in range(len(xlabels)): + sliced.append(zees[i][options.projection]) + slicedLPB.append(lastPlotBottom[i * len(ylabels) + options.projection]) + + # Plot a line + plt.plot([float(i) for i in xlabels], sliced, + label=options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name, + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + + # Stacked Bars + #plt.bar([i for i in xlabels], sliced, bottom=slicedLPB, + # label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + # color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + + plt.xticks([float(i) for i in xlabels], xlabels)#, rotation=45) + + elif t.tagType == TAG_TYPE.THREEDPROJECTY: + if options.projection == None: + print("need projection command line option") + return + else: + # Plot a line + plt.plot([float(i) for i in ylabels], zees[options.projection], + label=options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name, + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + + # Stacked Bars + #plt.bar([i for i in xlabels], zees[options.projection], bottom=lastPlotBottom[options.projection*len(ylabels) : options.projection*len(ylabels)+len(ylabels)], # THIS MAY NOT BE RIGHT + # label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + # color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + + plt.xticks([float(i) for i in ylabels], ylabels)#, rotation=45) + + elif t.tagType == TAG_TYPE.THREEDUNROLL: + width = .3 # width of bar + numX = len(xlabels) + + + # insert spaces + spacedLPB = lastPlotBottom.copy() + spacedDz = dz.copy() + for i in range(len(ylabels)): + if i != 0 and i != len(ylabels): + spacedDz.insert((i*len(ylabels)) + i - 1, 0) + spacedLPB.insert((i * len(ylabels)) + i - 1, 0) + + plt.bar(range(len(spacedDz)), spacedDz, bottom=spacedLPB, + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + if len(tags) > 1 and not skipLegend: + plt.legend() # bar always gets legend, must be after plotting + + xylabels = [] + for i in range(len(xlabels)): + if i != 0: + xylabels.append('') + for j in range(len(ylabels)): + xylabels.append(str(xlabels[i]) + ' - ' + str(int(ylabels[j]))) + plt.xticks(range(len(xylabels)), xylabels, rotation=45) + + lastPlotBottom = list( map(add, lastPlotBottom, dz) )# for stacking using average + + + tagIndex += 1; + + if len(legendArtistProxyShapes) > 1 and not skipLegend: # 3dbar plot = special legend + plt.legend(legendArtistProxyShapes, + options.custom_legend_labels if options.custom_legend_labels != None else tagLabels) + elif options.custom_legend_labels != None and len(options.custom_legend_labels) > 1 and not skipLegend: + if options.horizontal: + handles, labels = plt.gca().get_legend_handles_labels() + plt.legend(reversed(handles), reversed(labels)) + else: + # combine two ax into single legend + all_lines = [] + all_labels = [] + for a in plt.gcf().get_axes(): + lines, labels = a.get_legend_handles_labels() + all_lines += lines + all_labels += labels + + plt.legend(all_lines, all_labels, loc = 9) + + + xlims = plt.xlim() + ylims = plt.ylim() + ylims = ylims[0], ylims[1]+0.049 # make extra room for legend + + # side by side bars on y axis + # plt.vlines(xlims[0], ylims[0], ylims[1], color=dracula.COLORS[0], lw=1, zorder=4, clip_on=False) + # plt.vlines(xlims[0]+1.4, ylims[0], ylims[1], color=dracula.COLORS[1], lw=1, zorder=4, clip_on=False) + # plt.vlines(xlims[1], ylims[0], ylims[1], color=dracula.COLORS[2], lw=1, zorder=4, clip_on=False) + # plt.vlines(xlims[1]-1.4, ylims[0], ylims[1], color=dracula.COLORS[3], lw=1, zorder=4, clip_on=False) + + # split vertical + midpt = (ylims[1] - ylims[0])/1.4 # magic + plt.vlines(xlims[0], ylims[0], midpt, color=dracula.COLORS[1], lw=1, zorder=4, clip_on=False) + plt.vlines(xlims[0], midpt, ylims[1], color=dracula.COLORS[0], lw=1, zorder=4, clip_on=False) + plt.vlines(xlims[1], ylims[0], midpt, color=dracula.COLORS[3], lw=1, zorder=4, clip_on=False) + plt.vlines(xlims[1], midpt, ylims[1], color=dracula.COLORS[2], lw=1, zorder=4, clip_on=False) + + plt.xlim(xlims) + plt.ylim(ylims) + + + #plt.gca().spines.values()[0].set_edgecolor('green') + #side=0 + #for spine in plt.gca().spines.values(): + # if side == 0: + # spine.set_edgecolor(dracula.COLORS[0], dracula.COLORS[1]) + # if side == 1: + # spine.set_edgecolor(dracula.COLORS[2]) + # side += 1 + + if options.horizontal: + #plt.ylabel(options.xlabel) + plt.xlabel(options.ylabel) + else: + plt.xlabel(options.xlabel) + #plt.ylabel(options.ylabel) + plt.title(options.title) + plt.tight_layout() + + # hack for snail + if "EMP_MUL" in tagLabels: + plt.subplots_adjust(left=0.18) + + + if options.show: + plt.show() + if options.log_scale: + if options.horizontal: + plt.gca().set_xscale('log') + else: + plt.gca().set_yscale('log') + plt.savefig(options.graphpath) + print("saved figure to " + options.graphpath) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="JLog plotter") + parser.add_argument('--csvlog', + required=True, + nargs="+", + help=("Path to log file.")) + parser.add_argument('--graphpath', + required=True, + help=("Path to store output graphs.")) + parser.add_argument('--title', + required=False, + default="", + help=("Graph Title.")) + parser.add_argument('--xlabel', + required=False, + default="", + help=("Label for plot x axis.")) + parser.add_argument('--ylabel', + required=False, + default="", + help=("Label for plot y axis.")) + parser.add_argument('--zlabel', + required=False, + default="", + help=("Label for plot z axis.")) + parser.add_argument('--only-tags', + required=False, + nargs="+", + help=("Only plot specific (space seperated) tags in csv log files. Default plots all tags.")) + parser.add_argument('--custom-legend-labels', + required=False, + nargs="+", + help=("Attach a custom legend label to each tag's plot.")) + parser.add_argument('--color-theme', + required=False, + help=("Use a custom color theme - dracula suported")) + parser.add_argument('--color-offset', + required=False, + type=int, + default=0, + help=("Offset starting color in color theme")) + parser.add_argument('--projection', + required=False, + type=int, + help=("Index of x/y projection for 3d to 2d projections")) + parser.add_argument('--azimuth', + required=False, + type=int, + help=("Azimuth sets view for 3D plot")) + parser.add_argument('--elevation', + required=False, + type=int, + help=("Elevation sets view for 3D plot")) + parser.add_argument('--fig-w', + required=False, + type=float, + default=5, + help=("Override figure width")) + parser.add_argument('--fig-h', + required=False, + type=float, + default=4, + help=("Override figure height")) + parser.add_argument('--xfilter', + required=False, + nargs="+", + help=("Only use data which match one of these x values.")) + parser.add_argument('--rotate_x_labels', + required=False, + action="store_true", + help=("Rotate x axis labels.")) + parser.add_argument('--hide-legend', + required=False, + action="store_true", + help=("Hide legend.")) + parser.add_argument('--bar-label-dict', + required=False, + type=json.loads, + help=("Add labels to bars e.g. '{\"tagName\": [\"l1\", \"l2\", \"l3\"]}' ")) + parser.add_argument('--hatching', + required=False, + help=("Add hatching to bars e.g. \"||||\"")) + parser.add_argument('--show', + required=False, + action="store_true", + help=("Show plots as they are generated.")) + parser.add_argument('--print-raw', + required=False, + action="store_true", + help=("Print raw plotted data to command line.")) + parser.add_argument('--log-scale', + required=False, + action="store_true", + help=("Log scale.")) + parser.add_argument('--horizontal', + required=False, + action="store_true", + help=("Plot bar graphs horizontally.")) + options = parser.parse_args() + + try: + csvFilePaths = options.csvlog + for p in csvFilePaths: + with open(p, "r") as f: + print("found file " + p) + except FileNotFoundError: + print("file not found " + p) + exit() + + if options.only_tags != None: + print("Only plotting specific tags: ") + print(options.only_tags) + + tags = getTags(csvFilePaths, options.only_tags) + + if tags == None or len(tags) == 0: + print("ERROR - No tags found - Nothing to plot") + exit() + + if options.custom_legend_labels != None: + if len(options.custom_legend_labels) == len(tags): + print("Custom Tag Labels Enabled") + else: + print("Wrong Number of Custom Tag Labels") + print(options.custom_legend_labels) + print(tags) + exit() + + plot(csvFilePaths, options, tags) + + if os.path.exists(tmpFile): + os.remove(tmpFile) diff --git a/scripts/plotter.py b/scripts/plotter.py new file mode 100644 index 0000000..747391a --- /dev/null +++ b/scripts/plotter.py @@ -0,0 +1,696 @@ +import matplotlib.pyplot as plt +from mpl_toolkits.mplot3d import Axes3D +from matplotlib import cm +import matplotlib.colors +import numpy as np +from operator import add +import os +import argparse +import enum +import sys +import json +import re + +sentinalStr = "SeNtInAl" +tmpFile = "plotlines.tmp" +maxColumns = 7 + +# Plotter cheat sheet +# +# XY +# SeNtInAl, tag_type, user_defined, tag_name, x, y +# +# +# BAR | GROUPED_BAR | BARBOX | BOX +# SeNtInAl, tag_type, user_defined, tag_name, x_label, y +# +# tag_names - plotting multiple will create segmented bars with +# a legend. Legend can be overridden with --custom-legend-labels +# x_label - string used to denote X +# y - float value +# +# THREEDBAR +# SeNtInAl, tag_type, z, tag_name, x_label, y_label +# tag_names - plotting multiple will create segmented bars with +# a legend. Legend can be overridden with --custom-legend-labels +# x_label - string used to denote X +# y_label - string used to denote Y +# z - float + + + +class TAG_TYPE(enum.Enum): + UNDEFINED = 0 + XY = 1 + HISTOGRAM = 2 + CDF = 3 + BAR = 4 + BOX = 5 + BARBOX = 6 + GROUPED_BAR = 7 + THREEDBAR = 8 + THREEDSURF = 9 + THREEDPROJECTX = 10 + THREEDPROJECTY = 11 + THREEDUNROLL = 12 + +#class dracula(): +# CYAN = '#8be9fd' +# GREEN = '#50fa7b' +# ORANGE = '#ffb86c' +# PINK = '#ff79c6' +# PURPLE = '#bd93f9' +# RED = '#ff5555' +# YELLOW = '#f1fa8c' +# COLORS = [CYAN, ORANGE, GREEN, PINK, PURPLE, YELLOW, RED] + +class dracula(): + ZERO = '#D291AC' + ONE = '#779E44' + TWO = '#F6B7AA' + THREE = '#367E51' + FOUR = '#5F1333' + FIVE = '#426615' + SIX = '#AD5B4A' + #COLORS = [ZERO, ONE, TWO, THREE, FOUR, FIVE, SIX, ZERO, ONE, TWO, THREE, FOUR] + COLORS = [plt.cm.Pastel1(i) for i in range(9)] + + + +#cmap1 = matplotlib.colors.LinearSegmentedColormap.from_list("", [dracula.PINK, dracula.PURPLE]) +#cmap2 = matplotlib.colors.LinearSegmentedColormap.from_list("", [dracula.CYAN, dracula.ORANGE]) + +cmap1 = matplotlib.colors.LinearSegmentedColormap.from_list("", [dracula.ZERO, dracula.TWO]) +cmap2 = matplotlib.colors.LinearSegmentedColormap.from_list("", [dracula.FOUR, dracula.SIX]) + +#cms = [cm.inferno, cm.viridis] +#cms = [cm.inferno, cm.cool] +#cms = [cm.copper, cm.summer] +cms = [cmap1, cmap2] + + +class tag: + def __init__(self): + self.tagType = TAG_TYPE.UNDEFINED + self.name = "" + + + +def getTags(csvFilePaths, tagWhitelist): + tags = [] + + printlogwarning = True; + with open(tmpFile, "w") as aggregateFile: + for p in csvFilePaths: + with open(p, "r") as csvIn: + lines = csvIn.readlines() + + for line in lines: + if line[:len(sentinalStr)] == sentinalStr: + tokens = line.split(',') + + if len(tokens) < 6 or len(tokens) > 8: + if printlogwarning: + print("WARNING - Malformed log line") + print(line) + printlogwarning = False + continue + + # strip trailing \n, add right amount of commas, add \n back + line = line.rstrip() + (','*(maxColumns - len(tokens))) + '\n' + aggregateFile.write(line) + + if tokens[3] == "error": + print("WARNING - OVERFLOW ERROR IN LOG FILE") + print(line) + continue + + whitelisted = tagWhitelist == None + if not whitelisted: + for whitelistedTag in tagWhitelist: + whitelisted = whitelisted or re.match('^'+whitelistedTag+'$', tokens[3]) + if not whitelisted: + continue + + newTag = True + for t in tags: + if t.name == tokens[3]: # must be exact match to be old tag + newTag = False + break + + if newTag and whitelisted: + print("found new tag: " + tokens[3]) + t = tag() + if tokens[1] == "xy": + t.tagType = TAG_TYPE.XY + if tokens[1] == "box": + t.tagType = TAG_TYPE.BOX + elif tokens[1] == "histogram": + t.tagType = TAG_TYPE.HISTOGRAM + elif tokens[1] == "cdf": + t.tagType = TAG_TYPE.CDF + elif tokens[1] == "bar": + t.tagType = TAG_TYPE.BAR + elif tokens[1] == "grouped_bar": + t.tagType = TAG_TYPE.GROUPED_BAR + elif tokens[1] == "barbox": + t.tagType = TAG_TYPE.BARBOX + elif tokens[1] == "3dbar": + t.tagType = TAG_TYPE.THREEDBAR + elif tokens[1] == "3dsurf": + t.tagType = TAG_TYPE.THREEDSURF + elif tokens[1] == "3dprojectx": + t.tagType = TAG_TYPE.THREEDPROJECTX + elif tokens[1] == "3dprojecty": + t.tagType = TAG_TYPE.THREEDPROJECTY + elif tokens[1] == "3dunroll": + t.tagType = TAG_TYPE.THREEDUNROLL + t.name = tokens[3] + tags.append(t) + + # make sure tags in same order as tagWhitelist + if tagWhitelist != None: + newTags = [] + for tname in tagWhitelist: + for t in tags: + if re.match('^'+tname+'$', t.name): + newTags.append(t) + continue + return newTags + else: + return tags + + + +def plot(csvFilePaths, options, tags): + data = np.genfromtxt(tmpFile, dtype=None, invalid_raise = False, + delimiter=',', encoding=None, filling_values="0", + names="sentinal, type, func, tag, x, y, z") + + tagIndex = 0; + tagLabels = [] + lastPlotBottom = [] + legendArtistProxyShapes=[] + skipLegend = options.hide_legend + + for t in tags: + tagLabels.append(t.name) + + for t in tags: + print("plotting tag: " + t.name) + + plt.figure(0, figsize=(options.fig_w, options.fig_h), dpi=800) + if t.tagType == TAG_TYPE.XY: + exes = [] + whys = [] + for row in data: + if row[3] == t.name and (options.xfilter == None or row[4] in options.xfilter): + exes.append(float(row[4])); + whys.append(float(row[5])); + # decorate, sort, then undecorate + exes, whys = (np.array(t) for t in zip(*sorted(zip(exes, whys)))) + # average duplicate values + dedup_exes = np.unique(exes) + dedup_whys = np.empty(dedup_exes.shape) + for i, x in enumerate(dedup_exes): + dedup_whys[i] = np.mean(whys[exes == x]) # / 1e9 + + plt.plot(dedup_exes, dedup_whys, + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + + elif t.tagType == TAG_TYPE.HISTOGRAM: + print("plotting histogram tag: " + t.name) + print("TODO!") + + elif t.tagType == TAG_TYPE.BAR or t.tagType == TAG_TYPE.GROUPED_BAR or t.tagType == TAG_TYPE.BARBOX or t.tagType == TAG_TYPE.BOX: + labels = [] # vector of x-axis label strings + dataByLabel = [] # vector of all y values, indexed by label + offsetDataByLabel = [] # dataByLable, stacked + whys = [] # average y value indexed by label + width = 0.7 # width of a group of bars for grouped bar graph + + # collect labels for all tags + for row in data: + if len(row) < 6: + print("bar|grouped_bar|box|barbox data bad format") + exit() + for tagLabel in tagLabels: + if row[3] == tagLabel: + if row[4] not in labels and (options.xfilter == None or row[4] in options.xfilter): + labels.append(row[4]) + dataByLabel.append([]) + offsetDataByLabel.append([]) + + if len(lastPlotBottom) == 0: + lastPlotBottom = [0]*len(labels) + + # collect data for this tag + for row in data: + if row[3] == t.name and (options.xfilter == None or row[4] in options.xfilter): + try: + i = labels.index(row[4]) + dataByLabel[i].append(float(row[5])) + offsetDataByLabel[i].append(float(row[5]) + lastPlotBottom[i]) + except: + continue + for row in dataByLabel: + if len(row) != 0: + whys.append(np.average(row)) + else: + whys.append(0) + + if t.tagType == TAG_TYPE.BOX: + # no label on box plots + plt.boxplot(dataByLabel, positions=range(len(labels)))#, showmeans=True) + + edgecolor = matplotlib.colors.colorConverter.to_rgba('black', alpha=0.1) + + if t.tagType == TAG_TYPE.BAR: + if options.horizontal: + plt.barh(range(len(whys)), whys, bottom=lastPlotBottom, + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + else: + plt.bar(range(len(whys)), whys, bottom=lastPlotBottom, + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + if len(tags) > 1 and not skipLegend: + plt.legend() # bar always gets legend, must be after plotting + lastPlotBottom = list( map(add, lastPlotBottom, whys) )# for stacking using average + + if t.tagType == TAG_TYPE.GROUPED_BAR: + offset=len(tags)/2*width/5 + if options.horizontal: + rects = plt.barh(np.arange(len(whys)) - offset + (width*tagIndex/len(tags)), whys, width/len(tags), + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None), + edgecolor=edgecolor, linewidth=1, hatch=options.hatching) + else: + rects = plt.bar(np.arange(len(whys)) - offset + (width*tagIndex/len(tags)), whys, width/len(tags), + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None), + edgecolor=edgecolor, linewidth=1, hatch=options.hatching) + if len(tags) > 1 and not skipLegend: + plt.legend() # bar always gets legend, must be after plotting + lastPlotBottom = list( map(add, lastPlotBottom, whys) )# for stacking using average + + # label rectangles + if options.bar_label_dict and t.name in options.bar_label_dict: + if len(options.bar_label_dict[t.name]) != len(whys): + print('Error - wrong number of bar labels - %d (should be %d)\n\n\n' % (len(options.bar_label_dict[t.name]), len(whys)) ) + return + rectnum=0 + for rect in rects: + height = rect.get_height() + plt.text(rect.get_x() + rect.get_width()/2., 1.05*height, options.bar_label_dict[t.name][rectnum], + ha='center', va='bottom', + bbox=dict(facecolor='white', edgecolor=edgecolor, boxstyle='round,pad=0.2')) + rectnum=rectnum+1 + + + if t.tagType == TAG_TYPE.BARBOX: + # stack boxplot data - shift upwards by last avg + meanpointprops = dict(marker='o', markeredgecolor='black', markerfacecolor='black') + r = plt.boxplot(offsetDataByLabel, positions=range(len(labels)), showmeans=True, meanprops=meanpointprops) + + # need to parse returned means because they dont include outliers + whysExcludingOutliers = [] + for i in range(len(labels)): + whysExcludingOutliers.append(r['means'][i].get_ydata()[0] - lastPlotBottom[i]) + + plt.bar(range(len(whysExcludingOutliers)), whysExcludingOutliers, width, bottom=lastPlotBottom, + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None), + edgecolor='black', linewidth=1, hatch=options.hatching) + lastPlotBottom = list( map(add, lastPlotBottom, whys) )# for stacking using average + if len(tags) > 1 and not skipLegend: + plt.legend() + + if options.print_raw: + print(labels) + np.set_printoptions(threshold=sys.maxsize) + print(whys) + + if options.rotate_x_labels: + plt.xticks(range(len(labels)), labels, rotation=45) + else: + if options.horizontal: + plt.yticks(range(len(labels)), labels) + else: + plt.xticks(range(len(labels)), labels) + + + elif t.tagType == TAG_TYPE.THREEDBAR or t.tagType == TAG_TYPE.THREEDSURF or t.tagType == TAG_TYPE.THREEDPROJECTX or t.tagType == TAG_TYPE.THREEDPROJECTY or t.tagType == TAG_TYPE.THREEDUNROLL: + if t.tagType == TAG_TYPE.THREEDBAR or t.tagType == TAG_TYPE.THREEDSURF: + ax = plt.gca(projection = '3d') + + #plt.rcParams['legend.fontsize'] = 10 + xlabels = [] # vector of x-axis label strings + ylabels = [] # vector of y-axis label strings + + for row in data: + if len(row) < 7: + print("3d bar data bad format") + exit() + if row[3] == t.name: + if (row[4] not in xlabels and (options.xfilter == None or row[4] in options.xfilter)): + xlabels.append(row[4]) + if (row[5] not in ylabels): + ylabels.append(row[5]) + + if len(lastPlotBottom) == 0: + lastPlotBottom = [0]*len(xlabels)*len(ylabels) + elif len(lastPlotBottom) != len(xlabels)*len(ylabels): + print("ERROR - tags have different number of labels") + print(len(lastPlotBottom)) + print(xlabels) + print(ylabels) + exit() + + zDataByLabel = [[[] for x in range(len(ylabels))] for y in range(len(xlabels))] # vector of all z values, indexed by x/y label + zees = [[0 for x in range(len(ylabels))] for y in range(len(xlabels))] # average z value indexed by x/y labels + + for row in data: + if row[3] == t.name and (options.xfilter == None or row[4] in options.xfilter): + xi = xlabels.index(row[4]) + yi = ylabels.index(row[5]) + zDataByLabel[xi][yi].append(float(row[6])) + + for i in range(len(xlabels)): + for j in range(len(ylabels)): + zRow = zDataByLabel[i][j] + if len(zRow) != 0: + zees[i][j] = np.average(zRow) + + x = [] + y = [] + dz = [] + for i in range(len(xlabels)): + for j in range(len(ylabels)): + x.append(i); + y.append(j); + dz.append(zees[i][j]); + dx = np.ones(len(x)) + dy = np.ones(len(y)) + + if options.print_raw: + print(xlabels) + print(ylabels) + np.set_printoptions(threshold=sys.maxsize) + print(np.resize(dz, [len(xlabels), len(ylabels)])) + + ### Note - this should work: + ### surf = ax.bar3d(x, y, lastPlotBottom, dx, dy, dz, label=t.name) + ### But it looks terrible! + ### Plotting the 0 height bars breaks the rendering. + ### Now lets remove them carefully so we don't break lastPlotBottom + + filteredx = [] + filteredy = [] + filteredlpb = [] + filtereddz = [] + for i in range(len(xlabels)): + for j in range(len(ylabels)): + if zees[i][j] != 0: + filteredx.append(i) + filteredy.append(j) + filtereddz.append(zees[i][j]) + filteredlpb.append(lastPlotBottom[i*len(ylabels)+j]) + filtereddx = np.ones(len(filtereddz)) + filtereddy = np.ones(len(filtereddz)) + + if t.tagType == TAG_TYPE.THREEDBAR: + surf = ax.bar3d(filteredx, filteredy, filteredlpb, filtereddx, filtereddy, filtereddz, + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + #surf = ax.bar3d(x, y, lastPlotBottom, dx, dy, dz, label=t.name) + + # Default legend looks terrible + #surf._facecolors2d=surf._facecolors3d + #surf._edgecolors2d=surf._edgecolors3d + #plt.legend() + # Use this instead: https://stackoverflow.com/questions/5803015/how-to-create-a-legend-for-3d-bar-in-matplotlib + legendArtistProxyShapes.append(plt.Rectangle((0,0), 1, 1, + fc=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else plt.get_cmap("tab10")(tagIndex)))) + ax.set_zlabel(options.zlabel) + if options.azimuth: + ax.azim = options.azimuth + if options.elevation: + ax.elev = options.elevation + + plt.xticks(range(len(xlabels)), xlabels)#, rotation=45) + plt.yticks(range(len(ylabels)), ylabels)#, rotation=45) + + elif t.tagType == TAG_TYPE.THREEDSURF: + surf = ax.plot_trisurf(x, y, dz, cmap=cms[tagIndex+options.color_offset], linewidth=0, + antialiased=False, alpha=(tagIndex+1)/len(tags)) + clb = plt.colorbar(surf, shrink=0.5, aspect=14, + pad=(.16 if (tagIndex == len(tags)-1) else .04)) + clb.ax.set_title(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name) + # set_xlabel('hi') to put at bottom + ax.set_zlabel(options.zlabel) + plt.xticks(range(len(xlabels)), xlabels)#, rotation=45) + plt.yticks(range(len(ylabels)), ylabels)#, rotation=45) + plt.tight_layout()# dont move + skipLegend = True + + elif t.tagType == TAG_TYPE.THREEDPROJECTX: + if options.projection == None: + print("need projection command line option") + return + else: + sliced = [] + slicedLPB = [] + for i in range(len(xlabels)): + sliced.append(zees[i][options.projection]) + slicedLPB.append(lastPlotBottom[i * len(ylabels) + options.projection]) + + # Plot a line + plt.plot([float(i) for i in xlabels], sliced, + label=options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name, + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + + # Stacked Bars + #plt.bar([i for i in xlabels], sliced, bottom=slicedLPB, + # label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + # color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + + plt.xticks([float(i) for i in xlabels], xlabels)#, rotation=45) + + elif t.tagType == TAG_TYPE.THREEDPROJECTY: + if options.projection == None: + print("need projection command line option") + return + else: + # Plot a line + plt.plot([float(i) for i in ylabels], zees[options.projection], + label=options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name, + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + + # Stacked Bars + #plt.bar([i for i in xlabels], zees[options.projection], bottom=lastPlotBottom[options.projection*len(ylabels) : options.projection*len(ylabels)+len(ylabels)], # THIS MAY NOT BE RIGHT + # label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + # color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + + plt.xticks([float(i) for i in ylabels], ylabels)#, rotation=45) + + elif t.tagType == TAG_TYPE.THREEDUNROLL: + width = .3 # width of bar + numX = len(xlabels) + + + # insert spaces + spacedLPB = lastPlotBottom.copy() + spacedDz = dz.copy() + for i in range(len(ylabels)): + if i != 0 and i != len(ylabels): + spacedDz.insert((i*len(ylabels)) + i - 1, 0) + spacedLPB.insert((i * len(ylabels)) + i - 1, 0) + + plt.bar(range(len(spacedDz)), spacedDz, bottom=spacedLPB, + label=(options.custom_legend_labels[tagIndex] if options.custom_legend_labels != None else t.name), + color=(dracula.COLORS[tagIndex+options.color_offset] if options.color_theme == "dracula" else None)) + if len(tags) > 1 and not skipLegend: + plt.legend() # bar always gets legend, must be after plotting + + xylabels = [] + for i in range(len(xlabels)): + if i != 0: + xylabels.append('') + for j in range(len(ylabels)): + xylabels.append(str(xlabels[i]) + ' - ' + str(int(ylabels[j]))) + plt.xticks(range(len(xylabels)), xylabels, rotation=45) + + lastPlotBottom = list( map(add, lastPlotBottom, dz) )# for stacking using average + + + tagIndex += 1; + + if len(legendArtistProxyShapes) > 1 and not skipLegend: # 3dbar plot = special legend + plt.legend(legendArtistProxyShapes, + options.custom_legend_labels if options.custom_legend_labels != None else tagLabels) + elif options.custom_legend_labels != None and len(options.custom_legend_labels) > 1 and not skipLegend: + if options.horizontal: + handles, labels = plt.gca().get_legend_handles_labels() + plt.legend(reversed(handles), reversed(labels)) + else: + plt.legend() + + if options.horizontal: + plt.ylabel(options.xlabel) + plt.xlabel(options.ylabel) + else: + plt.xlabel(options.xlabel) + plt.ylabel(options.ylabel) + plt.title(options.title) + plt.tight_layout() + + # hack for snail + if "EMP_MUL" in tagLabels: + plt.subplots_adjust(left=0.18) + + + if options.show: + plt.show() + if options.log_scale: + if options.horizontal: + plt.gca().set_xscale('log') + else: + plt.gca().set_yscale('log') + plt.savefig(options.graphpath) + print("saved figure to " + options.graphpath) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="JLog plotter") + parser.add_argument('--csvlog', + required=True, + nargs="+", + help=("Path to log file.")) + parser.add_argument('--graphpath', + required=True, + help=("Path to store output graphs.")) + parser.add_argument('--title', + required=False, + default="", + help=("Graph Title.")) + parser.add_argument('--xlabel', + required=False, + default="", + help=("Label for plot x axis.")) + parser.add_argument('--ylabel', + required=False, + default="", + help=("Label for plot y axis.")) + parser.add_argument('--zlabel', + required=False, + default="", + help=("Label for plot z axis.")) + parser.add_argument('--only-tags', + required=False, + nargs="+", + help=("Only plot specific (space seperated) tags in csv log files. Default plots all tags.")) + parser.add_argument('--custom-legend-labels', + required=False, + nargs="+", + help=("Attach a custom legend label to each tag's plot.")) + parser.add_argument('--color-theme', + required=False, + help=("Use a custom color theme - dracula suported")) + parser.add_argument('--color-offset', + required=False, + type=int, + default=0, + help=("Offset starting color in color theme")) + parser.add_argument('--projection', + required=False, + type=int, + help=("Index of x/y projection for 3d to 2d projections")) + parser.add_argument('--azimuth', + required=False, + type=int, + help=("Azimuth sets view for 3D plot")) + parser.add_argument('--elevation', + required=False, + type=int, + help=("Elevation sets view for 3D plot")) + parser.add_argument('--fig-w', + required=False, + type=float, + default=5, + help=("Override figure width")) + parser.add_argument('--fig-h', + required=False, + type=float, + default=4, + help=("Override figure height")) + parser.add_argument('--xfilter', + required=False, + nargs="+", + help=("Only use data which match one of these x values.")) + parser.add_argument('--rotate_x_labels', + required=False, + action="store_true", + help=("Rotate x axis labels.")) + parser.add_argument('--hide-legend', + required=False, + action="store_true", + help=("Hide legend.")) + parser.add_argument('--bar-label-dict', + required=False, + type=json.loads, + help=("Add labels to bars e.g. '{\"tagName\": [\"l1\", \"l2\", \"l3\"]}' ")) + parser.add_argument('--hatching', + required=False, + help=("Add hatching to bars e.g. \"||||\"")) + parser.add_argument('--show', + required=False, + action="store_true", + help=("Show plots as they are generated.")) + parser.add_argument('--print-raw', + required=False, + action="store_true", + help=("Print raw plotted data to command line.")) + parser.add_argument('--log-scale', + required=False, + action="store_true", + help=("Log scale.")) + parser.add_argument('--horizontal', + required=False, + action="store_true", + help=("Plot bar graphs horizontally.")) + options = parser.parse_args() + + try: + csvFilePaths = options.csvlog + for p in csvFilePaths: + with open(p, "r") as f: + print("found file " + p) + except FileNotFoundError: + print("file not found " + p) + exit() + + if options.only_tags != None: + print("Only plotting specific tags: ") + print(options.only_tags) + + tags = getTags(csvFilePaths, options.only_tags) + + if tags == None or len(tags) == 0: + print("ERROR - No tags found - Nothing to plot") + exit() + + if options.custom_legend_labels != None: + if len(options.custom_legend_labels) == len(tags): + print("Custom Tag Labels Enabled") + else: + print("Wrong Number of Custom Tag Labels") + print(options.custom_legend_labels) + print(tags) + exit() + + plot(csvFilePaths, options, tags) + + if os.path.exists(tmpFile): + os.remove(tmpFile) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt new file mode 100644 index 0000000..199cc7d --- /dev/null +++ b/src/CMakeLists.txt @@ -0,0 +1,2 @@ +add_subdirectory(aby-float-server/) +add_subdirectory(emp-float-server/) diff --git a/src/aby-float-server/CMakeLists.txt b/src/aby-float-server/CMakeLists.txt new file mode 100644 index 0000000..acd083f --- /dev/null +++ b/src/aby-float-server/CMakeLists.txt @@ -0,0 +1,11 @@ +cmake_minimum_required(VERSION 3.12) +file(GLOB aby_float_server_src CONFIGURE_DEPENDS "*.h" "*.cpp") +list(FILTER aby_float_server_src EXCLUDE REGEX ".*server-gn\\.cpp$") +list(FILTER aby_float_server_src EXCLUDE REGEX ".*server-lm\\.cpp$") + +add_library(AbyFloatLocalization SHARED ${aby_float_server_src} ) +target_include_directories(AbyFloatLocalization PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/../common/") +target_compile_options(AbyFloatLocalization PRIVATE "-Wall" "-Wextra" "-fPIC") +target_link_libraries(AbyFloatLocalization ABY::aby ${OpenCV_LIBS}) + +# TODO - build server bins diff --git a/src/aby-float-server/gaussnewtonlocalization.cpp b/src/aby-float-server/gaussnewtonlocalization.cpp new file mode 100644 index 0000000..55b3215 --- /dev/null +++ b/src/aby-float-server/gaussnewtonlocalization.cpp @@ -0,0 +1,622 @@ +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +using namespace std; + +const uint32_t bitlen = 32; + +share* BuildGaussNewtonIteration(share* threeDPts[], share* y0[], int numPts, + share* f, share* cx, share* cy, share* x[], + BooleanCircuit* c, ABYParty* party, + e_role role) { + + std::vector& sharings = party->GetSharings(); + BooleanCircuit* bc = + (BooleanCircuit*)sharings[S_BOOL]->GetCircuitBuildRoutine(); + + share* temp; + + float zero = 0.0; + float one = 1.0; + share* one_gate = c->PutCONSGate((uint32_t*)&one, bitlen); + + // add constant ones to threeDPts + for (int p = 0; p < numPts; p++) { + threeDPts[3 * numPts + p] = one_gate; + } + + // Camera params (3x3) + share* K[9] = {f, + c->PutCONSGate((uint32_t*)&zero, bitlen), + cx, + c->PutCONSGate((uint32_t*)&zero, bitlen), + f, + cy, + c->PutCONSGate((uint32_t*)&zero, bitlen), + c->PutCONSGate((uint32_t*)&zero, bitlen), + c->PutCONSGate((uint32_t*)&one, bitlen)}; + + float jacob_epsilon = + JACOB_EPSILON; // this is needed, can't &JACOB_EPSILON directly + share* jacob_epsilon_gate = c->PutCONSGate((uint32_t*)&jacob_epsilon, bitlen); + + // project points using x guess + share** yHomog = new share*[3 * numPts]; + BuildProjectPointsCircuit(threeDPts, x, K, yHomog, numPts, c, true); + + // throw away last "row" of result (constant ones), + // interleave, and transpose y into 2nx1 vector + // e.g. [x1; y1; x2; y2 ...] + share** y = new share*[2 * numPts]; + for (int p = 0; p < numPts; ++p) { + y[p * 2] = yHomog[p]; + y[(p * 2) + 1] = yHomog[p + numPts]; + } + delete[] yHomog; + + // calculate jacobian (real 2D matrix not linearized) + // _ _ + // | du1/dr1 du1/dr2 du1/dr3 du1/t1 du1/t2 du1/t3 | + // | dv1/dr1 dv1/dr2 dv1/dr3 dv1/t1 dv1/t2 dv1/t3 | + // | du2/dr1 du2/dr2 du2/dr3 du2/t1 du2/t2 du2/t3 | + // | . | + // | . | + // |_ . (2n) _| + share*** jacob = new share**[numPts * 2]; + for (int p = 0; p < numPts * 2; p++) { + jacob[p] = new share*[6]; + } + for (int j = 0; j < 6; j++) { // for each DOF + // perturb x + temp = x[j]; + x[j] = c->PutFPGate(x[j], jacob_epsilon_gate, ADD, bitlen, 1, no_status); + + // project with epsilon + share** ytempHomog = new share*[3 * numPts]; + BuildProjectPointsCircuit(threeDPts, x, K, ytempHomog, numPts, c, true); + + // throw away last "row" of result, + // reshape, and transpose y into 2nx1 vector + // e.g. [x1; y1; x2; y2 ...] + share** ytemp = new share*[2 * numPts]; + for (int p = 0; p < numPts; ++p) { + ytemp[p * 2] = ytempHomog[p]; + ytemp[(p * 2) + 1] = ytempHomog[p + numPts]; + } + delete[] ytempHomog; + + // put into jacob + for (int p = 0; p < 2 * numPts; ++p) { + share* ytmy = c->PutFPGate(ytemp[p], y[p], SUB, bitlen, 1, no_status); + jacob[p][j] = + c->PutFPGate(ytmy, jacob_epsilon_gate, DIV, bitlen, 1, no_status); + delete ytmy; + } + + // un-perturb x + delete x[j]; + x[j] = temp; + } + + // calculate error + // dy = y0 - y; + share** dy = new share*[2 * numPts]; + for (int p = 0; p < 2 * numPts; p++) { + dy[p] = c->PutFPGate(y0[p], y[p], SUB, bitlen, 1, no_status); + } + + // compute pseudo inverse to solve: + // dy = J dx -> dx = Jdagger dy + // Note: invert requires real + // 2D arrays not linearized versions + share*** jacobI = new share**[6]; // 6x2n + for (int p = 0; p < 6; p++) { + jacobI[p] = new share*[numPts * 2]; + } + +#if PPL_FLOW == PPL_FLOW_DO || PPL_FLOW == PPL_FLOW_SiSL + BuildInvertCircuit(jacob, 2 * numPts, 6, jacobI, c, party, role, nullptr, + nullptr); + +#elif PPL_FLOW == PPL_FLOW_LOOP_LEAK + // Share Carryover. + // Invert circuit calls SVD which runs multiple + // circuit executions within. We need to store + // dy and x variables across these executions as + // raw shares. + // First, define a place to store dy and x raw shares + uint32_t* raw_dy = new uint32_t[2 * numPts]; + uint32_t* raw_x = new uint32_t[6]; + // if yao was used, convert from yao back to boolean circuit + if (c->GetContext() == S_YAO) { + for (int i = 0; i < 2 * numPts; ++i) { + temp = dy[i]; + dy[i] = bc->PutY2BGate(temp); + delete temp; + } + for (int i = 0; i < 6; ++i) { + temp = x[i]; + x[i] = bc->PutY2BGate(temp); + delete temp; + } + } + // Next, put output gates on the cirucuit + for (int i = 0; i < 2 * numPts; ++i) { + share* temp = dy[i]; + dy[i] = bc->PutSharedOUTGate(dy[i]); + delete temp; + } + for (int i = 0; i < 6; ++i) { + share* temp = x[i]; + x[i] = bc->PutSharedOUTGate(x[i]); + delete temp; + } + // Next, build lambda function to convert share object to raw share + std::function toRawShares = [dy, raw_dy, x, raw_x, numPts]() { + for (int i = 0; i < 2 * numPts; ++i) { + raw_dy[i] = dy[i]->get_clear_value(); + } + for (int i = 0; i < 6; ++i) { + raw_x[i] = x[i]->get_clear_value(); + } + }; + // Lastly, build lambda function to convert back to share object + std::function toShareObjects = [dy, raw_dy, x, raw_x, numPts, bc]() { + for (int i = 0; i < 2 * numPts; ++i) { + dy[i] = bc->PutSharedINGate(raw_dy[i], bitlen); + } + for (int i = 0; i < 6; ++i) { + x[i] = bc->PutSharedINGate(raw_x[i], bitlen); + } + }; + + BuildInvertCircuit(jacob, 2 * numPts, 6, jacobI, (BooleanCircuit*)c, party, + role, toRawShares, toShareObjects); + + // if yao was used, convert stored shares from bool back to yao circuit + if (c->GetContext() == S_YAO) { + for (int i = 0; i < 2 * numPts; ++i) { + temp = dy[i]; + dy[i] = c->PutB2YGate(temp); + delete temp; + } + for (int i = 0; i < 6; ++i) { + temp = x[i]; + x[i] = c->PutB2YGate(temp); + delete temp; + } + } + + // cleanup share carryover + delete[] raw_dy; + delete[] raw_x; +#endif + + // linearize for matmult + share** jacobILinear = new share*[6 * 2 * numPts]; + for (int p = 0; p < 6; p++) { + for (int pp = 0; pp < 2 * numPts; pp++) { + jacobILinear[(p * 2 * numPts) + pp] = jacobI[p][pp]; + } + } + share* dx[6]; + BuildMatmultCircuit(jacobILinear, 6, 2 * numPts, dy, 2 * numPts, + 1, // column vector + dx, c); + + // cleanup jacobian + for (int p = 0; p < numPts * 2; p++) { + // individual jacob shares are deleted in invert->svd() + delete[] jacob[p]; + } + delete[] jacob; + + for (int p = 0; p < 6; p++) { + delete[] jacobI[p]; + } + delete[] jacobILinear; + + // return if error under threshold + // abs(norm(dy, cv::NORM_L2SQR)) < MIN_ER + share* norm = BuildTwoNormSqCircuit(dx, 6, c); + BuildFabsCircuit(norm, c); + float min_er = MIN_ER; + share* minErGate = c->PutCONSGate((uint32_t*)&min_er, bitlen); + share* cmp = c->PutFPGate(minErGate, norm, CMP, bitlen, 1, no_status); + delete norm; + delete minErGate; + + // update pose + for (int p = 0; p < 6; p++) { +#if PPL_FLOW == PPL_FLOW_DO // set dx to zero if no error + share* zero_gate = c->PutCONSGate((uint32_t*)&zero, bitlen); + share* olddx = dx[p]; + dx[p] = c->PutMUXGate(dx[p], zero_gate, cmp); + delete olddx; + delete zero_gate; +#endif + temp = x[p]; + x[p] = c->PutFPGate(x[p], dx[p], ADD, bitlen, 1, no_status); + delete temp; + delete dx[p]; + } + + return cmp; +} + +// runs circuit using pre-shared inputs +bool RunGaussNewtonIteration(uint32_t* threeDPts, uint32_t* y0, int numPts, + uint32_t f, uint32_t cx, uint32_t cy, uint32_t* x, + BooleanCircuit* c, ABYParty* p, e_role role) { + + std::vector& sharings = p->GetSharings(); + BooleanCircuit* bc = + (BooleanCircuit*)sharings[S_BOOL]->GetCircuitBuildRoutine(); + + // Allocate space for shares + share** s_threeDPts = new share*[4 * numPts]; // leave room for contant 1's + share** s_y0 = new share*[2 * numPts]; + share** s_x = new share*[6]; + share* s_f; + share* s_cx; + share* s_cy; + share* temp; + + // Prepare inputs + for (int p = 0; p < 3 * numPts; p++) { + s_threeDPts[p] = bc->PutSharedINGate(threeDPts[p], bitlen); + } + for (int p = 0; p < 2 * numPts; p++) { + s_y0[p] = bc->PutSharedINGate(y0[p], bitlen); + } + for (int p = 0; p < 6; p++) { + s_x[p] = bc->PutSharedINGate(x[p], bitlen); + } + s_f = bc->PutSharedINGate(f, bitlen); + s_cx = bc->PutSharedINGate(cx, bitlen); + s_cy = bc->PutSharedINGate(cy, bitlen); + // if yao, convert bool inputs to yao + if (c->GetContext() == S_YAO) { + for (int p = 0; p < 3 * numPts; p++) { + temp = s_threeDPts[p]; + s_threeDPts[p] = c->PutB2YGate(temp); + delete temp; + } + for (int p = 0; p < 2 * numPts; p++) { + temp = s_y0[p]; + s_y0[p] = c->PutB2YGate(temp); + delete temp; + } + for (int p = 0; p < 6; p++) { + temp = s_x[p]; + s_x[p] = c->PutB2YGate(temp); + delete temp; + } + temp = s_f; + s_f = c->PutB2YGate(temp); + delete temp; + temp = s_cx; + s_cx = c->PutB2YGate(temp); + delete temp; + temp = s_cy; + s_cy = c->PutB2YGate(temp); + delete temp; + } + // prepare circuit + share* s_done = BuildGaussNewtonIteration(s_threeDPts, s_y0, numPts, s_f, + s_cx, s_cy, s_x, c, p, role); + + // if yao was used, convert from yao back to boolean circuit + if (c->GetContext() == S_YAO) { + for (int p = 0; p < 6; p++) { + temp = s_x[p]; + s_x[p] = bc->PutY2BGate(temp); + delete temp; + } + } + + // prepare output (only need x and done flag) + for (int p = 0; p < 6; p++) { + temp = s_x[p]; + s_x[p] = bc->PutSharedOUTGate(temp); // raw share + delete temp; + } + temp = s_done; + s_done = c->PutOUTGate(temp, ALL); // need cleartext + delete temp; + + p->ExecCircuit(); + + // get shared output + for (int p = 0; p < 6; p++) { + x[p] = s_x[p]->get_clear_value(); + delete s_x[p]; + } + delete[] s_x; + uint32_t done = s_done->get_clear_value(); + for (int p = 0; p < 3 * numPts; p++) { // ignore constant ones? + delete s_threeDPts[p]; + } + delete[] s_threeDPts; + for (int p = 0; p < 2 * numPts; p++) { + delete s_y0[p]; + } + delete[] s_y0; + delete s_f; + delete s_cx; + delete s_cy; + + collectTiming(); + collectCommunication(); + p->Reset(); + return done != 0; +} + +// uint32_t arguments must be from PutSharedOUTGate() +// then calling get_clear_value() and circuit->Reset(); +// This function builds/executes multiple circuits. +void RunGaussNewtonCircuit(uint32_t* threeDPts, uint32_t* y0, int numPts, + uint32_t f, uint32_t cx, uint32_t cy, uint32_t* x, + BooleanCircuit* c, ABYParty* party, e_role role) { + // GN Iteration + for (int i = 0; i < GN_MAX_ITR; i++) { + if (_verbosity & DBG_FLOW) { + cout << RED << "\n\nGN Iteration " << i << RESET << endl; + } + + // break if error under threshold + if (RunGaussNewtonIteration(threeDPts, y0, numPts, f, cx, cy, x, c, party, + role)) + break; + } +} + +// Wrapper function around RunGaussNewtonCircuit which +// takes shares instead of secret shares. +// This (and all inner function calls) creates multiple +// circuits such that whenever an intermediate plaintext value +// is required, the circuit ends and a new circuit begins. +// +// Doing so has the overhead of converting intermediate values +// to secret shares to be used in the next circuit execution. +// It does not however, require any of the "debug" functionality +// from the ABY library to retreive intermediate values. +// +// This wrapper creates dummy circuits to build secret shares +// to be passed to the top level GN function. +// +// Note: All 2D matrix inputs passed as single dimension array +// Note: the constant 1's can be ignored (but space must be allocated) +// +// 3D World Points (4xnumPts) which look like: +// _ _ +// | x1 x2 x3 | +// | y1 y2 y3 ... | +// | z1 z2 z3 | +// |_ 1 1 1 _| +// must be pass as single dimension: +// [x1, x2, ...; y1, y2, ...; z1, z2, ...; 1, 1, ... ] +// +// 2D Image Points (3xnumPts) which look like: +// _ _ +// | u1 u2 u3 | +// | v1 v2 v3 ... | +// |_ 1 1 1 _| +// must be passed as single dimension: +// [ u1, u2, ...; v1, v2, ...; 1, 1, ... ] +// +// x (initial guess) : 6x1 [ rotation angles, translations ] +void BuildAndRunGaussNewtonLoopLeak(share* s_threeDPts[], share* s_twoDPts[], + int numPts, share* s_f, share* s_cx, + share* s_cy, share* s_x[], + BooleanCircuit* c, ABYParty* party, + e_role role) { + + std::vector& sharings = party->GetSharings(); + BooleanCircuit* bc = + (BooleanCircuit*)sharings[S_BOOL]->GetCircuitBuildRoutine(); + + share* temp; + + // First transform input shares + // + // throw away last "row" of 2D points, + // reshape, and transpose into 2nx1 vector + // e.g. [x1; y1; x2; y2 ...] + share** s_y0 = new share*[2 * numPts]; + for (int p = 0; p < numPts; ++p) { + s_y0[2 * p] = s_twoDPts[p]; + s_y0[(2 * p) + 1] = s_twoDPts[p + numPts]; + } + + // Next, run dummy circuit to create raw shares from share objects + // stores secret-shared output + uint32_t* threeDPts = new uint32_t[4 * numPts]; + uint32_t* y0 = new uint32_t[2 * numPts]; + uint32_t* x = new uint32_t[6]; + uint32_t f; + uint32_t cx; + uint32_t cy; + + // if yao is prefered circuit and yao shares passed in, must + // convert to bool to get raw shares. + // Tests must pass in bool shares even if yao is preferred. + if (c->GetContext() == S_YAO && s_threeDPts[0]->get_share_type() == S_YAO) { + DEBUG_MSG("Converting yao shares to bool for shared output\n"); + for (int p = 0; p < 3 * numPts; p++) { + temp = s_threeDPts[p]; + s_threeDPts[p] = bc->PutY2BGate(temp); + delete temp; + } + for (int p = 0; p < 2 * numPts; p++) { + temp = s_y0[p]; + s_y0[p] = bc->PutY2BGate(temp); + delete temp; + } + for (int p = 0; p < 6; p++) { + temp = s_x[p]; + s_x[p] = bc->PutY2BGate(temp); + delete temp; + } + temp = s_f; + s_f = bc->PutY2BGate(temp); + delete temp; + temp = s_cx; + s_cx = bc->PutY2BGate(temp); + delete temp; + temp = s_cy; + s_cy = bc->PutY2BGate(temp); + delete temp; + } + + // build shared output objects to get share, ignore constant 1s + for (int p = 0; p < 3 * numPts; p++) { + temp = s_threeDPts[p]; + s_threeDPts[p] = bc->PutSharedOUTGate(temp); + delete temp; + } + for (int p = 0; p < 2 * numPts; p++) { + temp = s_y0[p]; + s_y0[p] = bc->PutSharedOUTGate(temp); + delete temp; + } + for (int p = 0; p < 6; p++) { + temp = s_x[p]; + s_x[p] = bc->PutSharedOUTGate(temp); + delete temp; + } + temp = s_f; + s_f = bc->PutSharedOUTGate(temp); + delete temp; + temp = s_cx; + s_cx = bc->PutSharedOUTGate(temp); + delete temp; + temp = s_cy; + s_cy = bc->PutSharedOUTGate(temp); + delete temp; + // run the dummy circuit + CLOCK(ShareInputs); + TIC(ShareInputs); + party->ExecCircuit(); + TOC(ShareInputs); + // get shared outputs, ignore constant 1s + for (int p = 0; p < 3 * numPts; p++) { + threeDPts[p] = s_threeDPts[p]->get_clear_value(); + delete s_threeDPts[p]; + } + for (int p = 0; p < 2 * numPts; p++) { + y0[p] = s_y0[p]->get_clear_value(); + delete s_y0[p]; + } + for (int p = 0; p < 6; p++) { + x[p] = s_x[p]->get_clear_value(); + delete s_x[p]; + } + f = s_f->get_clear_value(); + delete s_f; + cx = s_cx->get_clear_value(); + delete s_cx; + cy = s_cy->get_clear_value(); + delete s_cy; + + collectTiming(); + collectCommunication(); + party->Reset(); + // arrays now contain secret shared values of plaintext from each party + + CLOCK(GaussNewtonGD); + TIC(GaussNewtonGD); + RunGaussNewtonCircuit(threeDPts, y0, numPts, f, cx, cy, x, c, party, role); + TOC(GaussNewtonGD); + + // next start another dummy circuit to return share + // objects which are built from shares + for (int p = 0; p < 6; p++) { + s_x[p] = bc->PutSharedINGate(&x[p], bitlen); + } +} + +// Data Oblivious version of GN. Does not require executing +// multiple circuits to retreive intermediate values. +// +// Note: All 2D matrix inputs passed as single dimension array +// Note: the constant 1's can be ignored (but space must be allocated) +// +// 3D World Points (4xnumPts) which look like: +// _ _ +// | x1 x2 x3 | +// | y1 y2 y3 ... | +// | z1 z2 z3 | +// |_ 1 1 1 _| +// must be pass as single dimension: +// [x1, x2, ...; y1, y2, ...; z1, z2, ...; 1, 1, ... ] +// +// 2D Image Points (3xnumPts) which look like: +// _ _ +// | u1 u2 u3 | +// | v1 v2 v3 ... | +// |_ 1 1 1 _| +// must be passed as single dimension: +// [ u1, u2, ...; v1, v2, ...; 1, 1, ... ] +// +// x (initial guess) : 6x1 [ rotation angles, translations ] +void BuildAndRunGaussNewtonDO(share* s_threeDPts[], share* s_twoDPts[], + int numPts, share* s_f, share* s_cx, share* s_cy, + share* s_x[], BooleanCircuit* c, ABYParty* party, + e_role role) { + // First transform input shares + // + // throw away last "row" of 2D points, + // reshape, and transpose into 2nx1 vector + // e.g. [x1; y1; x2; y2 ...] + share** s_y0 = new share*[2 * numPts]; + for (int p = 0; p < numPts; ++p) { + s_y0[2 * p] = s_twoDPts[p]; + s_y0[(2 * p) + 1] = s_twoDPts[p + numPts]; + } + + // GN Iteration + for (int i = 0; i < GN_MAX_ITR; i++) { + if (_verbosity & DBG_FLOW) { + cout << RED << "\n\nGN Iteration " << i << RESET << endl; + } + + share* s_done = BuildGaussNewtonIteration(s_threeDPts, s_y0, numPts, s_f, + s_cx, s_cy, s_x, c, party, role); + delete s_done; // throw it away + } +} + +void BuildAndRunGaussNewton(share* s_threeDPts[], share* s_twoDPts[], + int numPts, share* s_f, share* s_cx, share* s_cy, + share* s_x[], BooleanCircuit* c, ABYParty* party, + e_role role) { +#if PPL_FLOW == PPL_FLOW_DO + BuildAndRunGaussNewtonDO(s_threeDPts, s_twoDPts, numPts, s_f, s_cx, s_cy, s_x, + c, party, role); +#elif PPL_FLOW == PPL_FLOW_LOOP_LEAK || PPL_FLOW == PPL_FLOW_SiSL + BuildAndRunGaussNewtonLoopLeak(s_threeDPts, s_twoDPts, numPts, s_f, s_cx, + s_cy, s_x, c, party, role); +#endif +} diff --git a/src/aby-float-server/gaussnewtonlocalization.h b/src/aby-float-server/gaussnewtonlocalization.h new file mode 100644 index 0000000..055868d --- /dev/null +++ b/src/aby-float-server/gaussnewtonlocalization.h @@ -0,0 +1,24 @@ +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +using namespace std; + +void BuildGaussNewtonCircuit(share* threeDPts[], int numThreeD, + share* twoDPts[], int numTwoD, share* f, share* cx, + share* cy, share* x[], BooleanCircuit* c); + +void BuildAndRunGaussNewton(share* s_threeDPts[], share* s_twoDPts[], + int numPts, share* s_f, share* s_cx, share* s_cy, + share* s_x[], BooleanCircuit* c, ABYParty* party, + e_role role); diff --git a/src/aby-float-server/invert.cpp b/src/aby-float-server/invert.cpp new file mode 100644 index 0000000..8e43aaf --- /dev/null +++ b/src/aby-float-server/invert.cpp @@ -0,0 +1,180 @@ +#include +#include +#include +#include +#include + +//#include +#include +#include +#include +#include + +#include "abycore/aby/abyparty.h" +#include "abycore/circuit/arithmeticcircuits.h" +#include "abycore/circuit/booleancircuits.h" +#include "abycore/circuit/circuit.h" +#include "abycore/sharing/sharing.h" + +#include +#include + +using namespace std; + +const uint32_t bitlen = 32; + +void BuildInvertCircuitWSubCircuits(share** s_in[], int m, int n, + share** s_res[], BooleanCircuit* c, + ABYParty* party, e_role role, + std::function toRawShares, + std::function toShareObjects) { + + assert(m >= n); + float zero = 0; + + // svd overwrites with s_in with u matrix, must copy shares + share*** s_u = new share**[m]; + for (int i = 0; i < m; i++) { + s_u[i] = new share*[n]; + for (int j = 0; j < n; j++) { + s_u[i][j] = s_in[i][j]; + } + } + + share** s_w = new share*[m]; // aka sigma, only diag + share*** s_v = new share**[n]; // nxn + for (int i = 0; i < n; i++) + s_v[i] = new share*[n]; + + BuildAndRunSvd(s_u, m, n, s_w, s_v, (BooleanCircuit*)c, party, role, + toRawShares, toShareObjects); + + // make new zero gate after BuildAndRunSvd because it makes a new circuit + share* zerogate = c->PutCONSGate((uint32_t*)&zero, bitlen); + + // res = inv(in) = v*inv(w)*uT + // inv(w)*uT + for (int j = 0; j < n; j++) { + // if (s_w[j]) { + share* ifw = s_w[j]->get_wire_ids_as_share(0); + for (uint32_t tt = 1; tt < bitlen - 1; + tt++) { // -1 -> do not include sign bit + share* tempbit = s_w[j]->get_wire_ids_as_share(tt); + share* temp = ifw; + ifw = c->PutORGate(temp, tempbit); + delete tempbit; + delete temp; + } + + for (int i = 0; i < m; i++) { + share* temp = c->PutFPGate(s_u[i][j], s_w[j], DIV, bitlen, 1, no_status); + share* temp2 = s_u[i][j]; + s_u[i][j] = c->PutMUXGate(temp, zerogate, ifw); + delete temp; + // delete temp2; DO NOT DELETE s_u, it is still part of _s_in + } + delete ifw; + //} + } + + // v*(w*uT) (don't use matmult so we can do transpose ourselves) + for (int j = 0; j < n; j++) { + for (int jj = 0; jj < m; jj++) { + // dont delete s_res[j][jj] - share may be elsewhere + s_res[j][jj] = c->PutCONSGate((uint32_t*)&zero, bitlen); + for (int k = 0; k < n; k++) { + // note u indices do transpose + share* temp = + c->PutFPGate(s_v[j][k], s_u[jj][k], MUL, bitlen, 1, no_status); + share* temp2 = s_res[j][jj]; + s_res[j][jj] = + c->PutFPGate(s_res[j][jj], temp, ADD, bitlen, 1, no_status); + delete temp; + delete temp2; + } + } + } +} + +void BuildInvertCircuitDO(share** s_in[], int m, int n, share** s_res[], + BooleanCircuit* c, ABYParty* party, e_role role) { + assert(m >= n); + float zero = 0; + + // svd overwrites with s_in with u matrix + // share*** s_in = new share**[m]; no need to copy shares because new circuits + // run by svd trash them anyway for(int i=0; iPutCONSGate((uint32_t*)&zero, bitlen); + + // res = inv(in) = v*inv(w)*uT + // inv(w)*uT + for (int j = 0; j < n; j++) { + // if (s_w[j]) { + share* ifw = s_w[j]->get_wire_ids_as_share(0); + for (uint32_t tt = 1; tt < bitlen - 1; + tt++) { // -1 -> do not include sign bit + share* tempbit = s_w[j]->get_wire_ids_as_share(tt); + share* temp = ifw; + ifw = c->PutORGate(temp, tempbit); + delete tempbit; + delete temp; + } + + for (int i = 0; i < m; i++) { + share* temp = c->PutFPGate(s_u[i][j], s_w[j], DIV, bitlen, 1, no_status); + share* temp2 = s_u[i][j]; + s_u[i][j] = c->PutMUXGate(temp, zerogate, ifw); + delete temp; + // delete temp2; DO NOT DELETE s_u, it is still part of _s_in + } + delete ifw; + //} + } + + // v*(w*uT) (don't use matmult so we can do transpose ourselves) + for (int j = 0; j < n; j++) { + for (int jj = 0; jj < m; jj++) { + // dont delete s_res[j][jj] - share may be elsewhere + s_res[j][jj] = c->PutCONSGate((uint32_t*)&zero, bitlen); + for (int k = 0; k < n; k++) { + // note u indices do transpose + share* temp = + c->PutFPGate(s_v[j][k], s_u[jj][k], MUL, bitlen, 1, no_status); + share* temp2 = s_res[j][jj]; + s_res[j][jj] = + c->PutFPGate(s_res[j][jj], temp, ADD, bitlen, 1, no_status); + delete temp; + delete temp2; + } + } + } +} + +void BuildInvertCircuit(share** s_in[], int m, int n, share** s_res[], + BooleanCircuit* c, ABYParty* party, e_role role, + std::function toRawShares, + std::function toShareObjects) { +#if PPL_FLOW == PPL_FLOW_DO || \ + PPL_FLOW == PPL_FLOW_SiSL // set dx to zero if no error + (void)toRawShares; + (void)toShareObjects; + BuildInvertCircuitDO(s_in, m, n, s_res, c, party, role); +#elif PPL_FLOW == PPL_FLOW_LOOP_LEAK + BuildInvertCircuitWSubCircuits(s_in, m, n, s_res, c, party, role, toRawShares, + toShareObjects); +#endif +} diff --git a/src/aby-float-server/invert.h b/src/aby-float-server/invert.h new file mode 100644 index 0000000..92a1519 --- /dev/null +++ b/src/aby-float-server/invert.h @@ -0,0 +1,10 @@ +#include "abycore/aby/abyparty.h" +#include "abycore/circuit/arithmeticcircuits.h" +#include "abycore/circuit/booleancircuits.h" +#include "abycore/circuit/circuit.h" +#include "abycore/sharing/sharing.h" + +void BuildInvertCircuit( + share** _s_in[], int m, int n, share** s_res[], BooleanCircuit* c, + ABYParty* party, e_role role, std::function toRawShares = []() {}, + std::function toShareObjects = []() {}); diff --git a/src/aby-float-server/lmlocalization.cpp b/src/aby-float-server/lmlocalization.cpp new file mode 100644 index 0000000..8a8e5d6 --- /dev/null +++ b/src/aby-float-server/lmlocalization.cpp @@ -0,0 +1,975 @@ +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +using namespace std; + +const uint32_t bitlen = 32; + +// returns error norm (not if lower than min), and this lambda +std::pair BuildLMIteration(share* threeDPts[], share* y0[], + int numPts, share* f, share* cx, + share* cy, share* x[], + share* prevErrNorm, share* lambda, + BooleanCircuit* c, ABYParty* party, + e_role role) { + + if (_verbosity & DBG_ARGS) { + for (int p = 0; p < 3 * numPts; p++) { + char s[50]; + sprintf(s, "threeDPts[%d]", p); + c->PutPrintValueGate(threeDPts[p], s); + } + for (int p = 0; p < 2 * numPts; p++) { + char s[50]; + sprintf(s, "y0[%d]", p); + c->PutPrintValueGate(y0[p], s); + } + c->PutPrintValueGate(f, "s_f"); + c->PutPrintValueGate(cx, "s_cx"); + c->PutPrintValueGate(cx, "s_cy"); + } + + std::vector& sharings = party->GetSharings(); + + share* temp; + + float zero = 0.0; + float one = 1.0; + share* one_gate = c->PutCONSGate((uint32_t*)&one, bitlen); + + // add constant ones to threeDPts + for (int p = 0; p < numPts; p++) { + threeDPts[3 * numPts + p] = one_gate; + } + + // Camera params (3x3) + share* K[9] = {f, + c->PutCONSGate((uint32_t*)&zero, bitlen), + cx, + c->PutCONSGate((uint32_t*)&zero, bitlen), + f, + cy, + c->PutCONSGate((uint32_t*)&zero, bitlen), + c->PutCONSGate((uint32_t*)&zero, bitlen), + c->PutCONSGate((uint32_t*)&one, bitlen)}; + + float jacob_epsilon = JACOB_EPSILON; + share* jacob_epsilon_gate = c->PutCONSGate((uint32_t*)&jacob_epsilon, bitlen); + + // project points using x guess + share** yHomog = new share*[3 * numPts]; + BuildProjectPointsCircuit(threeDPts, x, K, yHomog, numPts, c, true); + + // throw away last "row" of result (constant ones), + // interleave, and transpose y into 2nx1 vector + // e.g. [x1; y1; x2; y2 ...] + share** y = new share*[2 * numPts]; + for (int p = 0; p < numPts; ++p) { + y[p * 2] = yHomog[p]; + y[(p * 2) + 1] = yHomog[p + numPts]; + } + if (_verbosity & DBG_PROJECT) { + for (int p = 0; p < 2 * numPts; p++) { + char s[50]; + sprintf(s, "y[%d]", p); + c->PutPrintValueGate(y[p], s); + } + } + delete[] yHomog; + + // calculate jacobian (real 2D matrix not linearized) + // _ _ + // | du1/dr1 du1/dr2 du1/dr3 du1/t1 du1/t2 du1/t3 | + // | dv1/dr1 dv1/dr2 dv1/dr3 dv1/t1 dv1/t2 dv1/t3 | + // | du2/dr1 du2/dr2 du2/dr3 du2/t1 du2/t2 du2/t3 | + // | . | + // | . | + // |_ . (2n) _| + share*** jacob = new share**[numPts * 2]; + for (int p = 0; p < numPts * 2; p++) { + jacob[p] = new share*[6]; + } + for (int j = 0; j < 6; j++) { // for each DOF + // perturb x + temp = x[j]; + x[j] = c->PutFPGate(x[j], jacob_epsilon_gate, ADD, bitlen, 1, no_status); + + // project with epsilon + share** ytempHomog = new share*[3 * numPts]; + BuildProjectPointsCircuit(threeDPts, x, K, ytempHomog, numPts, c, true); + + // throw away last "row" of result, + // reshape, and transpose y into 2nx1 vector + // e.g. [x1; y1; x2; y2 ...] + share** ytemp = new share*[2 * numPts]; + for (int p = 0; p < numPts; ++p) { + ytemp[p * 2] = ytempHomog[p]; + ytemp[(p * 2) + 1] = ytempHomog[p + numPts]; + } + delete[] ytempHomog; + + // put into jacob + for (int p = 0; p < 2 * numPts; ++p) { + share* ytmy = c->PutFPGate(ytemp[p], y[p], SUB, bitlen, 1, no_status); + jacob[p][j] = + c->PutFPGate(ytmy, jacob_epsilon_gate, DIV, bitlen, 1, no_status); + delete ytmy; + } + + // un-perturb x + delete x[j]; + x[j] = temp; + } + if (_verbosity & DBG_JACOB) { + for (int p = 0; p < numPts * 2; p++) { + for (int pp = 0; pp < 6; pp++) { + char s[50]; + sprintf(s, "jacob[%d][%d]", p, pp); + c->PutPrintValueGate(jacob[p][pp], s); + } + } + } + + // calculate error + // dy = y0 - y; + share** dy = new share*[2 * numPts]; + for (int p = 0; p < 2 * numPts; p++) { + dy[p] = c->PutFPGate(y0[p], y[p], SUB, bitlen, 1, no_status); + } + + // LM with fletcher improvement + // inv(Jt*J + lambda*diag(Jt*J)) * Jt * dy + share*** JtJ = new share**[6]; // 6x6 + for (int p = 0; p < 6; p++) { + JtJ[p] = new share*[6]; + } + BuildMatmult2DwTransposeCircuit(jacob, numPts * 2, 6, true, jacob, numPts * 2, + 6, false, JtJ, + c); // must use 2D arrays, not 1D + // add lambda*diag(Jt*J) in-place + for (int p = 0; p < 6; p++) { + // add fletcher column-wise to JtJ + share* fletcher = + c->PutFPGate(lambda, JtJ[p][p], MUL, bitlen, 1, no_status); + + for (int pp = 0; pp < 6; pp++) { + share* temp = JtJ[pp][p]; + JtJ[pp][p] = + c->PutFPGate(JtJ[pp][p], fletcher, ADD, bitlen, 1, no_status); + delete temp; + } + delete fletcher; + } + + // compute pseudo inverse + // Note: invert requires real + // 2D arrays not linearized versions + share*** JtJ_i = new share**[6]; // 6x6 + for (int p = 0; p < 6; p++) { + JtJ_i[p] = new share*[6]; + } + +#if PPL_FLOW == PPL_FLOW_DO || PPL_FLOW == PPL_FLOW_SiSL + BuildInvertCircuit(JtJ, 6, 6, JtJ_i, c, party, role, nullptr, nullptr); + +#elif PPL_FLOW == PPL_FLOW_LOOP_LEAK + // Share Carryover. + // Invert circuit calls SVD which runs multiple + // circuit executions within. We need to store + // dy and jacob variables across these executions as + // raw shares. + BooleanCircuit* bc = + (BooleanCircuit*)sharings[S_BOOL]->GetCircuitBuildRoutine(); + + // First, define a place to store dy and jacob raw shares + uint32_t* raw_dy = new uint32_t[2 * numPts]; + uint32_t* raw_jacob = new uint32_t[6 * 2 * numPts]; + uint32_t* raw_x = new uint32_t[6]; + uint32_t* raw_lambda = new uint32_t; + uint32_t* raw_prevErrNorm = new uint32_t; + // if yao was used, convert from yao back to boolean circuit + if (c->GetContext() == S_YAO) { + DEBUG_MSG("converting to yao\n"); + for (int i = 0; i < 2 * numPts; ++i) { + temp = dy[i]; + dy[i] = bc->PutY2BGate(temp); + delete temp; + } + for (int p = 0; p < 2 * numPts; ++p) { + for (int pp = 0; pp < 6; ++pp) { + temp = jacob[p][pp]; + jacob[p][pp] = bc->PutY2BGate(temp); + delete temp; + } + } + for (int i = 0; i < 6; ++i) { + temp = x[i]; + x[i] = bc->PutY2BGate(temp); + delete temp; + } + temp = lambda; + lambda = bc->PutY2BGate(temp); + delete temp; + temp = prevErrNorm; + prevErrNorm = bc->PutY2BGate(temp); + delete temp; + } + // Next, put output gates on the cirucuit + for (int i = 0; i < 2 * numPts; ++i) { + temp = dy[i]; + dy[i] = bc->PutSharedOUTGate(dy[i]); + delete temp; + } + for (int p = 0; p < 2 * numPts; ++p) { + for (int pp = 0; pp < 6; ++pp) { + temp = jacob[p][pp]; + jacob[p][pp] = bc->PutSharedOUTGate(jacob[p][pp]); + delete temp; + } + } + for (int i = 0; i < 6; ++i) { + temp = x[i]; + x[i] = bc->PutSharedOUTGate(x[i]); + delete temp; + } + temp = lambda; + lambda = bc->PutSharedOUTGate(temp); + delete temp; + temp = prevErrNorm; + prevErrNorm = bc->PutSharedOUTGate(temp); + delete temp; + // Next, build lambda function to convert share object to raw share + std::function toRawShares = [dy, raw_dy, jacob, raw_jacob, x, raw_x, + lambda, raw_lambda, prevErrNorm, + raw_prevErrNorm, numPts]() { + for (int i = 0; i < 2 * numPts; ++i) { + raw_dy[i] = dy[i]->get_clear_value(); + } + for (int p = 0; p < 2 * numPts; ++p) { + for (int pp = 0; pp < 6; ++pp) { + raw_jacob[p * 6 + pp] = jacob[p][pp]->get_clear_value(); + } + } + for (int i = 0; i < 6; ++i) { + raw_x[i] = x[i]->get_clear_value(); + } + *raw_lambda = lambda->get_clear_value(); + *raw_prevErrNorm = prevErrNorm->get_clear_value(); + }; + // Lastly, build lambda function to convert back to share object + share** ref_lambda = λ + share** ref_prevErrNorm = &prevErrNorm; + std::function toShareObjects = + [dy, raw_dy, jacob, raw_jacob, x, raw_x, ref_lambda, raw_lambda, + ref_prevErrNorm, raw_prevErrNorm, numPts, bc]() { + for (int i = 0; i < 2 * numPts; ++i) { + dy[i] = bc->PutSharedINGate(raw_dy[i], bitlen); + } + for (int p = 0; p < 2 * numPts; ++p) { + for (int pp = 0; pp < 6; ++pp) { + jacob[p][pp] = bc->PutSharedINGate(raw_jacob[p * 6 + pp], bitlen); + } + } + for (int i = 0; i < 6; ++i) { + x[i] = bc->PutSharedINGate(raw_x[i], bitlen); + } + *ref_lambda = bc->PutSharedINGate(*raw_lambda, bitlen); + *ref_prevErrNorm = bc->PutSharedINGate(*raw_prevErrNorm, bitlen); + }; + + BuildInvertCircuit(JtJ, 6, 6, JtJ_i, (BooleanCircuit*)c, party, role, + toRawShares, toShareObjects); + + // if yao was used, convert stored shares from bool back to yao circuit + if (c->GetContext() == S_YAO) { + for (int i = 0; i < 2 * numPts; ++i) { + temp = dy[i]; + dy[i] = c->PutB2YGate(temp); + delete temp; + } + for (int p = 0; p < 2 * numPts; ++p) { + for (int pp = 0; pp < 6; ++pp) { + temp = jacob[p][pp]; + jacob[p][pp] = c->PutB2YGate(temp); + delete temp; + } + } + for (int i = 0; i < 6; ++i) { + temp = x[i]; + x[i] = c->PutB2YGate(temp); + delete temp; + } + temp = lambda; + lambda = c->PutB2YGate(temp); + delete temp; + temp = prevErrNorm; + prevErrNorm = c->PutB2YGate(temp); + delete temp; + } + + // cleanup share carryover + delete[] raw_dy; + delete[] raw_jacob; + delete[] raw_x; + delete raw_lambda; + delete raw_prevErrNorm; +#endif + + share*** JtJ_i_Jt = new share**[6]; // 6x2n + for (int p = 0; p < 6; p++) { + JtJ_i_Jt[p] = new share*[numPts * 2]; + } + BuildMatmult2DwTransposeCircuit(JtJ_i, 6, 6, false, jacob, numPts * 2, 6, + true, JtJ_i_Jt, c); + + // linearize for matmult + share** JtJ_i_Jt_linear = new share*[6 * 2 * numPts]; + for (int p = 0; p < 6; p++) { + for (int pp = 0; pp < 2 * numPts; pp++) { + JtJ_i_Jt_linear[(p * 2 * numPts) + pp] = JtJ_i_Jt[p][pp]; + } + } + share* dx[6]; + BuildMatmultCircuit(JtJ_i_Jt_linear, 6, 2 * numPts, dy, 2 * numPts, + 1, // column vector + dx, c); + + // cleanup + delete[] JtJ_i_Jt_linear; + for (int p = 0; p < numPts * 2; p++) { + delete[] jacob[p]; + } + delete[] jacob; + + for (int p = 0; p < 6; p++) { + delete[] JtJ[p]; + delete[] JtJ_i[p]; + delete[] JtJ_i_Jt[p]; + } + delete[] JtJ; + delete[] JtJ_i; + delete[] JtJ_i_Jt; + + // Constants used after invert (invert may break circuit, define after) + float min_er = MIN_ER; + share* min_er_gate = c->PutCONSGate((uint32_t*)&min_er, bitlen); + + float lambda_max = LM_LAMBDA_MAX; + share* lambda_max_gate = c->PutCONSGate((uint32_t*)&lambda_max, bitlen); + + float lambda_min = LM_LAMBDA_MIN; + share* lambda_min_gate = c->PutCONSGate((uint32_t*)&lambda_min, bitlen); + + float ten = 10; + share* ten_gate = c->PutCONSGate((uint32_t*)&ten, bitlen); + + // check error under threshold + // abs(norm(dy, cv::NORM_L2SQR)) < MIN_ER + share* absnorm = BuildTwoNormSqCircuit(dx, 6, c); + BuildFabsCircuit(absnorm, c); + share* er_flag = + c->PutFPGate(min_er_gate, absnorm, CMP, bitlen, 1, no_status); + if (_verbosity & DBG_ER) { + c->PutPrintValueGate(absnorm, "absnorm"); + } + + share* biggerLambda = + c->PutFPGate(lambda, ten_gate, MUL, bitlen, 1, no_status); + share* smallerLambda = + c->PutFPGate(lambda, ten_gate, DIV, bitlen, 1, no_status); + + share* errWentDown = + c->PutFPGate(prevErrNorm, absnorm, CMP, bitlen, 1, no_status); + share* oldlam = lambda; + lambda = c->PutMUXGate(smallerLambda, biggerLambda, errWentDown); + delete oldlam; + delete smallerLambda; + delete biggerLambda; + delete errWentDown; + + share* lambdaTooBig = + c->PutFPGate(lambda, lambda_max_gate, CMP, bitlen, 1, no_status); + oldlam = lambda; + lambda = c->PutMUXGate(lambda_max_gate, lambda, lambdaTooBig); + delete oldlam; + delete lambdaTooBig; + share* lambdaTooSmall = + c->PutFPGate(lambda_min_gate, lambda, CMP, bitlen, 1, no_status); + oldlam = lambda; + share* newlambda = c->PutMUXGate(lambda_min_gate, lambda, lambdaTooSmall); + delete oldlam; + delete lambdaTooSmall; + if (_verbosity & DBG_LAMBDA) { + c->PutPrintValueGate(newlambda, "lambda"); + } + + // update pose - note: this is usually done after checking error + for (int p = 0; p < 6; p++) { +#if PPL_FLOW == PPL_FLOW_DO // set dx to zero if no error + share* olddx = dx[p]; + share* zero_gate = c->PutCONSGate((uint32_t*)&zero, bitlen); + dx[p] = c->PutMUXGate(dx[p], zero_gate, er_flag); + delete olddx; +#endif + temp = x[p]; + x[p] = c->PutFPGate(x[p], dx[p], ADD, bitlen, 1, no_status); + delete temp; + delete dx[p]; + } + + delete er_flag; + return {absnorm, newlambda}; +} + +// runs circuit using pre-shared inputs +bool RunLMIteration(uint32_t* threeDPts, uint32_t* y0, int numPts, uint32_t f, + uint32_t cx, uint32_t cy, uint32_t* x, + uint32_t* prevErrNorm, uint32_t* lambda, BooleanCircuit* c, + ABYParty* p, e_role role) { + + std::vector& sharings = p->GetSharings(); + BooleanCircuit* bc = + (BooleanCircuit*)sharings[S_BOOL]->GetCircuitBuildRoutine(); + + // Allocate space for shares + share** s_threeDPts = new share*[4 * numPts]; // leave room for contant 1's + share** s_y0 = new share*[2 * numPts]; + share** s_x = new share*[6]; + share* s_f; + share* s_cx; + share* s_cy; + share* s_prevErrNorm; + share* s_lambda; + share* temp; + + // Prepare inputs + for (int p = 0; p < 3 * numPts; p++) { + s_threeDPts[p] = bc->PutSharedINGate(threeDPts[p], bitlen); + } + for (int p = 0; p < 2 * numPts; p++) { + s_y0[p] = bc->PutSharedINGate(y0[p], bitlen); + } + for (int p = 0; p < 6; p++) { + s_x[p] = bc->PutSharedINGate(x[p], bitlen); + } + s_f = bc->PutSharedINGate(f, bitlen); + s_cx = bc->PutSharedINGate(cx, bitlen); + s_cy = bc->PutSharedINGate(cy, bitlen); + s_prevErrNorm = bc->PutSharedINGate(*prevErrNorm, bitlen); + s_lambda = bc->PutSharedINGate(*lambda, bitlen); + // if yao, convert bool inputs to yao + if (c->GetContext() == S_YAO) { + for (int p = 0; p < 3 * numPts; p++) { + temp = s_threeDPts[p]; + s_threeDPts[p] = c->PutB2YGate(temp); + delete temp; + } + for (int p = 0; p < 2 * numPts; p++) { + temp = s_y0[p]; + s_y0[p] = c->PutB2YGate(temp); + delete temp; + } + for (int p = 0; p < 6; p++) { + temp = s_x[p]; + s_x[p] = c->PutB2YGate(temp); + delete temp; + } + temp = s_f; + s_f = c->PutB2YGate(temp); + delete temp; + temp = s_cx; + s_cx = c->PutB2YGate(temp); + delete temp; + temp = s_cy; + s_cy = c->PutB2YGate(temp); + delete temp; + temp = s_prevErrNorm; + s_prevErrNorm = c->PutB2YGate(temp); + delete temp; + temp = s_lambda; + s_lambda = c->PutB2YGate(temp); + delete temp; + } + // build circuit + auto s_res = BuildLMIteration(s_threeDPts, s_y0, numPts, s_f, s_cx, s_cy, s_x, + s_prevErrNorm, s_lambda, c, p, role); + + // manage the outputs + share* s_absnorm = s_res.first; + share* s_newlambda = s_res.second; + float min_er = MIN_ER; + share* s_minErGate = c->PutCONSGate((uint32_t*)&min_er, bitlen); + share* s_done = + c->PutFPGate(s_minErGate, s_absnorm, CMP, bitlen, 1, no_status); + delete s_minErGate; + + // if yao was used, convert from yao back to boolean circuit + if (c->GetContext() == S_YAO) { + for (int p = 0; p < 6; p++) { + temp = s_x[p]; + s_x[p] = bc->PutY2BGate(temp); + delete temp; + } + temp = s_absnorm; + s_absnorm = bc->PutY2BGate(temp); + delete temp; + temp = s_newlambda; + s_newlambda = bc->PutY2BGate(temp); + delete temp; + // do not convert done, it is not raw share + } + + // prepare output (only need x and done flag) + for (int p = 0; p < 6; p++) { + temp = s_x[p]; + s_x[p] = bc->PutSharedOUTGate(temp); // raw share + delete temp; + } + temp = s_absnorm; + s_absnorm = bc->PutSharedOUTGate(temp); // raw share + delete temp; + temp = s_newlambda; + s_newlambda = bc->PutSharedOUTGate(temp); // raw share + delete temp; + temp = s_done; + s_done = c->PutOUTGate(temp, ALL); // loop leak needs cleartext + delete temp; + + p->ExecCircuit(); + + // get shared output + for (int p = 0; p < 6; p++) { + x[p] = s_x[p]->get_clear_value(); + delete s_x[p]; + } + delete[] s_x; + *prevErrNorm = s_absnorm->get_clear_value(); + delete s_absnorm; + *lambda = s_newlambda->get_clear_value(); + delete s_newlambda; + uint32_t done = s_done->get_clear_value(); + delete s_done; + + // cleanup + for (int p = 0; p < 3 * numPts; p++) { // ignore constant ones? + delete s_threeDPts[p]; + } + delete[] s_threeDPts; + for (int p = 0; p < 2 * numPts; p++) { + delete s_y0[p]; + } + delete[] s_y0; + + collectTiming(); + collectCommunication(); + p->Reset(); + return done != 0; +} + +// uint32_t arguments must be from PutSharedOUTGate() +// then calling get_clear_value() and circuit->Reset(); +// This function builds/executes multiple circuits. +void RunLMCircuit(uint32_t* threeDPts, uint32_t* y0, int numPts, uint32_t f, + uint32_t cx, uint32_t cy, uint32_t* x, uint32_t* prevErrNorm, + uint32_t* lambda, BooleanCircuit* c, ABYParty* party, + e_role role) { + // LM Iteration + for (int i = 0; i < LM_MAX_ITR; i++) { + if (_verbosity & DBG_PROJECT) { + cout << RED << "\n\nLM Iteration " << i << RESET << endl; + } + + bool done = RunLMIteration(threeDPts, y0, numPts, f, cx, cy, x, prevErrNorm, + lambda, c, party, role); + + // break if error under threshold + if (done) + break; + } +} + +// Wrapper function around RunLMCircuit which +// takes shares instead of secret shares. +// This (and inner function calls) creates multiple +// circuits such that whenever an intermediate plaintext value +// is required, the circuit ends and a new circuit begins. +// +// Doing so has the overhead of converting intermediate values +// to secret shares to be used in the next circuit execution. +// It does not, however, require any of the "debug" functionality +// from the ABY library to retreive intermediate values. +// +// This wrapper creates dummy circuits to build secret shares +// to be passed to the top level LM function. +// +// Note: All 2D matrix inputs passed as single dimension array +// Note: the constant 1's can be ignored (but space must be allocated) +// +// 3D World Points (4xnumPts) which look like: +// _ _ +// | x1 x2 x3 | +// | y1 y2 y3 ... | +// | z1 z2 z3 | +// |_ 1 1 1 _| +// must be pass as single dimension: +// [x1, x2, ...; y1, y2, ...; z1, z2, ...; 1, 1, ... ] +// +// 2D Image Points (3xnumPts) which look like: +// _ _ +// | u1 u2 u3 | +// | v1 v2 v3 ... | +// |_ 1 1 1 _| +// must be passed as single dimension: +// [ u1, u2, ...; v1, v2, ...; 1, 1, ... ] +// +// x (initial guess) : 6x1 [ rotation angles, translations ] +void BuildAndRunLMLoopLeak(share* s_threeDPts[], share* s_twoDPts[], int numPts, + share* s_f, share* s_cx, share* s_cy, share* s_x[], + BooleanCircuit* c, ABYParty* party, e_role role) { + std::vector& sharings = party->GetSharings(); + BooleanCircuit* bc = + (BooleanCircuit*)sharings[S_BOOL]->GetCircuitBuildRoutine(); + + share* temp; + + // initialize state required between iterations + float prevErrNorm_init = std::numeric_limits::max(); + share* s_prevErrNorm = bc->PutCONSGate((uint32_t*)&prevErrNorm_init, bitlen); + float lambda_init = LM_LAMBDA_INIT; + share* s_lambda = bc->PutCONSGate((uint32_t*)&lambda_init, bitlen); + + // First transform input shares + // + // throw away last "row" of 2D points, + // reshape, and transpose into 2nx1 vector + // e.g. [x1; y1; x2; y2 ...] + share** s_y0 = new share*[2 * numPts]; + for (int p = 0; p < numPts; ++p) { + s_y0[2 * p] = s_twoDPts[p]; + s_y0[(2 * p) + 1] = s_twoDPts[p + numPts]; + } + + // Next, run dummy circuit to create raw shares from share objects + // stores secret-shared output + uint32_t* threeDPts = new uint32_t[4 * numPts]; + uint32_t* y0 = new uint32_t[2 * numPts]; + uint32_t* x = new uint32_t[6]; + uint32_t f; + uint32_t cx; + uint32_t cy; + uint32_t prevErrNorm; + uint32_t lambda; + + // if yao is prefered circuit and yao shares passed in, must + // convert to bool to get raw shares. + // Tests must pass in bool shares even if yao is preferred. + if (c->GetContext() == S_YAO && s_threeDPts[0]->get_share_type() == S_YAO) { + for (int p = 0; p < 3 * numPts; p++) { + temp = s_threeDPts[p]; + s_threeDPts[p] = bc->PutY2BGate(temp); + delete temp; + } + for (int p = 0; p < 2 * numPts; p++) { + temp = s_y0[p]; + s_y0[p] = bc->PutY2BGate(temp); + delete temp; + } + for (int p = 0; p < 6; p++) { + temp = s_x[p]; + s_x[p] = bc->PutY2BGate(temp); + delete temp; + } + temp = s_f; + s_f = bc->PutY2BGate(temp); + delete temp; + temp = s_cx; + s_cx = bc->PutY2BGate(temp); + delete temp; + temp = s_cy; + s_cy = bc->PutY2BGate(temp); + delete temp; + // dont need prevErrNorm or lambda, they're always created with bool circuit + } + + // build shared output objects to get share, ignore constant 1s + for (int p = 0; p < 3 * numPts; p++) { + temp = s_threeDPts[p]; + s_threeDPts[p] = bc->PutSharedOUTGate(temp); + delete temp; + } + for (int p = 0; p < 2 * numPts; p++) { + temp = s_y0[p]; + s_y0[p] = bc->PutSharedOUTGate(temp); + delete temp; + } + for (int p = 0; p < 6; p++) { + temp = s_x[p]; + s_x[p] = bc->PutSharedOUTGate(temp); + delete temp; + } + temp = s_f; + s_f = bc->PutSharedOUTGate(temp); + delete temp; + temp = s_cx; + s_cx = bc->PutSharedOUTGate(temp); + delete temp; + temp = s_cy; + s_cy = bc->PutSharedOUTGate(temp); + delete temp; + temp = s_prevErrNorm; + s_prevErrNorm = bc->PutSharedOUTGate(temp); + delete temp; + temp = s_lambda; + s_lambda = bc->PutSharedOUTGate(temp); + delete temp; + // run the dummy circuit + CLOCK(ShareInputs); + TIC(ShareInputs); + party->ExecCircuit(); + TOC(ShareInputs); + // get shared outputs, ignore constant 1s + for (int p = 0; p < 3 * numPts; p++) { + threeDPts[p] = s_threeDPts[p]->get_clear_value(); + delete s_threeDPts[p]; + } + for (int p = 0; p < 2 * numPts; p++) { + y0[p] = s_y0[p]->get_clear_value(); + delete s_y0[p]; + } + for (int p = 0; p < 6; p++) { + x[p] = s_x[p]->get_clear_value(); + delete s_x[p]; + } + f = s_f->get_clear_value(); + delete s_f; + cx = s_cx->get_clear_value(); + delete s_cx; + cy = s_cy->get_clear_value(); + delete s_cy; + prevErrNorm = s_prevErrNorm->get_clear_value(); + delete s_prevErrNorm; + lambda = s_lambda->get_clear_value(); + delete s_lambda; + + collectTiming(); + collectCommunication(); + party->Reset(); + // arrays now contain secret shared values of plaintext from each party + + CLOCK(LM); + TIC(LM); + RunLMCircuit(threeDPts, y0, numPts, f, cx, cy, x, &prevErrNorm, &lambda, c, + party, role); + TOC(LM); + + // next start another dummy circuit to return share + // objects which are built from shares + for (int p = 0; p < 6; p++) { + s_x[p] = bc->PutSharedINGate(&x[p], bitlen); + } +} + +// Data Oblivious version of GN. Does not require executing +// multiple circuits to retreive intermediate values. +// +// Note: All 2D matrix inputs passed as single dimension array +// Note: the constant 1's can be ignored (but space must be allocated) +// +// 3D World Points (4xnumPts) which look like: +// _ _ +// | x1 x2 x3 | +// | y1 y2 y3 ... | +// | z1 z2 z3 | +// |_ 1 1 1 _| +// must be pass as single dimension: +// [x1, x2, ...; y1, y2, ...; z1, z2, ...; 1, 1, ... ] +// +// 2D Image Points (3xnumPts) which look like: +// _ _ +// | u1 u2 u3 | +// | v1 v2 v3 ... | +// |_ 1 1 1 _| +// must be passed as single dimension: +// [ u1, u2, ...; v1, v2, ...; 1, 1, ... ] +// +// x (initial guess) : 6x1 [ rotation angles, translations ] +void BuildAndRunLMDO(share* s_threeDPts[], share* s_twoDPts[], int numPts, + share* s_f, share* s_cx, share* s_cy, share* s_x[], + BooleanCircuit* c, ABYParty* party, e_role role) { + // First transform input shares + // + // throw away last "row" of 2D points, + // reshape, and transpose into 2nx1 vector + // e.g. [x1; y1; x2; y2 ...] + share** s_y0 = new share*[2 * numPts]; + for (int p = 0; p < numPts; ++p) { + s_y0[2 * p] = s_twoDPts[p]; + s_y0[(2 * p) + 1] = s_twoDPts[p + numPts]; + } + + // LM Iteration + float lambda = LM_LAMBDA_INIT; + share* lambda_gate = c->PutCONSGate((uint32_t*)&lambda, bitlen); + + float f_max = std::numeric_limits::max(); + share* prev_err_norm_gate = c->PutCONSGate((uint32_t*)&f_max, bitlen); + + for (int i = 0; i < LM_MAX_ITR; i++) { + if (_verbosity & DBG_PROJECT) { + cout << RED << "\n\nLM Iteration " << i << RESET << endl; + } + + auto res = + BuildLMIteration(s_threeDPts, s_y0, numPts, s_f, s_cx, s_cy, s_x, + prev_err_norm_gate, lambda_gate, c, party, role); + prev_err_norm_gate = res.first; + lambda_gate = res.second; + } +} + +void BuildAndRunLM(share* s_threeDPts[], share* s_twoDPts[], int numPts, + share* s_f, share* s_cx, share* s_cy, share* s_x[], + BooleanCircuit* c, ABYParty* party, e_role role) { +#if PPL_FLOW == PPL_FLOW_DO + BuildAndRunLMDO(s_threeDPts, s_twoDPts, numPts, s_f, s_cx, s_cy, s_x, c, + party, role); +#elif PPL_FLOW == PPL_FLOW_LOOP_LEAK || PPL_FLOW == PPL_FLOW_SiSL + BuildAndRunLMLoopLeak(s_threeDPts, s_twoDPts, numPts, s_f, s_cx, s_cy, s_x, c, + party, role); +#endif +} + +uint32_t test_lm_circuit( + e_role role, const std::string& address, uint16_t port, seclvl seclvl, + uint32_t nthreads, e_mt_gen_alg mt_alg, e_sharing sharing, + vector threeDPts, vector twoDPts, float f, + float cx, float cy, + float* x /* initial guess for { r1, r2, r3, t1, t2, t3 } */) { + + uint32_t reservegates = 65536; + const std::string& abycircdir = "../../extern/ABY/bin/circ"; + ABYParty* party = new ABYParty(role, address, port, seclvl, bitlen, nthreads, + mt_alg, reservegates, abycircdir); + std::vector& sharings = party->GetSharings(); + Circuit* circ = sharings[sharing]->GetCircuitBuildRoutine(); +#if PPL_FLOW == PPL_FLOW_LOOP_LEAK + // always use boolean shares (not yao) so BuildAndRunLM can get raw shares. + // this is because you cant use Y2B shares on Yao input gates. + Circuit* bc = sharings[S_BOOL]->GetCircuitBuildRoutine(); +#else + // if data oblivious, either yao or bool can be used + Circuit* bc = circ; +#endif + + // sharings[S_BOOL]->SetPreCompPhaseValue(ePreCompRAMWrite); + + int numPts = threeDPts.size(); + + // Allocate space for shares + share** s_threeDPts = new share*[4 * numPts]; + share** s_twoDPts = new share*[3 * numPts]; + share* s_f; + share* s_cx; + share* s_cy; + share** s_x = new share*[6]; + + // Prepare inputs + if (role == SERVER) { + // float one=1.0; + for (int p = 0; p < numPts; p++) { + s_threeDPts[p] = bc->PutINGate((uint32_t*)&threeDPts[p].x, bitlen, role); + } + for (int p = 0; p < numPts; p++) { + s_threeDPts[numPts + p] = + bc->PutINGate((uint32_t*)&threeDPts[p].y, bitlen, role); + } + for (int p = 0; p < numPts; p++) { + s_threeDPts[(2 * numPts) + p] = + bc->PutINGate((uint32_t*)&threeDPts[p].z, bitlen, role); + } + // for(int p=0; pPutINGate((uint32_t*) &one, bitlen, + // role); + // } + for (int p = 0; p < numPts; p++) { + s_twoDPts[p] = bc->PutINGate((uint32_t*)&twoDPts[p].x, bitlen, role); + } + for (int p = 0; p < numPts; p++) { + s_twoDPts[numPts + p] = + bc->PutINGate((uint32_t*)&twoDPts[p].y, bitlen, role); + } + // for(int p=0; pPutINGate((uint32_t*) &one, bitlen, + // role); + // } + s_f = bc->PutINGate((uint32_t*)&f, bitlen, role); + s_cx = bc->PutINGate((uint32_t*)&cx, bitlen, role); + s_cy = bc->PutINGate((uint32_t*)&cy, bitlen, role); + for (int p = 0; p < 6; p++) { + s_x[p] = bc->PutINGate((uint32_t*)&x[p], bitlen, role); + } + } else { + for (int p = 0; p < 3 * numPts; p++) { // ignore constant 1s + s_threeDPts[p] = bc->PutDummyINGate(bitlen); + } + for (int p = 0; p < 2 * numPts; p++) { // ignore constant 1s + s_twoDPts[p] = bc->PutDummyINGate(bitlen); + } + s_f = bc->PutDummyINGate(bitlen); + s_cx = bc->PutDummyINGate(bitlen); + s_cy = bc->PutDummyINGate(bitlen); + for (int p = 0; p < 6; p++) { + s_x[p] = bc->PutDummyINGate(bitlen); + } + } + + BuildAndRunLM(s_threeDPts, s_twoDPts, numPts, s_f, s_cx, s_cy, s_x, + (BooleanCircuit*)circ, party, role); + + for (int i = 0; i < 6; i++) { + share* temp = s_x[i]; + s_x[i] = bc->PutOUTGate(s_x[i], ALL); + delete temp; + } + + party->ExecCircuit(); + + for (int i = 0; i < 6; i++) { + uint32_t* output; + uint32_t out_bitlen, out_nvals; + + // This method only works for an output length of maximum 64 bits in + // general, if the output length is higher you must use get_clear_value_ptr + s_x[i]->get_clear_value_vec(&output, &out_bitlen, &out_nvals); + // s_out[i]->get_clear_value_ptr(); + + // cout << *(float*)output << " "; + + x[i] = *(float*)output; + } + + delete party; + + return 0; +} diff --git a/src/aby-float-server/lmlocalization.h b/src/aby-float-server/lmlocalization.h new file mode 100644 index 0000000..193c8d5 --- /dev/null +++ b/src/aby-float-server/lmlocalization.h @@ -0,0 +1,26 @@ +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +using namespace std; + +void BuildAndRunLM(share* s_threeDPts[], share* s_twoDPts[], int numPts, + share* s_f, share* s_cx, share* s_cy, share* s_x[], + BooleanCircuit* c, ABYParty* party, e_role role); + +uint32_t test_lm_circuit( + e_role role, const std::string& address, uint16_t port, seclvl seclvl, + uint32_t nthreads, e_mt_gen_alg mt_alg, e_sharing sharing, + vector threeDPts, vector twoDPts, float f, + float cx, float cy, + float* x /* initial guess for { r1, r2, r3, t1, t2, t3 } */); diff --git a/src/aby-float-server/matmult.cpp b/src/aby-float-server/matmult.cpp new file mode 100644 index 0000000..d1be03b --- /dev/null +++ b/src/aby-float-server/matmult.cpp @@ -0,0 +1,76 @@ +#include +#include +#include +#include + +#include "abycore/aby/abyparty.h" +#include "abycore/circuit/arithmeticcircuits.h" +#include "abycore/circuit/booleancircuits.h" +#include "abycore/circuit/circuit.h" +#include "abycore/sharing/sharing.h" + +using namespace std; + +const uint32_t bitlen = 32; + +// M mxn (M is secret, m n are public) +// N mmxnn +// res mxnn +void BuildMatmultCircuit(share* M[], int m, int n, share* N[], int mm, int nn, + share* res[], BooleanCircuit* c) { + (void)mm; + assert(n == mm); + for (int i = 0; i < m; i++) { // row + for (int j = 0; j < nn; j++) { // col + float z = 0; + res[i * nn + j] = c->PutCONSGate((uint32_t*)&z, 32); + for (int k = 0; k < n; k++) { + share* temp = c->PutFPGate(M[i * n + k], N[k * nn + j], MUL, bitlen, 1, + no_status); + share* temp2 = res[i * nn + j]; + res[i * nn + j] = + c->PutFPGate(res[i * nn + j], temp, ADD, bitlen, 1, no_status); + delete temp; + delete temp2; + } + } + } +} + +// M mxn (M is secret, m n are public) +// N mmxnn +// res mxnn +void BuildMatmult2DwTransposeCircuit(share** M[], int m, int n, + const bool transposeM, share** N[], int mm, + int nn, const bool transposeN, + share** res[], BooleanCircuit* c) { + if (transposeM) + std::swap(m, n); + if (transposeN) + std::swap(mm, nn); + assert(n == mm); + + float z = 0; + for (int i = 0; i < m; i++) { // row + for (int j = 0; j < nn; j++) { // col + res[i][j] = c->PutCONSGate((uint32_t*)&z, 32); + + for (int k = 0; k < n; k++) { + share* prod; + if (!transposeM && !transposeN) + prod = c->PutFPGate(M[i][k], N[k][j], MUL, bitlen, 1, no_status); + else if (!transposeM && transposeN) + prod = c->PutFPGate(M[i][k], N[j][k], MUL, bitlen, 1, no_status); + else if (transposeM && !transposeN) + prod = c->PutFPGate(M[k][i], N[k][j], MUL, bitlen, 1, no_status); + else if (transposeM && transposeN) + prod = c->PutFPGate(M[k][i], N[j][k], MUL, bitlen, 1, no_status); + + share* oldres = res[i][j]; + res[i][j] = c->PutFPGate(res[i][j], prod, ADD, bitlen, 1, no_status); + delete prod; + delete oldres; + } + } + } +} diff --git a/src/aby-float-server/matmult.h b/src/aby-float-server/matmult.h new file mode 100644 index 0000000..0026b58 --- /dev/null +++ b/src/aby-float-server/matmult.h @@ -0,0 +1,19 @@ +#include "abycore/aby/abyparty.h" +#include "abycore/circuit/arithmeticcircuits.h" +#include "abycore/circuit/booleancircuits.h" +#include "abycore/circuit/circuit.h" +#include "abycore/sharing/sharing.h" + +// M mxn (M is secret, m n are public) +// N mmxnn +// res mxnn +void BuildMatmultCircuit(share* M[], int m, int n, share* N[], int mm, int nn, + share* res[], BooleanCircuit* c); + +// M mxn (M is secret, m n are public) +// N mmxnn +// res mxnn +void BuildMatmult2DwTransposeCircuit(share** M[], int m, int n, + const bool transposeM, share** N[], int mm, + int nn, const bool transposeN, + share** res[], BooleanCircuit* c); diff --git a/src/aby-float-server/projectpoints.cpp b/src/aby-float-server/projectpoints.cpp new file mode 100644 index 0000000..5ce173d --- /dev/null +++ b/src/aby-float-server/projectpoints.cpp @@ -0,0 +1,63 @@ +#include +#include +#include +#include + +#include "abycore/aby/abyparty.h" +#include "abycore/circuit/arithmeticcircuits.h" +#include "abycore/circuit/booleancircuits.h" +#include "abycore/circuit/circuit.h" +#include "abycore/sharing/sharing.h" + +#include +#include + +using namespace std; + +// Note: All 2D matrix inputs passed as single dimension array +// P=4xnumPoints [x1, x2, ...; y1, y2, ...; z1, z2, ...; 1, 1, ... ] +// x=6x1 [ rotation angles; translation] (column) +// K=3x3 camera intrinsic +// result=3xnumPoints preallocated, only first two rows are x,y. +// last row needed for intermediate calculation (homogeneous) +// [ u1, u2, ...; v1, v2, ...; 1, 1, ... ] +void BuildProjectPointsCircuit(share* _P[], share* _x[], share* _K[], + share* _res[], int numPoints, BooleanCircuit* c, + bool skipOneGate) { + + uint32_t bitlen = 32; + + float one = 1; + share* one_gate = c->PutCONSGate((uint32_t*)&one, bitlen); + + share* _R[12]; + + BuildRodriguesCircuit(_x, _R, (BooleanCircuit*)c); + + // copy translation into last column of R + _R[3] = _x[3]; + _R[7] = _x[4]; + _R[11] = _x[5]; + + // res = K*R*P; + share* scratch[3 * 4]; + BuildMatmultCircuit(_K, 3, 3, _R, 3, 4, scratch, c); + BuildMatmultCircuit(scratch, 3, 4, _P, 4, numPoints, _res, c); + + // homogeneous coords to x,y + for (int i = 0; i < numPoints; i++) { + _res[i] = c->PutFPGate(_res[i], _res[2 * numPoints + i], DIV, bitlen, 1, + no_status); + _res[numPoints + i] = + c->PutFPGate(_res[numPoints + i], _res[2 * numPoints + i], DIV, bitlen, + 1, no_status); + if (!skipOneGate) { + _res[2 * numPoints + i] = one_gate; // can be excluded for efficiency + } + } + + for (int i = 0; i < 12; i++) { + if (i % 4 != 3) + delete _R[i]; // skip 3,7,11 (don't delete _x's) + } +} diff --git a/src/aby-float-server/projectpoints.h b/src/aby-float-server/projectpoints.h new file mode 100644 index 0000000..c708d02 --- /dev/null +++ b/src/aby-float-server/projectpoints.h @@ -0,0 +1,19 @@ +#include "abycore/aby/abyparty.h" +#include "abycore/circuit/arithmeticcircuits.h" +#include "abycore/circuit/booleancircuits.h" +#include "abycore/circuit/circuit.h" +#include "abycore/sharing/sharing.h" + +#include +#include + +// Note: All 2D matrix inputs passed as single dimension array +// P=4xnumPoints [x1, x2, ...; y1, y2, ...; z1, z2, ...; 1, 1, ... ] +// x=6x1 [ rotation angles; translation] (column) +// K=3x3 camera intrinsic +// result=3xnumPoints preallocated, only first two rows are x,y. +// last row needed for intermediate calculation (homogeneous) +// [ u1, u2, ...; v1, v2, ...; 1, 1, ... ] +void BuildProjectPointsCircuit(share* _P[], share* _x[], share* _K[], + share* _res[], int numPoints, BooleanCircuit* c, + bool skipOneGate = false); diff --git a/src/aby-float-server/rodrigues.cpp b/src/aby-float-server/rodrigues.cpp new file mode 100644 index 0000000..563dce3 --- /dev/null +++ b/src/aby-float-server/rodrigues.cpp @@ -0,0 +1,146 @@ +#include +#include +#include +#include + +//#include +#include +#include +#include +#include + +#include "abycore/aby/abyparty.h" +#include "abycore/circuit/arithmeticcircuits.h" +#include "abycore/circuit/booleancircuits.h" +#include "abycore/circuit/circuit.h" +#include "abycore/sharing/sharing.h" + +#include +#include + +using namespace std; + +// r=3x1, R=3x4 (only 3x3 is used here) +// values are secret, sizes are public +void BuildRodriguesCircuit(share* r[], share* R[], BooleanCircuit* c) { + + uint32_t bitlen = 32; + float one = 1; + float zero = 0; + share* one_gate = c->PutCONSGate((uint32_t*)&one, bitlen); + share* zero_gate = c->PutCONSGate((uint32_t*)&zero, bitlen); + + // calculate theta + share* rzerosq = c->PutFPGate(r[0], r[0], MUL, bitlen, 1, no_status); + share* ronesq = c->PutFPGate(r[1], r[1], MUL, bitlen, 1, no_status); + share* rtwosq = c->PutFPGate(r[2], r[2], MUL, bitlen, 1, no_status); + share* theta1 = c->PutFPGate(rzerosq, ronesq, ADD, bitlen, 1, no_status); + share* theta2 = c->PutFPGate(theta1, rtwosq, ADD, bitlen, 1, no_status); + share* theta = c->PutFPGate(theta2, SQRT); + + share* co = BuildCosCircuit(theta, c); + share* si = BuildSinCircuit(theta, c); + + share* c1 = c->PutFPGate(one_gate, co, SUB, bitlen, 1, no_status); + + share* itheta = c->PutFPGate(one_gate, theta, DIV, bitlen, 1, no_status); + // if theta == 0 then itheta = 0 + share* thetanotz = theta->get_wire_ids_as_share(0); + for (uint32_t tt = 1; tt < bitlen - 1; + tt++) { // -1 -> do not include sign bit + share* tempbit = theta->get_wire_ids_as_share(tt); + share* temp = thetanotz; + thetanotz = c->PutORGate(temp, tempbit); + delete tempbit; + delete temp; + } + share* tempitheta = itheta; + itheta = c->PutMUXGate(itheta, zero_gate, thetanotz); + delete tempitheta; + + share* x = c->PutFPGate(r[0], itheta, MUL, bitlen, 1, no_status); + share* y = c->PutFPGate(r[1], itheta, MUL, bitlen, 1, no_status); + share* z = c->PutFPGate(r[2], itheta, MUL, bitlen, 1, no_status); + + share* xx = c->PutFPGate(x, x, MUL, bitlen, 1, no_status); + share* yy = c->PutFPGate(y, y, MUL, bitlen, 1, no_status); + share* zz = c->PutFPGate(z, z, MUL, bitlen, 1, no_status); + + share* xy = c->PutFPGate(x, y, MUL, bitlen, 1, no_status); + share* xz = c->PutFPGate(x, z, MUL, bitlen, 1, no_status); + share* yz = c->PutFPGate(y, z, MUL, bitlen, 1, no_status); + + share* sx = c->PutFPGate(si, x, MUL, bitlen, 1, no_status); + share* sy = c->PutFPGate(si, y, MUL, bitlen, 1, no_status); + share* sz = c->PutFPGate(si, z, MUL, bitlen, 1, no_status); + + share* temp; + + temp = c->PutFPGate(c1, xx, MUL, bitlen, 1, no_status); + R[0] = c->PutFPGate(temp, co, ADD, bitlen, 1, no_status); + delete temp; + + temp = c->PutFPGate(c1, xy, MUL, bitlen, 1, no_status); + R[1] = c->PutFPGate(temp, sz, SUB, bitlen, 1, no_status); + delete temp; + + temp = c->PutFPGate(c1, xz, MUL, bitlen, 1, no_status); + R[2] = c->PutFPGate(temp, sy, ADD, bitlen, 1, no_status); + delete temp; + + temp = c->PutFPGate(c1, xy, MUL, bitlen, 1, no_status); + R[4] = c->PutFPGate(temp, sz, ADD, bitlen, 1, no_status); + delete temp; + + temp = c->PutFPGate(c1, yy, MUL, bitlen, 1, no_status); + R[5] = c->PutFPGate(temp, co, ADD, bitlen, 1, no_status); + delete temp; + + temp = c->PutFPGate(c1, yz, MUL, bitlen, 1, no_status); + R[6] = c->PutFPGate(temp, sx, SUB, bitlen, 1, no_status); + delete temp; + + temp = c->PutFPGate(c1, xz, MUL, bitlen, 1, no_status); + R[8] = c->PutFPGate(temp, sy, SUB, bitlen, 1, no_status); + delete temp; + + temp = c->PutFPGate(c1, yz, MUL, bitlen, 1, no_status); + R[9] = c->PutFPGate(temp, sx, ADD, bitlen, 1, no_status); + delete temp; + + temp = c->PutFPGate(c1, zz, MUL, bitlen, 1, no_status); + R[10] = c->PutFPGate(temp, co, ADD, bitlen, 1, no_status); + delete temp; + + delete rzerosq; + delete ronesq; + delete rtwosq; + + delete theta1; + delete theta2; + delete theta; + delete itheta; + + delete c1; + delete co; + delete si; + + delete x; + delete y; + delete z; + + delete xx; + delete yy; + delete zz; + + delete xy; + delete xz; + delete yz; + + delete sx; + delete sy; + delete sz; + + delete one_gate; + delete zero_gate; +} diff --git a/src/aby-float-server/rodrigues.h b/src/aby-float-server/rodrigues.h new file mode 100644 index 0000000..64d0474 --- /dev/null +++ b/src/aby-float-server/rodrigues.h @@ -0,0 +1,20 @@ +#include +#include +#include +#include + +//#include +//#include +//#include +//#include +//#include + +#include "abycore/aby/abyparty.h" +#include "abycore/circuit/arithmeticcircuits.h" +#include "abycore/circuit/booleancircuits.h" +#include "abycore/circuit/circuit.h" +#include "abycore/sharing/sharing.h" + +// r=3x1, R=3x4 (only 3x3 is used here) +// values are secret, sizes are public +void BuildRodriguesCircuit(share* r[], share* R[], BooleanCircuit* c); diff --git a/src/aby-float-server/svd.cpp b/src/aby-float-server/svd.cpp new file mode 100644 index 0000000..ec58f39 --- /dev/null +++ b/src/aby-float-server/svd.cpp @@ -0,0 +1,2405 @@ +#include +#include +#include +#include +#include +#include + +#include "abycore/aby/abyparty.h" +#include "abycore/circuit/arithmeticcircuits.h" +#include "abycore/circuit/booleancircuits.h" +#include "abycore/circuit/circuit.h" +#include "abycore/circuit/share.h" +#include "abycore/sharing/sharing.h" + +#include + +#include + +#define SIGN(a, b) ((b) > 0.0 ? fabs(a) : -fabs(a)) + +static float maxarg1, maxarg2; +#define FMAX(a, b) \ + (maxarg1 = (a), maxarg2 = (b), (maxarg1) > (maxarg2) ? (maxarg1) : (maxarg2)) + +static int iminarg1, iminarg2; +#define IMIN(a, b) \ + (iminarg1 = (a), iminarg2 = (b), \ + (iminarg1 < (iminarg2) ? (iminarg1) : iminarg2)) + +static float sqrarg; +#define SQR(a) ((sqrarg = (a)) == 0.0 ? 0.0 : sqrarg * sqrarg) + +using namespace std; + +const uint32_t bitlen = 32; + +//#define TWODINDEX(mat,ncols,a,b) (mat[(a*ncols)+b]) + +void BuildFabsCircuit(share* a, BooleanCircuit* c) { + // sets first bit of a to zero (can be public) + uint32_t zero = 0; + share* zero_gate = c->PutCONSGate((uint32_t*)&zero, 32); + a->set_wire_id(31, zero_gate->get_wire_id(31)); +} + +void BuildSignCircuit(share* a, share* b, BooleanCircuit* c) { + // WARNING: + // cannot simply set msb of a to msb of b because + // of the case when b is 0.0 + // TODO - fix memory leak + uint32_t zero = 0; + share* zero_gate = c->PutCONSGate((uint32_t*)&zero, 32); + share* cmp = c->PutFPGate(b, zero_gate, CMP); + share* tmp = c->PutINVGate(cmp); + a->set_wire_id(31, tmp->get_wire_id(0)); +} + +share* BuildFMAXCircuit(share* a, share* b, BooleanCircuit* c) { + // returns which a or b is bigger + share* cmp = c->PutFPGate(a, b, CMP); + share* ret = c->PutMUXGate(a, b, cmp); + delete cmp; + return ret; +} + +void BuildNegativeCircuit(share* a, BooleanCircuit* c) { + // flip msb + // TODO - fix memory leak + share* tmp = a->get_wire_ids_as_share(31); + tmp = c->PutINVGate(tmp); + // cout << tmp->get_bitlength() << endl; // tmp is one bit + a->set_wire_id(31, tmp->get_wire_id(0)); +} + +share* BuildPythagCircuit(share* a, share* b, BooleanCircuit* c, + uint32_t bitlen) { + share* aa = c->PutFPGate(a, a, MUL, bitlen, 1, no_status); + share* bb = c->PutFPGate(b, b, MUL, bitlen, 1, no_status); + share* sum = c->PutFPGate(aa, bb, ADD, bitlen, 1, no_status); + share* res = c->PutFPGate(sum, SQRT); + delete aa; + delete bb; + delete sum; + return res; + + // Doing the same pythag optimization as cleartext is + // actually slower than the long way to compute pythag + // float zero = 0; + // share* zero_gate = c->PutCONSGate((uint32_t*)&zero, bitlen); + // float one = 1; + // share* one_gate = c->PutCONSGate((uint32_t*)&one, bitlen); + + // boolshare *acopy = new boolshare(a->get_wires(), c);//copy to new share + // boolshare *bcopy = new boolshare(b->get_wires(), c);//copy to new share + // BuildFabsCircuit(acopy, c); + // BuildFabsCircuit(bcopy, c); + + // share *adb = c->PutFPGate(acopy, bcopy, DIV, bitlen, 1, no_status); + // share *sqadb = c->PutFPGate(adb, adb, MUL, bitlen, 1, no_status); + // share *addone = c->PutFPGate(sqadb, one_gate, ADD, bitlen, 1, no_status); + // share *sqrt = c->PutFPGate(addone, SQRT); + + // share *mul = BuildFMAXCircuit(acopy, bcopy, c); + + // share *res = c->PutFPGate(mul, sqrt, MUL, bitlen, 1, no_status); + + // share *cmp = c->PutFPGate(mul, zero_gate, CMP); // mul > 0 + // share *resorzero = c->PutMUXGate(res, zero_gate, cmp); + + // delete zero_gate; + // delete one_gate; + // delete acopy; + // delete bcopy; + // delete adb; + // delete sqadb; + // delete addone; + // delete sqrt; + // delete mul; + // delete cmp; + // delete res; + // return resorzero; +} + +// r=3x1, R=3x4 (only 3x3 is used here) +// values are secret, sizes are public +void BuildSvdPart1Circuit(share** a[], int nRows, int nCols, share* w[], + share** v[], share** anorm, share* rv1[], + BooleanCircuit* c) { + + float zero = 0; + share* zero_gate = c->PutCONSGate((uint32_t*)&zero, bitlen); + float one = 1; + share* one_gate = c->PutCONSGate((uint32_t*)&one, bitlen); + share* temp; + + int i, j, k, l; // local plaintext + share *f, *g, *h, *s, + *scale; // local secrets (are reset before being used elsewhere) + + // g = scale = anorm = zero_gate; + g = new boolshare(zero_gate->get_wires(), c); // copy to new share + scale = new boolshare(zero_gate->get_wires(), c); // copy to new share + *anorm = new boolshare(zero_gate->get_wires(), c); // copy to new share + + for (i = 0; i < nCols; i++) { + l = i + 1; + rv1[i] = c->PutFPGate(scale, g, MUL, bitlen, 1, no_status); + g = new boolshare(zero_gate->get_wires(), c); // copy to new share + s = new boolshare(zero_gate->get_wires(), c); // copy to new share + scale = new boolshare(zero_gate->get_wires(), c); // copy to new share + if (i < nRows) { + for (k = i; k < nRows; k++) { + boolshare* shcopy = + new boolshare(a[k][i]->get_wires(), c); // copy to new share + BuildFabsCircuit(shcopy, c); + scale = c->PutFPGate(scale, shcopy, ADD, bitlen, 1, no_status); + } + { // if (scale) + share* ifscale = scale->get_wire_ids_as_share(0); + for (uint32_t tt = 1; tt < bitlen - 1; + tt++) { // -1 -> do not include sign bit + share* tempbit = scale->get_wire_ids_as_share(tt); + temp = ifscale; + ifscale = c->PutORGate(temp, tempbit); + delete tempbit; + delete temp; + } + + for (k = i; k < nRows; k++) { + temp = c->PutFPGate(a[k][i], scale, DIV, bitlen, 1, no_status); + a[k][i] = c->PutMUXGate(temp, a[k][i], ifscale); + temp = c->PutFPGate(a[k][i], SQR, bitlen, 1, no_status); + s = c->PutFPGate(s, temp, ADD, bitlen, 1, no_status); + } + f = new boolshare(a[i][i]->get_wires(), c); // no mux okay + temp = c->PutFPGate(s, SQRT); + BuildSignCircuit(temp, f, c); + BuildNegativeCircuit(temp, c); + g = c->PutMUXGate(temp, g, ifscale); + temp = c->PutFPGate(f, g, MUL, bitlen, 1, no_status); + h = c->PutFPGate(temp, s, SUB, bitlen, 1, no_status); // no mux okay + temp = c->PutFPGate(f, g, SUB, bitlen, 1, no_status); + a[i][i] = c->PutMUXGate(temp, a[i][i], ifscale); + for (j = l; j < nCols; j++) { + s = new boolshare(zero_gate->get_wires(), c); + for (k = i; k < nRows; k++) { + temp = c->PutFPGate(a[k][i], a[k][j], MUL, bitlen, 1, no_status); + s = c->PutFPGate(s, temp, ADD, bitlen, 1, no_status); + } + f = c->PutFPGate(s, h, DIV, bitlen, 1, no_status); + for (k = i; k < nRows; k++) { + temp = c->PutFPGate(f, a[k][i], MUL, bitlen, 1, no_status); + temp = c->PutFPGate(a[k][j], temp, ADD, bitlen, 1, no_status); + a[k][j] = c->PutMUXGate(temp, a[k][j], ifscale); + } + } + for (k = i; k < nRows; k++) { + temp = c->PutFPGate(a[k][i], scale, MUL, bitlen, 1, no_status); + a[k][i] = c->PutMUXGate(temp, a[k][i], ifscale); + } + } + } + w[i] = c->PutFPGate(scale, g, MUL, bitlen, 1, no_status); + g = new boolshare(zero_gate->get_wires(), c); // copy to new share + s = new boolshare(zero_gate->get_wires(), c); // copy to new share + scale = new boolshare(zero_gate->get_wires(), c); // copy to new share + if (i < nRows && i != nCols - 1) { + for (k = l; k < nCols; k++) { + boolshare* shcopy = + new boolshare(a[i][k]->get_wires(), c); // copy to new share + BuildFabsCircuit(shcopy, c); + scale = c->PutFPGate(scale, shcopy, ADD, bitlen, 1, no_status); + } + { // if (scale) + share* ifscale = scale->get_wire_ids_as_share(0); + for (uint32_t tt = 1; tt < bitlen - 1; + tt++) { // -1 -> do not include sign bit + share* tempbit = scale->get_wire_ids_as_share(tt); + temp = ifscale; + ifscale = c->PutORGate(temp, tempbit); + delete tempbit; + delete temp; + } + + for (k = l; k < nCols; k++) { + a[i][k] = c->PutFPGate(a[i][k], scale, DIV, bitlen, 1, no_status); + temp = c->PutFPGate(a[i][k], SQR); //, bitlen, 1, no_status); + s = c->PutFPGate(s, temp, ADD, bitlen, 1, no_status); + } + f = new boolshare(a[i][l]->get_wires(), + c); // not used again, no mux okay + temp = c->PutFPGate(s, SQRT); + BuildSignCircuit(temp, f, c); + BuildNegativeCircuit(temp, c); + g = c->PutMUXGate(temp, g, ifscale); + temp = c->PutFPGate(f, g, MUL, bitlen, 1, no_status); + h = c->PutFPGate(temp, s, SUB, bitlen, 1, no_status); + share* htoone = new boolshare(one_gate->get_wires(), c); + h = c->PutMUXGate( + h, htoone, + ifscale); // h is used to set rv1 if scale==0, set to 1 for noop + temp = c->PutFPGate(f, g, SUB, bitlen, 1, no_status); + a[i][l] = c->PutMUXGate(temp, a[i][l], ifscale); + for (k = l; k < nCols; k++) { + rv1[k] = c->PutFPGate(a[i][k], h, DIV, bitlen, 1, + no_status); // no mux because h is one + } + for (j = l; j < nRows; j++) { + s = new boolshare(zero_gate->get_wires(), c); + for (k = l; k < nCols; k++) { + temp = c->PutFPGate(a[j][k], a[i][k], MUL, bitlen, 1, no_status); + s = c->PutFPGate(s, temp, ADD, bitlen, 1, no_status); + } + for (k = l; k < nCols; k++) { + temp = c->PutFPGate(s, rv1[k], MUL, bitlen, 1, no_status); + temp = c->PutFPGate(a[j][k], temp, ADD, bitlen, 1, no_status); + a[j][k] = c->PutMUXGate(temp, a[j][k], ifscale); + } + } + for (k = l; k < nCols; k++) { + temp = c->PutFPGate(a[i][k], scale, MUL, bitlen, 1, no_status); + a[i][k] = c->PutMUXGate(temp, a[i][k], ifscale); + } + } + } + boolshare* shcopy1 = + new boolshare(w[i]->get_wires(), c); // copy to new share + boolshare* shcopy2 = + new boolshare(rv1[i]->get_wires(), c); // copy to new share + + BuildFabsCircuit(shcopy1, c); + BuildFabsCircuit(shcopy2, c); + + temp = c->PutFPGate(shcopy1, shcopy2, ADD, bitlen, 1, no_status); + *anorm = BuildFMAXCircuit(*anorm, temp, c); + printf("."); + fflush(stdout); + } + + for (i = nCols - 1; i >= 0; i--) { + if (i < nCols - 1) { + { // if (g) + share* ifg = g->get_wire_ids_as_share(0); + for (uint32_t tt = 1; tt < bitlen - 1; + tt++) { // -1 -> do not include sign bit + share* tempbit = g->get_wire_ids_as_share(tt); + temp = ifg; + ifg = c->PutORGate(temp, tempbit); + delete tempbit; + delete temp; + } + for (j = l; j < nCols; j++) { + temp = c->PutFPGate(a[i][j], a[i][l], DIV, bitlen, 1, no_status); + temp = c->PutFPGate(temp, g, DIV, bitlen, 1, no_status); + v[j][i] = c->PutMUXGate(temp, g, ifg); + } + for (j = l; j < nCols; j++) { + s = new boolshare(zero_gate->get_wires(), c); + for (k = l; k < nCols; k++) { + temp = c->PutFPGate(a[i][k], v[k][j], MUL, bitlen, 1, no_status); + s = c->PutFPGate(s, temp, ADD, bitlen, 1, + no_status); // s only used here so no mux okay + } + for (k = l; k < nCols; k++) { + temp = c->PutFPGate(s, v[k][i], MUL, bitlen, 1, no_status); + temp = c->PutFPGate(v[k][j], temp, ADD, bitlen, 1, no_status); + v[k][j] = c->PutMUXGate(temp, v[k][j], ifg); + } + } + } + for (j = l; j < nCols; j++) { + v[i][j] = new boolshare(zero_gate->get_wires(), c); + v[j][i] = new boolshare(zero_gate->get_wires(), c); + } + } + v[i][i] = new boolshare(one_gate->get_wires(), c); + g = new boolshare(rv1[i]->get_wires(), c); + l = i; + printf(":"); + fflush(stdout); + } + + for (i = IMIN(nRows, nCols) - 1; i >= 0; i--) { + l = i + 1; + g = new boolshare(w[i]->get_wires(), c); + for (j = l; j < nCols; j++) + a[i][j] = new boolshare(zero_gate->get_wires(), c); + { // if (g) + share* ifg = g->get_wire_ids_as_share(0); + for (uint32_t tt = 1; tt < bitlen - 1; + tt++) { // -1 -> do not include sign bit + share* tempbit = g->get_wire_ids_as_share(tt); + temp = ifg; + ifg = c->PutORGate(temp, tempbit); + delete tempbit; + delete temp; + } + g = c->PutFPGate(one_gate, g, DIV, bitlen, 1, no_status); + for (j = l; j < nCols; j++) { + s = new boolshare(zero_gate->get_wires(), c); + for (k = l; k < nRows; k++) { + temp = c->PutFPGate(a[k][i], a[k][j], MUL, bitlen, 1, no_status); + s = c->PutFPGate( + s, temp, ADD, bitlen, 1, + no_status); // s is only used here so no mux gate okay + } + temp = c->PutFPGate(s, a[i][i], DIV, bitlen, 1, no_status); + f = c->PutFPGate(temp, g, MUL, bitlen, 1, + no_status); // f is only used here so no mux gate okay + for (k = i; k < nRows; k++) { + temp = c->PutFPGate(f, a[k][i], MUL, bitlen, 1, no_status); + temp = c->PutFPGate(a[k][j], temp, ADD, bitlen, 1, no_status); + a[k][j] = + c->PutMUXGate(temp, a[k][j], ifg); // do nothing if(g) is false + } + } + for (j = i; j < nRows; j++) { + temp = c->PutFPGate(a[j][i], g, MUL, bitlen, 1, no_status); + share* zcpy = new boolshare(zero_gate->get_wires(), c); + a[j][i] = c->PutMUXGate(temp, zcpy, ifg); // set to zero ifg is false + delete zcpy; + delete temp; + } + } + //} else - done in for loop above with mux + // for (j = i; j < nRows; j++) + // a[j][i] = 0.0; + a[i][i] = c->PutFPGate(a[i][i], one_gate, ADD, bitlen, 1, no_status); + printf("|"); + fflush(stdout); + } +} + +void RunSvdPart1Circuit(uint32_t** a, int nRows, int nCols, uint32_t* w, + uint32_t** v, uint32_t* anorm, uint32_t* rv1, + BooleanCircuit* c, ABYParty* p) { + + std::vector& sharings = p->GetSharings(); + BooleanCircuit* bc = + (BooleanCircuit*)sharings[S_BOOL]->GetCircuitBuildRoutine(); + + // Allocate space for shares + share* temp; + share*** s_a = new share**[nRows]; + for (int i = 0; i < nRows; i++) { + s_a[i] = new share*[nCols]; + } + share** s_w = new share*[nCols]; + share*** s_v = new share**[nCols]; + for (int i = 0; i < nCols; i++) { + s_v[i] = new share*[nCols]; + } + share* s_anorm; + share** s_rv1 = new share*[nCols]; + + // Prepare inputs + for (int i = 0; i < nRows; i++) { + for (int j = 0; j < nCols; j++) { + s_a[i][j] = bc->PutSharedINGate(&a[i][j], bitlen); + } + } + // if yao, convert bool inputs to yao + if (c->GetContext() == S_YAO) { + for (int i = 0; i < nRows; i++) { + for (int j = 0; j < nCols; j++) { + temp = s_a[i][j]; + s_a[i][j] = c->PutB2YGate(temp); + delete temp; + } + } + } + // prepare circuit + BuildSvdPart1Circuit(s_a, nRows, nCols, s_w, s_v, &s_anorm, s_rv1, c); + + // if yao was used, convert from yao back to boolean circuit + if (c->GetContext() == S_YAO) { + for (int t = 0; t < nRows; t++) { + for (int tt = 0; tt < nCols; tt++) { + temp = s_a[t][tt]; + s_a[t][tt] = bc->PutY2BGate(temp); + delete temp; + } + } + for (int t = 0; t < nCols; t++) { + temp = s_w[t]; + s_w[t] = bc->PutY2BGate(temp); + delete temp; + } + for (int t = 0; t < nCols; t++) { + for (int tt = 0; tt < nCols; tt++) { + temp = s_v[t][tt]; + s_v[t][tt] = bc->PutY2BGate(temp); + delete temp; + } + } + temp = s_anorm; + s_anorm = bc->PutY2BGate(temp); + delete temp; + for (int t = 0; t < nCols; t++) { + temp = s_rv1[t]; + s_rv1[t] = bc->PutY2BGate(temp); + delete temp; + } + } + + // prepare output + for (int i = 0; i < nRows; i++) { + for (int j = 0; j < nCols; j++) { + temp = s_a[i][j]; + s_a[i][j] = bc->PutSharedOUTGate(temp); + delete temp; + } + } + for (int i = 0; i < nCols; i++) { + temp = s_w[i]; + s_w[i] = bc->PutSharedOUTGate(temp); + delete temp; + } + for (int i = 0; i < nCols; i++) { + for (int j = 0; j < nCols; j++) { + temp = s_v[i][j]; + s_v[i][j] = bc->PutSharedOUTGate(temp); + delete temp; + } + } + + temp = s_anorm; + s_anorm = bc->PutSharedOUTGate(temp); + delete temp; + for (int i = 0; i < nCols; i++) { + temp = s_rv1[i]; + s_rv1[i] = bc->PutSharedOUTGate(temp); + delete temp; + } + + p->ExecCircuit(); + + // get shared output + for (int i = 0; i < nRows; i++) { + for (int j = 0; j < nCols; j++) { + a[i][j] = s_a[i][j]->get_clear_value(); + delete s_a[i][j]; + } + delete[] s_a[i]; + } + delete[] s_a; + for (int i = 0; i < nCols; i++) { + w[i] = s_w[i]->get_clear_value(); + delete s_w[i]; + } + delete[] s_w; + for (int i = 0; i < nCols; i++) { + for (int j = 0; j < nCols; j++) { + v[i][j] = s_v[i][j]->get_clear_value(); + delete s_v[i][j]; + } + delete[] s_v[i]; + } + delete[] s_v; + + *anorm = s_anorm->get_clear_value(); + delete s_anorm; + + for (int i = 0; i < nCols; i++) { + rv1[i] = s_rv1[i]->get_clear_value(); + delete s_rv1[i]; + } + collectTiming(); + collectCommunication(); + p->Reset(); +} + +share* BuildSvdCheckZeroCircuit(share* val, share* anorm, BooleanCircuit* c) { + // This evaluates: + // ((fabs(val) + anorm) == anorm) + BuildFabsCircuit(val, + c); // overwriting is okay because this is separate circuit + share* vpa = c->PutFPGate(val, anorm, ADD, bitlen, 1, no_status); + share* gt = + c->PutFPGate(vpa, anorm, CMP); // = val+anorm > anorm (val always > 0) + share* res = c->PutINVGate(gt); + delete vpa; + delete gt; + return res; +} + +uint32_t RunSvdCheckZeroCircuit(uint32_t* val, uint32_t* anorm, + BooleanCircuit* c, ABYParty* p) { + + std::vector& sharings = p->GetSharings(); + BooleanCircuit* bc = + (BooleanCircuit*)sharings[S_BOOL]->GetCircuitBuildRoutine(); + + // Allocate space for shares + share* s_val; + share* s_anorm; + // Prepare inputs + s_val = bc->PutSharedINGate(val, bitlen); + s_anorm = bc->PutSharedINGate(anorm, bitlen); + // if yao, convert bool inputs to yao + if (c->GetContext() == S_YAO) { + share* temp = s_val; + s_val = c->PutB2YGate(temp); + delete temp; + temp = s_anorm; + s_anorm = c->PutB2YGate(temp); + delete temp; + } + // prepare circuit + share* s_res = BuildSvdCheckZeroCircuit(s_val, s_anorm, c); + // prepare CLEARTEXT output + share* temp = s_res; + s_res = c->PutOUTGate(temp, ALL); + delete temp; + + p->ExecCircuit(); + + // get shared output + uint32_t res = s_res->get_clear_value(); + delete s_res; + delete s_val; + delete s_anorm; + + collectTiming(); + collectCommunication(); + p->Reset(); + return res; +} + +void BuildSvdFlag1Circuit(share** cc, share** f, share** s, share** rv1i, + BooleanCircuit* c) { + *f = c->PutFPGate(*s, *rv1i, MUL, bitlen, 1, no_status); + *rv1i = c->PutFPGate(*cc, *rv1i, MUL, bitlen, 1, no_status); +} +void RunSvdFlag1Circuit(uint32_t* cc, uint32_t* f, uint32_t* s, uint32_t* rv1i, + BooleanCircuit* c, ABYParty* p) { + + std::vector& sharings = p->GetSharings(); + BooleanCircuit* bc = + (BooleanCircuit*)sharings[S_BOOL]->GetCircuitBuildRoutine(); + + // Allocate space for shares + share* temp; + share* s_cc; + share* s_f; + share* s_s; + share* s_rv1i; + // Prepare inputs + s_cc = bc->PutSharedINGate(cc, bitlen); + s_f = bc->PutSharedINGate(f, bitlen); + s_s = bc->PutSharedINGate(s, bitlen); + s_rv1i = bc->PutSharedINGate(rv1i, bitlen); + // if yao, convert bool inputs to yao + if (c->GetContext() == S_YAO) { + temp = s_cc; + s_cc = c->PutB2YGate(temp); + delete temp; + temp = s_f; + s_f = c->PutB2YGate(temp); + delete temp; + temp = s_s; + s_s = c->PutB2YGate(temp); + delete temp; + temp = s_rv1i; + s_rv1i = c->PutB2YGate(temp); + delete temp; + } + // prepare circuit + BuildSvdFlag1Circuit(&s_cc, &s_f, &s_s, &s_rv1i, c); + // if yao was used, convert from yao back to boolean circuit + if (c->GetContext() == S_YAO) { + temp = s_f; + s_f = bc->PutY2BGate(temp); + delete temp; + temp = s_rv1i; + s_rv1i = bc->PutY2BGate(temp); + delete temp; + } + // prepare output + temp = s_f; + s_f = bc->PutSharedOUTGate(temp); + delete temp; + temp = s_rv1i; + s_rv1i = bc->PutSharedOUTGate(temp); + delete temp; + + p->ExecCircuit(); + + // get shared output + *f = s_f->get_clear_value(); + *rv1i = s_rv1i->get_clear_value(); + delete s_cc; + delete s_f; + delete s_s; + delete s_rv1i; + + collectTiming(); + collectCommunication(); + p->Reset(); +} + +void BuildSvdFlag2Circuit(share* aSTARi[], share* aSTARnm[], int nRows, + share** wi, share** cc, share** f, share** s, + BooleanCircuit* c) { + // non secret requires: i, nm + // secret requires: a[*][i], a[*][nm], w[i], cc, f + share *g, *h, *y, *z; + + float one = 1.0; + share* one_gate = c->PutCONSGate((uint32_t*)&one, bitlen); + + g = new boolshare((*wi)->get_wires(), c); + h = BuildPythagCircuit(*f, g, c, bitlen); + *wi = new boolshare(h->get_wires(), c); + h = c->PutFPGate(one_gate, h, DIV, bitlen, 1, no_status); + *cc = c->PutFPGate(g, h, MUL, bitlen, 1, no_status); + share* temp = c->PutFPGate(*f, h, MUL, bitlen, 1, no_status); + BuildNegativeCircuit(temp, c); + *s = temp; + for (int j = 0; j < nRows; j++) { + y = aSTARnm[j]; // y = a[j][nm]; + z = aSTARi[j]; // z = a[j][i]; + share* tmp1 = c->PutFPGate(y, *cc, MUL, bitlen, 1, no_status); + share* tmp2 = c->PutFPGate(z, *s, MUL, bitlen, 1, no_status); + aSTARnm[j] = c->PutFPGate(tmp1, tmp2, ADD, bitlen, 1, no_status); + // a[j][nm] + delete tmp1; + delete tmp2; + + tmp1 = c->PutFPGate(z, *cc, MUL, bitlen, 1, no_status); + tmp2 = c->PutFPGate(y, *s, MUL, bitlen, 1, no_status); + aSTARi[j] = c->PutFPGate(tmp1, tmp2, SUB, bitlen, 1, no_status); + // a[j][i] + delete tmp1; + delete tmp2; + } +} +void RunSvdFlag2Circuit(uint32_t* aSTARi, uint32_t* aSTARnm, int nRows, + uint32_t* wi, uint32_t* cc, uint32_t* f, uint32_t* s, + BooleanCircuit* c, ABYParty* p) { + std::vector& sharings = p->GetSharings(); + BooleanCircuit* bc = + (BooleanCircuit*)sharings[S_BOOL]->GetCircuitBuildRoutine(); + + // Allocate space for shares + share* temp; + share** s_aSTARi = new share*[nRows]; + share** s_aSTARnm = new share*[nRows]; + share* s_wi; + share* s_cc; + share* s_f; + share* s_s; + // Prepare inputs + for (int i = 0; i < nRows; i++) { + s_aSTARi[i] = bc->PutSharedINGate(&aSTARi[i], bitlen); + s_aSTARnm[i] = bc->PutSharedINGate(&aSTARnm[i], bitlen); + } + s_wi = bc->PutSharedINGate(wi, bitlen); + s_cc = bc->PutSharedINGate(cc, bitlen); + s_f = bc->PutSharedINGate(f, bitlen); + s_s = bc->PutSharedINGate(s, bitlen); + // if yao, convert bool inputs to yao + if (c->GetContext() == S_YAO) { + for (int i = 0; i < nRows; i++) { + temp = s_aSTARi[i]; + s_aSTARi[i] = c->PutB2YGate(temp); + delete temp; + temp = s_aSTARnm[i]; + s_aSTARnm[i] = c->PutB2YGate(temp); + delete temp; + } + temp = s_wi; + s_wi = c->PutB2YGate(temp); + delete temp; + temp = s_cc; + s_cc = c->PutB2YGate(temp); + delete temp; + temp = s_f; + s_f = c->PutB2YGate(temp); + delete temp; + temp = s_s; + s_s = c->PutB2YGate(temp); + delete temp; + } + // prepare circuit + BuildSvdFlag2Circuit(s_aSTARi, s_aSTARnm, nRows, &s_wi, &s_cc, &s_f, &s_s, c); + // if yao was used, convert from yao back to boolean circuit + if (c->GetContext() == S_YAO) { + for (int i = 0; i < nRows; i++) { + temp = s_aSTARi[i]; + s_aSTARi[i] = bc->PutY2BGate(temp); + delete temp; + temp = s_aSTARnm[i]; + s_aSTARnm[i] = bc->PutY2BGate(temp); + delete temp; + } + temp = s_wi; + s_wi = bc->PutY2BGate(temp); + delete temp; + temp = s_cc; + s_cc = bc->PutY2BGate(temp); + delete temp; + temp = s_f; + s_f = bc->PutY2BGate(temp); + delete temp; + temp = s_s; + s_s = bc->PutY2BGate(temp); + delete temp; + } + // prepare output (everything except f which is not an output) + for (int star = 0; star < nRows; star++) { + temp = s_aSTARi[star]; + s_aSTARi[star] = bc->PutSharedOUTGate(temp); + delete temp; + temp = s_aSTARnm[star]; + s_aSTARnm[star] = bc->PutSharedOUTGate(temp); + delete temp; + } + temp = s_wi; + s_wi = bc->PutSharedOUTGate(temp); + delete temp; + temp = s_cc; + s_cc = bc->PutSharedOUTGate(temp); + delete temp; + temp = s_s; + s_s = bc->PutSharedOUTGate(temp); + delete temp; + + p->ExecCircuit(); + + // get shared output + for (int star = 0; star < nRows; star++) { + aSTARi[star] = s_aSTARi[star]->get_clear_value(); + delete s_aSTARi[star]; + aSTARnm[star] = s_aSTARnm[star]->get_clear_value(); + delete s_aSTARnm[star]; + } + delete[] s_aSTARi; + delete[] s_aSTARnm; + *wi = s_wi->get_clear_value(); + delete s_wi; + *cc = s_cc->get_clear_value(); + delete s_cc; + delete s_f; + *s = s_s->get_clear_value(); + delete s_s; + + collectTiming(); + collectCommunication(); + p->Reset(); +} + +void BuildSvdZCircuit(int nCols, share** z, share** wk, share* vSTARk[], + BooleanCircuit* c) { + // secret requires: z, w[k], v[*][k] + + float zero = 0; + share* zero_gate = c->PutCONSGate((uint32_t*)&zero, bitlen); + + // if (z < 0.0) + share* isneg = c->PutFPGate(zero_gate, *z, CMP); + share* temp = new boolshare((*z)->get_wires(), c); + BuildNegativeCircuit(temp, c); + *wk = c->PutMUXGate(temp, *wk, isneg); + for (int j = 0; j < nCols; j++) { + temp = new boolshare(vSTARk[j]->get_wires(), c); + BuildNegativeCircuit(temp, c); + vSTARk[j] = c->PutMUXGate(temp, vSTARk[j], isneg); // v[j][k] + } +} +void RunSvdZCircuit(int nCols, uint32_t* z, uint32_t* wk, uint32_t* vSTARk, + BooleanCircuit* c, ABYParty* p) { + + std::vector& sharings = p->GetSharings(); + BooleanCircuit* bc = + (BooleanCircuit*)sharings[S_BOOL]->GetCircuitBuildRoutine(); + + // Allocate space for shares + share* temp; + share* s_z; + share* s_wk; + share** s_vSTARk = new share*[nCols]; + // Prepare inputs + s_z = bc->PutSharedINGate(z, bitlen); + s_wk = bc->PutSharedINGate(wk, bitlen); + for (int i = 0; i < nCols; i++) { + s_vSTARk[i] = bc->PutSharedINGate(&vSTARk[i], bitlen); + } + // if yao, convert bool inputs to yao + if (c->GetContext() == S_YAO) { + temp = s_z; + s_z = c->PutB2YGate(temp); + delete temp; + temp = s_wk; + s_wk = c->PutB2YGate(temp); + delete temp; + for (int i = 0; i < nCols; i++) { + temp = s_vSTARk[i]; + s_vSTARk[i] = c->PutB2YGate(temp); + delete temp; + } + } + // prepare circuit + BuildSvdZCircuit(nCols, &s_z, &s_wk, s_vSTARk, c); + // if yao was used, convert from yao back to boolean circuit + if (c->GetContext() == S_YAO) { + temp = s_wk; + s_wk = bc->PutY2BGate(temp); + delete temp; + for (int i = 0; i < nCols; i++) { + temp = s_vSTARk[i]; + s_vSTARk[i] = bc->PutY2BGate(temp); + delete temp; + } + } + // prepare output (everything except z which is not an output) + for (int star = 0; star < nCols; star++) { + temp = s_vSTARk[star]; + s_vSTARk[star] = bc->PutSharedOUTGate(temp); + delete temp; + } + temp = s_wk; + s_wk = bc->PutSharedOUTGate(temp); + delete temp; + + p->ExecCircuit(); + + // get shared output + delete s_z; + *wk = s_wk->get_clear_value(); + delete s_wk; + for (int star = 0; star < nCols; star++) { + vSTARk[star] = s_vSTARk[star]->get_clear_value(); + delete s_vSTARk[star]; + } + delete[] s_vSTARk; + + collectTiming(); + collectCommunication(); + p->Reset(); +} + +void BuildSvdPart2Circuit(share** a[], int nRows, int nCols, share* w[], + share** v[], share* rv1[], share** z, int k, int l, + int nm, BooleanCircuit* c) { + // non secret requires: k, l, nm + // secret requires: a, w, v, z, rv1 (all elements) + + float zero = 0; + share* zero_gate = c->PutCONSGate((uint32_t*)&zero, bitlen); + float one = 1; + share* one_gate = c->PutCONSGate((uint32_t*)&one, bitlen); + float two = 2; + share* two_gate = c->PutCONSGate((uint32_t*)&two, bitlen); + + int jj; // local plaintext (not used anywhere else) + share *cc, *s, *x, *y, *g, *h, *f; + + x = new boolshare(w[l]->get_wires(), c); + y = new boolshare(w[nm]->get_wires(), c); + g = new boolshare(rv1[nm]->get_wires(), c); + h = new boolshare(rv1[k]->get_wires(), c); + { // f = ((y - z) * (y + z) + (g - h) * (g + h)) / (2.0 * h * y); + share* ymz = c->PutFPGate(y, *z, SUB, bitlen, 1, no_status); + share* ypz = c->PutFPGate(y, *z, ADD, bitlen, 1, no_status); + share* temp1 = c->PutFPGate(ymz, ypz, MUL, bitlen, 1, no_status); + share* gmh = c->PutFPGate(g, h, SUB, bitlen, 1, no_status); + share* gph = c->PutFPGate(g, h, ADD, bitlen, 1, no_status); + share* temp2 = c->PutFPGate(gmh, gph, MUL, bitlen, 1, no_status); + share* num = c->PutFPGate(temp1, temp2, ADD, bitlen, 1, no_status); + share* den = c->PutFPGate(two_gate, h, MUL, bitlen, 1, no_status); + den = c->PutFPGate(den, y, MUL, bitlen, 1, no_status); + f = c->PutFPGate(num, den, DIV, bitlen, 1, no_status); + } + g = BuildPythagCircuit(f, one_gate, c, bitlen); + { // f = ((x - z) * (x + z) + h * ((y / (f + SIGN(g,f)))- h)) / x; + share* xmz = c->PutFPGate(x, *z, SUB, bitlen, 1, no_status); + share* xpz = c->PutFPGate(x, *z, ADD, bitlen, 1, no_status); + share* term1 = c->PutFPGate(xmz, xpz, MUL, bitlen, 1, no_status); + share* sgf = new boolshare(g->get_wires(), c); + BuildSignCircuit(sgf, f, c); + share* fpsgf = c->PutFPGate(f, sgf, ADD, bitlen, 1, no_status); + share* ydfpsgf = c->PutFPGate(y, fpsgf, DIV, bitlen, 1, no_status); + share* uggg = c->PutFPGate(ydfpsgf, h, SUB, bitlen, 1, no_status); + share* term2 = c->PutFPGate(h, uggg, MUL, bitlen, 1, no_status); + share* num2 = c->PutFPGate(term1, term2, ADD, bitlen, 1, no_status); + f = c->PutFPGate(num2, x, DIV, bitlen, 1, no_status); + } + cc = new boolshare(one_gate->get_wires(), c); + s = new boolshare(one_gate->get_wires(), c); + + for (int j = l; j <= nm; j++) { + int i = j + 1; + g = new boolshare(rv1[i]->get_wires(), c); + y = new boolshare(w[i]->get_wires(), c); + h = c->PutFPGate(s, g, MUL, bitlen, 1, no_status); + g = c->PutFPGate(cc, g, MUL, bitlen, 1, no_status); + *z = BuildPythagCircuit(f, h, c, bitlen); + rv1[j] = new boolshare((*z)->get_wires(), c); + cc = c->PutFPGate(f, *z, DIV, bitlen, 1, no_status); + s = c->PutFPGate(h, *z, DIV, bitlen, 1, no_status); + { // f = x * c + g * s; + share* term1 = c->PutFPGate(x, cc, MUL, bitlen, 1, no_status); + share* term2 = c->PutFPGate(g, s, MUL, bitlen, 1, no_status); + f = c->PutFPGate(term1, term2, ADD, bitlen, 1, no_status); + } + { // g = g * c - x * s; + share* term1 = c->PutFPGate(g, cc, MUL, bitlen, 1, no_status); + share* term2 = c->PutFPGate(x, s, MUL, bitlen, 1, no_status); + g = c->PutFPGate(term1, term2, SUB, bitlen, 1, no_status); + } + h = c->PutFPGate(y, s, MUL, bitlen, 1, no_status); + y = c->PutFPGate(y, cc, MUL, bitlen, 1, no_status); + for (jj = 0; jj < nCols; jj++) { + x = new boolshare(v[jj][j]->get_wires(), c); + *z = new boolshare(v[jj][i]->get_wires(), c); + { // v[jj][j] = x * c + z * s; + share* term1 = c->PutFPGate(x, cc, MUL, bitlen, 1, no_status); + share* term2 = c->PutFPGate(*z, s, MUL, bitlen, 1, no_status); + v[jj][j] = c->PutFPGate(term1, term2, ADD, bitlen, 1, no_status); + } + { // v[jj][i] = z * c - x * s; + share* term1 = c->PutFPGate(*z, cc, MUL, bitlen, 1, no_status); + share* term2 = c->PutFPGate(x, s, MUL, bitlen, 1, no_status); + v[jj][i] = c->PutFPGate(term1, term2, SUB, bitlen, 1, no_status); + } + } + *z = BuildPythagCircuit(f, h, c, bitlen); + w[j] = new boolshare((*z)->get_wires(), c); + { // if (z) { + share* temp; + share* ifz = (*z)->get_wire_ids_as_share(0); + for (uint32_t tt = 1; tt < bitlen - 1; + tt++) { // -1 -> do not include sign bit + share* tempbit = (*z)->get_wire_ids_as_share(tt); + temp = ifz; + ifz = c->PutORGate(temp, tempbit); + delete tempbit; + delete temp; + } + + temp = c->PutFPGate(one_gate, *z, DIV, bitlen, 1, no_status); + *z = c->PutMUXGate(temp, *z, ifz); + delete temp; + + temp = c->PutFPGate(f, *z, MUL, bitlen, 1, no_status); + cc = c->PutMUXGate(temp, cc, ifz); + delete temp; + + temp = c->PutFPGate(h, *z, MUL, bitlen, 1, no_status); + s = c->PutMUXGate(temp, s, ifz); + delete temp; + } + { // f = c * g + s * y; + share* term1 = c->PutFPGate(cc, g, MUL, bitlen, 1, no_status); + share* term2 = c->PutFPGate(s, y, MUL, bitlen, 1, no_status); + f = c->PutFPGate(term1, term2, ADD, bitlen, 1, no_status); + } + { // x = c * y - s * g; + share* term1 = c->PutFPGate(cc, y, MUL, bitlen, 1, no_status); + share* term2 = c->PutFPGate(s, g, MUL, bitlen, 1, no_status); + x = c->PutFPGate(term1, term2, SUB, bitlen, 1, no_status); + } + for (jj = 0; jj < nRows; jj++) { + y = new boolshare(a[jj][j]->get_wires(), c); + *z = new boolshare(a[jj][i]->get_wires(), c); + { // a[jj][j] = y * c + z * s; + share* term1 = c->PutFPGate(y, cc, MUL, bitlen, 1, no_status); + share* term2 = c->PutFPGate(*z, s, MUL, bitlen, 1, no_status); + a[jj][j] = c->PutFPGate(term1, term2, ADD, bitlen, 1, no_status); + } + { // a[jj][i] = z * c - y * s; + share* term1 = c->PutFPGate(*z, cc, MUL, bitlen, 1, no_status); + share* term2 = c->PutFPGate(y, s, MUL, bitlen, 1, no_status); + a[jj][i] = c->PutFPGate(term1, term2, SUB, bitlen, 1, no_status); + } + } + } + rv1[l] = new boolshare(zero_gate->get_wires(), c); + rv1[k] = new boolshare(f->get_wires(), c); + w[k] = new boolshare(x->get_wires(), c); +} +void RunSvdPart2Circuit(uint32_t** a, int nRows, int nCols, uint32_t* w, + uint32_t** v, uint32_t* rv1, uint32_t* z, int k, int l, + int nm, BooleanCircuit* c, ABYParty* p) { + + std::vector& sharings = p->GetSharings(); + BooleanCircuit* bc = + (BooleanCircuit*)sharings[S_BOOL]->GetCircuitBuildRoutine(); + + // Allocate space for shares + share* temp; + share*** s_a = new share**[nRows]; + for (int t = 0; t < nRows; t++) { + s_a[t] = new share*[nCols]; + } + share** s_w = new share*[nCols]; + share*** s_v = new share**[nCols]; + for (int t = 0; t < nCols; t++) { + s_v[t] = new share*[nCols]; + } + share** s_rv1 = new share*[nCols]; + share* s_z; + + // Prepare inputs + for (int t = 0; t < nRows; t++) { + for (int tt = 0; tt < nCols; tt++) { + s_a[t][tt] = bc->PutSharedINGate(&a[t][tt], bitlen); + } + } + for (int t = 0; t < nCols; t++) { + s_w[t] = bc->PutSharedINGate(&w[t], bitlen); + } + for (int t = 0; t < nCols; t++) { + for (int tt = 0; tt < nCols; tt++) { + s_v[t][tt] = bc->PutSharedINGate(&v[t][tt], bitlen); + } + } + for (int t = 0; t < nCols; t++) { + s_rv1[t] = bc->PutSharedINGate(&rv1[t], bitlen); + } + s_z = bc->PutSharedINGate(z, bitlen); + + // if yao, convert bool inputs to yao + if (c->GetContext() == S_YAO) { + for (int t = 0; t < nRows; t++) { + for (int tt = 0; tt < nCols; tt++) { + share* temp = s_a[t][tt]; + s_a[t][tt] = c->PutB2YGate(temp); + delete temp; + } + } + for (int t = 0; t < nCols; t++) { + share* temp = s_w[t]; + s_w[t] = c->PutB2YGate(temp); + delete temp; + } + for (int t = 0; t < nCols; t++) { + for (int tt = 0; tt < nCols; tt++) { + share* temp = s_v[t][tt]; + s_v[t][tt] = c->PutB2YGate(temp); + delete temp; + } + } + for (int t = 0; t < nCols; t++) { + temp = s_rv1[t]; + s_rv1[t] = c->PutB2YGate(temp); + delete temp; + } + temp = s_z; + s_z = c->PutB2YGate(temp); + delete temp; + } + + // prepare circuit + BuildSvdPart2Circuit(s_a, nRows, nCols, s_w, s_v, s_rv1, &s_z, k, l, nm, c); + + // if yao was used, convert from yao back to boolean circuit + if (c->GetContext() == S_YAO) { + for (int t = 0; t < nRows; t++) { + for (int tt = 0; tt < nCols; tt++) { + share* temp = s_a[t][tt]; + s_a[t][tt] = bc->PutY2BGate(temp); + delete temp; + } + } + for (int t = 0; t < nCols; t++) { + share* temp = s_w[t]; + s_w[t] = bc->PutY2BGate(temp); + delete temp; + } + for (int t = 0; t < nCols; t++) { + for (int tt = 0; tt < nCols; tt++) { + share* temp = s_v[t][tt]; + s_v[t][tt] = bc->PutY2BGate(temp); + delete temp; + } + } + for (int t = 0; t < nCols; t++) { + temp = s_rv1[t]; + s_rv1[t] = bc->PutY2BGate(temp); + delete temp; + } + temp = s_z; + s_z = bc->PutY2BGate(temp); + delete temp; + } + // prepare output (everything except z which is not an output) + for (int t = 0; t < nRows; t++) { + for (int tt = 0; tt < nCols; tt++) { + share* temp = s_a[t][tt]; + s_a[t][tt] = bc->PutSharedOUTGate(temp); + delete temp; + } + } + for (int t = 0; t < nCols; t++) { + share* temp = s_w[t]; + s_w[t] = bc->PutSharedOUTGate(temp); + delete temp; + } + for (int t = 0; t < nCols; t++) { + for (int tt = 0; tt < nCols; tt++) { + share* temp = s_v[t][tt]; + s_v[t][tt] = bc->PutSharedOUTGate(temp); + delete temp; + } + } + for (int t = 0; t < nCols; t++) { + temp = s_rv1[t]; + s_rv1[t] = bc->PutSharedOUTGate(temp); + delete temp; + } + temp = s_z; + s_z = bc->PutSharedOUTGate(temp); + delete temp; + + p->ExecCircuit(); + + // get shared output + for (int i = 0; i < nRows; i++) { + for (int j = 0; j < nCols; j++) { + a[i][j] = s_a[i][j]->get_clear_value(); + delete s_a[i][j]; + } + delete[] s_a[i]; + } + delete[] s_a; + for (int i = 0; i < nCols; i++) { + w[i] = s_w[i]->get_clear_value(); + delete s_w[i]; + } + delete[] s_w; + for (int i = 0; i < nCols; i++) { + for (int j = 0; j < nCols; j++) { + v[i][j] = s_v[i][j]->get_clear_value(); + delete s_v[i][j]; + } + delete[] s_v[i]; + } + delete[] s_v; + for (int i = 0; i < nCols; i++) { + rv1[i] = s_rv1[i]->get_clear_value(); + delete s_rv1[i]; + } + delete[] s_rv1; + *z = s_z->get_clear_value(); + delete s_z; + + collectTiming(); + collectCommunication(); + p->Reset(); +} + +// uint32_t arguments must be from PutSharedOUTGate() +// then calling get_clear_value() and circuit->Reset(); +// This function builds/executes multiple circuits. +void RunSvdLoopLeak(uint32_t** a, int nRows, int nCols, uint32_t* w, + uint32_t** v, BooleanCircuit* c, ABYParty* p, e_role role) { + + // Following values are shared between subcircuits + int flag, i, its, /*j, jj,*/ k, l, nm; + uint32_t anorm, cc, f, /*g, h,*/ s, /*scale, x, y,*/ z; + uint32_t* rv1 = new uint32_t[nCols]; + + RunSvdPart1Circuit(a, nRows, nCols, w, v, &anorm, rv1, c, p); + + for (k = nCols - 1; k >= 0; k--) { + for (its = 0; its < 30; its++) { + cout << k; + flag = 1; + for (l = k; l >= 0; l--) { + nm = l - 1; + if (RunSvdCheckZeroCircuit(&rv1[l], &anorm, c, p)) { + flag = 0; + break; + } + if (RunSvdCheckZeroCircuit(&w[nm], &anorm, c, p)) { + break; + } + } + + if (flag) { + // setting both shares to zero = constant 0 + cc = 0; + // setting party0 to 0 and party1 to 1 = constant 1 + float frole = (float)role; + uint32_t* frolep = (uint32_t*)&frole; + s = *frolep; + + for (i = l; i <= k; i++) { + RunSvdFlag1Circuit(&cc, &f, &s, &rv1[i], c, p); + + if (RunSvdCheckZeroCircuit(&f, &anorm, c, p)) { + break; + } + + // Slice out column i of a (a[*][i]) and column nm (a[*][nn]) + uint32_t* aSTARi = new uint32_t[nRows]; + uint32_t* aSTARnm = new uint32_t[nRows]; + for (int star = 0; star < nRows; star++) { + aSTARi[star] = a[star][i]; + aSTARnm[star] = a[star][nm]; + } + RunSvdFlag2Circuit(aSTARi, aSTARnm, nRows, &w[i], &cc, &f, &s, c, p); + // Copy column back into matrix + for (int star = 0; star < nRows; star++) { + a[star][i] = aSTARi[star]; + a[star][nm] = aSTARnm[star]; + } + delete[] aSTARi; + delete[] aSTARnm; + } + } + z = w[k]; + if (l == k) { + // Slice out column k of v (v[*][k]) + uint32_t* vSTARk = new uint32_t[nCols]; + for (int star = 0; star < nCols; star++) { + vSTARk[star] = v[star][k]; + } + RunSvdZCircuit(nCols, &z, &w[k], vSTARk, c, p); + // Copy column back into matrix + for (int star = 0; star < nCols; star++) { + v[star][k] = vSTARk[star]; + } + delete[] vSTARk; + break; + } + if (its == 29) + printf("no convergence in 30 svdcmp iterations\n"); + nm = k - 1; + + RunSvdPart2Circuit(a, nRows, nCols, w, v, rv1, &z, k, l, nm, c, p); + } + printf("/"); + fflush(stdout); + } +} + +// Wrapper function around RunSvdLoopLeak which +// takes shares instead of secret shares. +// Creates dummy circuits to build secret shares. +void BuildAndRunSvdLoopLeak(share*** s_a, int nRows, int nCols, share** s_w, + share*** s_v, BooleanCircuit* c, ABYParty* party, + e_role role, std::function toRawShares, + std::function toShareObjects) { + + std::vector& sharings = party->GetSharings(); + BooleanCircuit* bc = + (BooleanCircuit*)sharings[S_BOOL]->GetCircuitBuildRoutine(); + + share* temp; + + // stores secret-shared output + uint32_t** a = new uint32_t*[nRows]; + for (int i = 0; i < nRows; i++) { + a[i] = new uint32_t[nCols]; + } + uint32_t* w = new uint32_t[nCols]; + uint32_t** v = new uint32_t*[nCols]; + for (int i = 0; i < nCols; i++) { + v[i] = new uint32_t[nCols]; + } + + // if yao is prefered circuit and yao shares passed in, must + // convert to bool to get raw shares. + // Tests must pass in bool shares even if yao is preferred. + if (c->GetContext() == S_YAO && s_a[0][0]->get_share_type() == S_YAO) { + for (int i = 0; i < nRows; i++) { + for (int j = 0; j < nCols; j++) { + temp = s_a[i][j]; + s_a[i][j] = bc->PutY2BGate(temp); + delete temp; + } + } + } + // build shared output objects + for (int i = 0; i < nRows; i++) { + for (int j = 0; j < nCols; j++) { + temp = s_a[i][j]; + s_a[i][j] = bc->PutSharedOUTGate(temp); + delete temp; + } + } + + // run the dummy circuit + CLOCK(ShareSvdInputs); + TIC(ShareSvdInputs); + party->ExecCircuit(); + TOC(ShareSvdInputs); + // Get raw output shares + for (int i = 0; i < nRows; i++) { + for (int j = 0; j < nCols; j++) { + a[i][j] = s_a[i][j]->get_clear_value(); + delete s_a[i][j]; + } + } + + toRawShares(); // allow caller to store shares across sub-circuit executions + + collectTiming(); + collectCommunication(); + party->Reset(); + // a[][] now contains secret shared values of plaintext from each party + + // Run the svd on the secret shared data + CLOCK(SVD); + TIC(SVD); + RunSvdLoopLeak(a, nRows, nCols, w, v, (BooleanCircuit*)c, party, role); + TOC(SVD); + + //// next run another dummy circuit to get the cleartext output from shared + /// output + // shared inputs to circuit + for (int i = 0; i < nRows; i++) { + for (int j = 0; j < nCols; j++) { + s_a[i][j] = bc->PutSharedINGate(&a[i][j], bitlen); + } + } + for (int i = 0; i < nCols; i++) { + s_w[i] = bc->PutSharedINGate(&w[i], bitlen); + } + for (int i = 0; i < nCols; i++) { + for (int j = 0; j < nCols; j++) { + s_v[i][j] = bc->PutSharedINGate(&v[i][j], bitlen); + } + } + // if yao, convert bool back to yao for future executions + if (c->GetContext() == S_YAO) { + for (int i = 0; i < nRows; i++) { + for (int j = 0; j < nCols; j++) { + temp = s_a[i][j]; + s_a[i][j] = c->PutB2YGate(temp); + delete temp; + } + } + for (int i = 0; i < nCols; i++) { + temp = s_w[i]; + s_w[i] = c->PutB2YGate(temp); + delete temp; + } + for (int i = 0; i < nCols; i++) { + for (int j = 0; j < nCols; j++) { + temp = s_v[i][j]; + s_v[i][j] = c->PutB2YGate(temp); + delete temp; + } + } + } + + toShareObjects(); // allow caller to recover shares across sub-circuit + // executions +} + +// Build SVD circuit. Since it does not need +// intermediate values, it does not use subcircuits +// thus only builds the circuit but does not execute. +void BuildSvdDO(share*** a, int nRows, int nCols, share** w, share*** v, + BooleanCircuit* c, ABYParty* party, e_role role) { + + // Following values are shared between subcircuits + int /*flag,*/ i, its, j, jj, k, l, nm; + share *anorm = nullptr, *cc = nullptr, *f = nullptr, *g = nullptr, + *h = nullptr, *s = nullptr, *scale = nullptr, *x = nullptr, + *y = nullptr, *z = nullptr; + share** rv1 = new share*[nCols]; + + float zero = 0; + share* zero_gate = c->PutCONSGate((uint32_t*)&zero, bitlen); + int int_zero = 0; + share* int_zero_gate = c->PutCONSGate((uint32_t*)&int_zero, bitlen); + float one = 1; + share* one_gate = c->PutCONSGate((uint32_t*)&one, bitlen); + int int_one = 1; + share* int_one_gate = c->PutCONSGate((uint32_t*)&int_one, bitlen); + float two = 2; + share* two_gate = c->PutCONSGate((uint32_t*)&two, bitlen); + uint8_t true_val = 1; + share* true_gate = c->PutCONSGate((uint8_t*)&true_val, 1); + uint8_t false_val = 0; + share* false_gate = c->PutCONSGate((uint8_t*)&false_val, 1); + share* temp; + + x = new boolshare(zero_gate->get_wires(), c); // x must be initialized + + BuildSvdPart1Circuit(a, nRows, nCols, w, v, &anorm, rv1, c); + + for (k = nCols - 1; k >= 0; k--) { + share* converged = new boolshare(false_gate->get_wires(), c); + for (its = 0; its < 30; its++) { + cout << k << flush; + share* flag = new boolshare(true_gate->get_wires(), c); + share* l_found = new boolshare(false_gate->get_wires(), c); + share* priv_l = new boolshare(int_zero_gate->get_wires(), c); + + for (l = k; l >= 0; l--) { + nm = l - 1; + + share* this_l = c->PutCONSGate((uint32_t*)&l, bitlen); + + share* rv1lcopy = new boolshare(rv1[l]->get_wires(), + c); // check zero overwrites with Fabs + share* rv1isanorm = BuildSvdCheckZeroCircuit(rv1lcopy, anorm, c); + delete rv1lcopy; + // flag = flag & !(rv1isanorm & !l_found); + share* nl_found = c->PutINVGate(l_found); + share* r_a_nl_found = c->PutANDGate(rv1isanorm, nl_found); + share* nr_a_nl_found = c->PutINVGate(r_a_nl_found); + temp = flag; + flag = c->PutANDGate(flag, nr_a_nl_found); + delete temp; + temp = priv_l; + priv_l = c->PutMUXGate(this_l, priv_l, r_a_nl_found); + delete temp; + temp = l_found; + l_found = c->PutORGate(l_found, rv1isanorm); + delete temp; + delete nr_a_nl_found; + delete r_a_nl_found; + delete nl_found; + delete rv1isanorm; + + if (nm >= 0) { + share* wnmcopy = new boolshare(w[nm]->get_wires(), + c); // check zero overwrites with Fabs + share* wnmisanorm = BuildSvdCheckZeroCircuit(wnmcopy, anorm, c); + delete wnmcopy; + share* nl_found = c->PutINVGate(l_found); + share* wnm_a_nl_found = c->PutANDGate(wnmisanorm, nl_found); + temp = priv_l; + priv_l = c->PutMUXGate(this_l, priv_l, wnm_a_nl_found); + delete temp; + temp = l_found; + l_found = c->PutORGate(l_found, wnmisanorm); + delete temp; + delete wnm_a_nl_found; + delete nl_found; + delete wnmisanorm; + } + } + + share* priv_nm = + c->PutSUBGate(priv_l, int_one_gate); // integer subtraction + { // if (flag) + cc = new boolshare(zero_gate->get_wires(), c); + s = new boolshare(one_gate->get_wires(), c); + for (i = 1; i <= k; i++) { + share* priv_i = c->PutCONSGate((uint32_t*)&i, bitlen); + temp = c->PutGTGate(priv_l, priv_i); + share* igel = c->PutINVGate(temp); + delete temp; + delete priv_i; + temp = c->PutANDGate(flag, igel); + share* nconverged = c->PutINVGate(converged); + share* modify = c->PutANDGate(temp, nconverged); + delete temp; + delete nconverged; + + if (f) + delete f; + f = c->PutFPGate(s, rv1[i], MUL, bitlen, 1, no_status); + temp = rv1[i]; + share* tmp2 = c->PutFPGate(cc, rv1[i], MUL, bitlen, 1, no_status); + rv1[i] = c->PutMUXGate(tmp2, rv1[i], modify); + delete temp; + delete tmp2; + + share* fcopy = new boolshare(f->get_wires(), + c); // check zero overwrites with Fabs + share* fisanorm = BuildSvdCheckZeroCircuit(fcopy, anorm, c); + delete fcopy; + share* nfisanorm = c->PutINVGate(fisanorm); + temp = modify; + modify = c->PutANDGate(modify, nfisanorm); + delete temp; + delete nfisanorm; + delete fisanorm; + + if (g) + delete g; + g = new boolshare(w[i]->get_wires(), c); + if (h) + delete h; + h = BuildPythagCircuit(f, g, c, bitlen); + temp = w[i]; + w[i] = c->PutMUXGate(h, w[i], modify); + delete temp; + share* oldh = h; + h = c->PutFPGate(one_gate, h, DIV, bitlen, 1, no_status); + delete oldh; + temp = c->PutFPGate(g, h, MUL, bitlen, 1, no_status); + share* oldcc = cc; + cc = c->PutMUXGate(temp, cc, modify); + delete temp; + delete oldcc; + temp = c->PutFPGate(f, h, MUL, bitlen, 1, no_status); + BuildNegativeCircuit(temp, c); + share* olds = s; + s = c->PutMUXGate(temp, s, modify); + delete temp; + delete olds; + for (int j = 0; j < nRows; j++) { + for (int nm_finder = 0; nm_finder < k; nm_finder++) { + share* priv_nm_finder = + c->PutCONSGate((uint32_t*)&nm_finder, bitlen); + share* found_nm = c->PutEQGate(priv_nm, priv_nm_finder); + delete priv_nm_finder; + share* mafnm = c->PutANDGate(modify, found_nm); + delete found_nm; + + y = new boolshare(a[j][nm_finder]->get_wires(), c); + z = new boolshare(a[j][i]->get_wires(), c); + share* tmp1 = c->PutFPGate(y, cc, MUL, bitlen, 1, no_status); + share* tmp2 = c->PutFPGate(z, s, MUL, bitlen, 1, no_status); + share* tmp3 = c->PutFPGate(tmp1, tmp2, ADD, bitlen, 1, no_status); + temp = a[j][nm_finder]; + a[j][nm_finder] = c->PutMUXGate(tmp3, a[j][nm_finder], mafnm); + delete temp; + delete tmp1; + delete tmp2; + delete tmp3; + + tmp1 = c->PutFPGate(z, cc, MUL, bitlen, 1, no_status); + tmp2 = c->PutFPGate(y, s, MUL, bitlen, 1, + no_status); // y is broken? + tmp3 = c->PutFPGate(tmp1, tmp2, SUB, bitlen, 1, no_status); + temp = a[j][i]; + a[j][i] = c->PutMUXGate(tmp3, a[j][i], mafnm); + delete temp; + delete tmp1; + delete tmp2; + delete tmp3; + + delete y; + delete z; + delete mafnm; + } + } + } + } + z = w[k]; + share* tempk = c->PutCONSGate((uint32_t*)&k, 32); + share* newconverged = c->PutEQGate(priv_l, tempk); + delete tempk; + { // if (l == k) + { // if (z < 0.0) singular value is made nonnegative + share* isneg = c->PutFPGate(zero_gate, z, CMP); + share* modify = c->PutANDGate(isneg, newconverged); + share* nconverged = c->PutINVGate(converged); + share* oldmodify = modify; + modify = c->PutANDGate(modify, nconverged); + delete oldmodify; + + share* temp = new boolshare(z->get_wires(), c); + BuildNegativeCircuit(temp, c); + w[k] = c->PutMUXGate(temp, w[k], modify); + for (int j = 0; j < nCols; j++) { + share* negv = new boolshare(v[j][k]->get_wires(), c); + BuildNegativeCircuit(negv, c); + share* oldvjk = v[j][k]; + v[j][k] = c->PutMUXGate(negv, v[j][k], modify); + delete negv; + delete oldvjk; + } + } + } + share* oldconverged = converged; + converged = c->PutORGate(converged, newconverged); + delete oldconverged; + if (k == 0) + break; // if k == 0 we are done, code below segfaults from nm + + // shift from bottom 2-by-2 minor + for (int l_finder = 0; l_finder < k; l_finder++) { + share* pl_finder = c->PutCONSGate((uint32_t*)&l_finder, bitlen); + share* found_l = c->PutEQGate(priv_l, pl_finder); + delete pl_finder; + share* flanc = c->PutINVGate(converged); + share* oldflanc = flanc; + flanc = c->PutANDGate(found_l, flanc); + delete oldflanc; + delete found_l; + share* oldx = x; + x = c->PutMUXGate(w[l_finder], x, flanc); + delete oldx; + delete flanc; + } + + nm = k - 1; + y = new boolshare(w[nm]->get_wires(), + c); // do not delete before assigning + if (g) + delete g; + g = new boolshare(rv1[nm]->get_wires(), c); + if (h) + delete h; + h = new boolshare(rv1[k]->get_wires(), c); + { // f = ((y - z) * (y + z) + (g - h) * (g + h)) / (2.0 * h * y); + share* ymz = c->PutFPGate(y, z, SUB, bitlen, 1, no_status); + share* ypz = c->PutFPGate(y, z, ADD, bitlen, 1, no_status); + share* temp1 = c->PutFPGate(ymz, ypz, MUL, bitlen, 1, no_status); + share* gmh = c->PutFPGate(g, h, SUB, bitlen, 1, no_status); + share* gph = c->PutFPGate(g, h, ADD, bitlen, 1, no_status); + share* temp2 = c->PutFPGate(gmh, gph, MUL, bitlen, 1, no_status); + share* num = c->PutFPGate(temp1, temp2, ADD, bitlen, 1, no_status); + share* den1 = c->PutFPGate(two_gate, h, MUL, bitlen, 1, no_status); + share* den2 = c->PutFPGate(den1, y, MUL, bitlen, 1, no_status); + f = c->PutFPGate(num, den2, DIV, bitlen, 1, no_status); + delete ymz; + delete ypz; + delete temp1; + delete gmh; + delete gph; + delete temp2; + delete num; + delete den1; + delete den2; + } + g = BuildPythagCircuit(f, one_gate, c, bitlen); + { // f = ((x - z) * (x + z) + h * ((y / (f + SIGN(g,f)))- h)) / x; + share* xmz = c->PutFPGate(x, z, SUB, bitlen, 1, no_status); + share* xpz = c->PutFPGate(x, z, ADD, bitlen, 1, no_status); + share* term1 = c->PutFPGate(xmz, xpz, MUL, bitlen, 1, no_status); + share* sgf = new boolshare(g->get_wires(), c); + BuildSignCircuit(sgf, f, c); + share* fpsgf = c->PutFPGate(f, sgf, ADD, bitlen, 1, no_status); + share* ydfpsgf = c->PutFPGate(y, fpsgf, DIV, bitlen, 1, no_status); + share* uggg = c->PutFPGate(ydfpsgf, h, SUB, bitlen, 1, no_status); + share* term2 = c->PutFPGate(h, uggg, MUL, bitlen, 1, no_status); + share* num2 = c->PutFPGate(term1, term2, ADD, bitlen, 1, no_status); + delete f; + f = c->PutFPGate(num2, x, DIV, bitlen, 1, no_status); + delete xmz; + delete xpz; + delete term1; + delete sgf; + delete fpsgf; + delete ydfpsgf; + delete uggg; + delete term2; + delete num2; + } + cc = new boolshare(one_gate->get_wires(), c); + s = new boolshare(one_gate->get_wires(), c); + + for (int j = 0; j <= nm; j++) { + share* priv_j = c->PutCONSGate((uint32_t*)&j, bitlen); + share* jgtl = c->PutGTGate(priv_l, priv_j); // have to invert for >= + share* oldjgtl = jgtl; + jgtl = c->PutINVGate(jgtl); + delete oldjgtl; + share* nconverged = c->PutINVGate(converged); + share* modify = c->PutANDGate(jgtl, nconverged); + delete nconverged; + delete jgtl; + delete priv_j; + + int i = j + 1; + delete g; + g = new boolshare(rv1[i]->get_wires(), c); + delete y; + y = new boolshare(w[i]->get_wires(), c); + delete h; + h = c->PutFPGate(s, g, MUL, bitlen, 1, no_status); + share* oldg = g; + g = c->PutFPGate(cc, g, MUL, bitlen, 1, no_status); + delete oldg; + delete z; + z = BuildPythagCircuit(f, h, c, bitlen); + share* oldr = rv1[j]; + rv1[j] = c->PutMUXGate(z, rv1[j], modify); + delete oldr; + temp = c->PutFPGate(f, z, DIV, bitlen, 1, no_status); + share* oldcc = cc; + cc = c->PutMUXGate(temp, cc, modify); + delete oldcc; + delete temp; + share* olds = s; + temp = c->PutFPGate(h, z, DIV, bitlen, 1, no_status); + s = c->PutMUXGate(temp, s, modify); + delete olds; + delete temp; + { // f = x * c + g * s; + share* term1 = c->PutFPGate(x, cc, MUL, bitlen, 1, no_status); + share* term2 = c->PutFPGate(g, s, MUL, bitlen, 1, no_status); + share* added = c->PutFPGate(term1, term2, ADD, bitlen, 1, no_status); + share* oldf = f; + f = c->PutMUXGate(added, f, modify); + delete oldf; + delete added; + delete term1; + delete term2; + } + { // g = g * c - x * s; + share* term1 = c->PutFPGate(g, cc, MUL, bitlen, 1, no_status); + share* term2 = c->PutFPGate(x, s, MUL, bitlen, 1, no_status); + share* oldg = g; + g = c->PutFPGate(term1, term2, SUB, bitlen, 1, no_status); + delete oldg; + delete term1; + delete term2; + } + h = c->PutFPGate(y, s, MUL, bitlen, 1, no_status); + y = c->PutFPGate(y, cc, MUL, bitlen, 1, no_status); + for (jj = 0; jj < nCols; jj++) { + share* oldx = x; + x = c->PutMUXGate(v[jj][j], x, modify); + delete oldx; + z = new boolshare(v[jj][i]->get_wires(), c); + { // v[jj][j] = x * c + z * s; + share* term1 = c->PutFPGate(x, cc, MUL, bitlen, 1, no_status); + share* term2 = c->PutFPGate(z, s, MUL, bitlen, 1, no_status); + share* added = + c->PutFPGate(term1, term2, ADD, bitlen, 1, no_status); + share* oldv = v[jj][j]; + v[jj][j] = c->PutMUXGate(added, v[jj][j], modify); + delete oldv; + delete added; + delete term1; + delete term2; + } + { // v[jj][i] = z * c - x * s; + share* term1 = c->PutFPGate(z, cc, MUL, bitlen, 1, no_status); + share* term2 = c->PutFPGate(x, s, MUL, bitlen, 1, no_status); + share* subbed = + c->PutFPGate(term1, term2, SUB, bitlen, 1, no_status); + share* oldv = v[jj][i]; + v[jj][i] = c->PutMUXGate(subbed, v[jj][i], modify); + delete oldv; + delete subbed; + delete term1; + delete term2; + } + } + z = BuildPythagCircuit(f, h, c, bitlen); + share* oldw = w[j]; + w[j] = c->PutMUXGate(z, w[j], modify); + delete oldw; + { // if (z) { + share* temp; + share* ifz = z->get_wire_ids_as_share(0); + for (uint32_t tt = 1; tt < bitlen - 1; + tt++) { // -1 -> do not include sign bit + share* tempbit = z->get_wire_ids_as_share(tt); + temp = ifz; + ifz = c->PutORGate(temp, tempbit); + delete tempbit; + delete temp; + } + share* ifzam = c->PutANDGate(ifz, modify); + delete ifz; + + temp = c->PutFPGate(one_gate, z, DIV, bitlen, 1, no_status); + z = c->PutMUXGate(temp, z, ifzam); + delete temp; + + temp = c->PutFPGate(f, z, MUL, bitlen, 1, no_status); + cc = c->PutMUXGate(temp, cc, ifzam); + delete temp; + + temp = c->PutFPGate(h, z, MUL, bitlen, 1, no_status); + s = c->PutMUXGate(temp, s, ifzam); + delete temp; + } + { // f = c * g + s * y; + share* term1 = c->PutFPGate(cc, g, MUL, bitlen, 1, no_status); + share* term2 = c->PutFPGate(s, y, MUL, bitlen, 1, no_status); + share* added = c->PutFPGate(term1, term2, ADD, bitlen, 1, no_status); + share* oldf = f; + f = c->PutMUXGate(added, f, modify); + delete oldf; + delete added; + delete term1; + delete term2; + } + { // x = c * y - s * g; + share* term1 = c->PutFPGate(cc, y, MUL, bitlen, 1, no_status); + share* term2 = c->PutFPGate(s, g, MUL, bitlen, 1, no_status); + share* subbed = c->PutFPGate(term1, term2, SUB, bitlen, 1, no_status); + share* oldx = x; + x = c->PutMUXGate(subbed, x, modify); + delete oldx; + delete subbed; + delete term1; + delete term2; + } + for (jj = 0; jj < nRows; jj++) { + y = new boolshare(a[jj][j]->get_wires(), c); + z = new boolshare(a[jj][i]->get_wires(), c); + { // a[jj][j] = y * c + z * s; + share* term1 = c->PutFPGate(y, cc, MUL, bitlen, 1, no_status); + share* term2 = c->PutFPGate(z, s, MUL, bitlen, 1, no_status); + share* added = + c->PutFPGate(term1, term2, ADD, bitlen, 1, no_status); + share* olda = a[jj][j]; + a[jj][j] = c->PutMUXGate(added, a[jj][j], modify); + delete olda; + delete added; + delete term1; + delete term2; + } + { // a[jj][i] = z * c - y * s; + share* term1 = c->PutFPGate(z, cc, MUL, bitlen, 1, no_status); + share* term2 = c->PutFPGate(y, s, MUL, bitlen, 1, no_status); + share* subbed = + c->PutFPGate(term1, term2, SUB, bitlen, 1, no_status); + share* olda = a[jj][i]; + a[jj][i] = c->PutMUXGate(subbed, a[jj][i], modify); + delete olda; + delete subbed; + delete term1; + delete term2; + } + } + } + for (int l_finder = 0; l_finder < k; l_finder++) { + share* pl_finder = c->PutCONSGate((uint32_t*)&l_finder, 32); + share* found_l = c->PutEQGate(priv_l, pl_finder); + delete pl_finder; + share* flanc = c->PutINVGate(converged); + share* oldflanc = flanc; + flanc = c->PutANDGate(found_l, flanc); + delete oldflanc; + delete found_l; + share* oldr = rv1[l_finder]; + rv1[l_finder] = c->PutMUXGate(zero_gate, rv1[l_finder], flanc); + delete oldr; + delete flanc; + } + share* nconverged = c->PutINVGate(converged); + share* oldr = rv1[k]; + rv1[k] = c->PutMUXGate(f, rv1[k], nconverged); + delete oldr; + share* oldw = w[k]; + w[k] = c->PutMUXGate(x, w[k], nconverged); + delete oldw; + delete nconverged; + } + delete converged; + printf("/"); + fflush(stdout); + } +} + +// Build SVD circuit. Since it does not need +// intermediate values, it does not use subcircuits +// thus only builds the circuit but does not execute. +void BuildSvdOSL(share*** a, int nRows, int nCols, share** w, share*** v, + BooleanCircuit* c, ABYParty* party, e_role role) { + + // Following values are shared between subcircuits + int /*flag,*/ i, its, j, jj, k, l, nm; + share *anorm = nullptr, *cc = nullptr, *f = nullptr, *g = nullptr, + *h = nullptr, *s = nullptr, *scale = nullptr, *x = nullptr, + *y = nullptr, *z = nullptr; + share** rv1 = new share*[nCols]; + + float zero = 0; + share* zero_gate = c->PutCONSGate((uint32_t*)&zero, bitlen); + int int_zero = 0; + share* int_zero_gate = c->PutCONSGate((uint32_t*)&int_zero, bitlen); + float one = 1; + share* one_gate = c->PutCONSGate((uint32_t*)&one, bitlen); + int int_one = 1; + share* int_one_gate = c->PutCONSGate((uint32_t*)&int_one, bitlen); + float two = 2; + share* two_gate = c->PutCONSGate((uint32_t*)&two, bitlen); + uint8_t true_val = 1; + share* true_gate = c->PutCONSGate((uint8_t*)&true_val, 1); + uint8_t false_val = 0; + share* false_gate = c->PutCONSGate((uint8_t*)&false_val, 1); + share* temp; + + x = new boolshare(zero_gate->get_wires(), c); // x must be initialized + + BuildSvdPart1Circuit(a, nRows, nCols, w, v, &anorm, rv1, c); + + for (k = nCols - 1; k >= 0; k--) { + share* converged = new boolshare(false_gate->get_wires(), c); + for (its = 0; its < 2; its++) { + cout << k << flush; + share* priv_l = new boolshare(int_zero_gate->get_wires(), c); + + z = w[k]; + share* tempk = c->PutCONSGate((uint32_t*)&k, 32); + share* newconverged = c->PutEQGate(priv_l, tempk); + delete tempk; + { // if (l == k) + { // if (z < 0.0) singular value is made nonnegative + share* isneg = c->PutFPGate(zero_gate, z, CMP); + share* modify = c->PutANDGate(isneg, newconverged); + share* nconverged = c->PutINVGate(converged); + share* oldmodify = modify; + modify = c->PutANDGate(modify, nconverged); + delete oldmodify; + + share* temp = new boolshare(z->get_wires(), c); + BuildNegativeCircuit(temp, c); + w[k] = c->PutMUXGate(temp, w[k], modify); + for (int j = 0; j < nCols; j++) { + share* negv = new boolshare(v[j][k]->get_wires(), c); + BuildNegativeCircuit(negv, c); + share* oldvjk = v[j][k]; + v[j][k] = c->PutMUXGate(negv, v[j][k], modify); + delete negv; + delete oldvjk; + } + } + } + share* oldconverged = converged; + converged = c->PutORGate(converged, newconverged); + delete oldconverged; + if (k == 0) + break; // if k == 0 we are done, code below segfaults from nm + + // shift from bottom 2-by-2 minor + for (int l_finder = 0; l_finder < k; l_finder++) { + share* pl_finder = c->PutCONSGate((uint32_t*)&l_finder, bitlen); + share* found_l = c->PutEQGate(priv_l, pl_finder); + delete pl_finder; + share* flanc = c->PutINVGate(converged); + share* oldflanc = flanc; + flanc = c->PutANDGate(found_l, flanc); + delete oldflanc; + delete found_l; + share* oldx = x; + x = c->PutMUXGate(w[l_finder], x, flanc); + delete oldx; + delete flanc; + } + + nm = k - 1; + y = new boolshare(w[nm]->get_wires(), + c); // do not delete before assigning + if (g) + delete g; + g = new boolshare(rv1[nm]->get_wires(), c); + if (h) + delete h; + h = new boolshare(rv1[k]->get_wires(), c); + { // f = ((y - z) * (y + z) + (g - h) * (g + h)) / (2.0 * h * y); + share* ymz = c->PutFPGate(y, z, SUB, bitlen, 1, no_status); + share* ypz = c->PutFPGate(y, z, ADD, bitlen, 1, no_status); + share* temp1 = c->PutFPGate(ymz, ypz, MUL, bitlen, 1, no_status); + share* gmh = c->PutFPGate(g, h, SUB, bitlen, 1, no_status); + share* gph = c->PutFPGate(g, h, ADD, bitlen, 1, no_status); + share* temp2 = c->PutFPGate(gmh, gph, MUL, bitlen, 1, no_status); + share* num = c->PutFPGate(temp1, temp2, ADD, bitlen, 1, no_status); + share* den1 = c->PutFPGate(two_gate, h, MUL, bitlen, 1, no_status); + share* den2 = c->PutFPGate(den1, y, MUL, bitlen, 1, no_status); + f = c->PutFPGate(num, den2, DIV, bitlen, 1, no_status); + delete ymz; + delete ypz; + delete temp1; + delete gmh; + delete gph; + delete temp2; + delete num; + delete den1; + delete den2; + } + g = BuildPythagCircuit(f, one_gate, c, bitlen); + { // f = ((x - z) * (x + z) + h * ((y / (f + SIGN(g,f)))- h)) / x; + share* xmz = c->PutFPGate(x, z, SUB, bitlen, 1, no_status); + share* xpz = c->PutFPGate(x, z, ADD, bitlen, 1, no_status); + share* term1 = c->PutFPGate(xmz, xpz, MUL, bitlen, 1, no_status); + share* sgf = new boolshare(g->get_wires(), c); + BuildSignCircuit(sgf, f, c); + share* fpsgf = c->PutFPGate(f, sgf, ADD, bitlen, 1, no_status); + share* ydfpsgf = c->PutFPGate(y, fpsgf, DIV, bitlen, 1, no_status); + share* uggg = c->PutFPGate(ydfpsgf, h, SUB, bitlen, 1, no_status); + share* term2 = c->PutFPGate(h, uggg, MUL, bitlen, 1, no_status); + share* num2 = c->PutFPGate(term1, term2, ADD, bitlen, 1, no_status); + delete f; + f = c->PutFPGate(num2, x, DIV, bitlen, 1, no_status); + delete xmz; + delete xpz; + delete term1; + delete sgf; + delete fpsgf; + delete ydfpsgf; + delete uggg; + delete term2; + delete num2; + } + cc = new boolshare(one_gate->get_wires(), c); + s = new boolshare(one_gate->get_wires(), c); + + for (int j = 0; j <= nm; j++) { + share* priv_j = c->PutCONSGate((uint32_t*)&j, bitlen); + share* jgtl = c->PutGTGate(priv_l, priv_j); // have to invert for >= + share* oldjgtl = jgtl; + jgtl = c->PutINVGate(jgtl); + delete oldjgtl; + share* nconverged = c->PutINVGate(converged); + share* modify = c->PutANDGate(jgtl, nconverged); + delete nconverged; + delete jgtl; + delete priv_j; + + int i = j + 1; + delete g; + g = new boolshare(rv1[i]->get_wires(), c); + delete y; + y = new boolshare(w[i]->get_wires(), c); + delete h; + h = c->PutFPGate(s, g, MUL, bitlen, 1, no_status); + share* oldg = g; + g = c->PutFPGate(cc, g, MUL, bitlen, 1, no_status); + delete oldg; + delete z; + z = BuildPythagCircuit(f, h, c, bitlen); + share* oldr = rv1[j]; + rv1[j] = c->PutMUXGate(z, rv1[j], modify); + delete oldr; + temp = c->PutFPGate(f, z, DIV, bitlen, 1, no_status); + share* oldcc = cc; + cc = c->PutMUXGate(temp, cc, modify); + delete oldcc; + delete temp; + share* olds = s; + temp = c->PutFPGate(h, z, DIV, bitlen, 1, no_status); + s = c->PutMUXGate(temp, s, modify); + delete olds; + delete temp; + { // f = x * c + g * s; + share* term1 = c->PutFPGate(x, cc, MUL, bitlen, 1, no_status); + share* term2 = c->PutFPGate(g, s, MUL, bitlen, 1, no_status); + share* added = c->PutFPGate(term1, term2, ADD, bitlen, 1, no_status); + share* oldf = f; + f = c->PutMUXGate(added, f, modify); + delete oldf; + delete added; + delete term1; + delete term2; + } + { // g = g * c - x * s; + share* term1 = c->PutFPGate(g, cc, MUL, bitlen, 1, no_status); + share* term2 = c->PutFPGate(x, s, MUL, bitlen, 1, no_status); + share* oldg = g; + g = c->PutFPGate(term1, term2, SUB, bitlen, 1, no_status); + delete oldg; + delete term1; + delete term2; + } + h = c->PutFPGate(y, s, MUL, bitlen, 1, no_status); + y = c->PutFPGate(y, cc, MUL, bitlen, 1, no_status); + for (jj = 0; jj < nCols; jj++) { + share* oldx = x; + x = c->PutMUXGate(v[jj][j], x, modify); + delete oldx; + z = new boolshare(v[jj][i]->get_wires(), c); + { // v[jj][j] = x * c + z * s; + share* term1 = c->PutFPGate(x, cc, MUL, bitlen, 1, no_status); + share* term2 = c->PutFPGate(z, s, MUL, bitlen, 1, no_status); + share* added = + c->PutFPGate(term1, term2, ADD, bitlen, 1, no_status); + share* oldv = v[jj][j]; + v[jj][j] = c->PutMUXGate(added, v[jj][j], modify); + delete oldv; + delete added; + delete term1; + delete term2; + } + { // v[jj][i] = z * c - x * s; + share* term1 = c->PutFPGate(z, cc, MUL, bitlen, 1, no_status); + share* term2 = c->PutFPGate(x, s, MUL, bitlen, 1, no_status); + share* subbed = + c->PutFPGate(term1, term2, SUB, bitlen, 1, no_status); + share* oldv = v[jj][i]; + v[jj][i] = c->PutMUXGate(subbed, v[jj][i], modify); + delete oldv; + delete subbed; + delete term1; + delete term2; + } + } + z = BuildPythagCircuit(f, h, c, bitlen); + share* oldw = w[j]; + w[j] = c->PutMUXGate(z, w[j], modify); + delete oldw; + { // if (z) { + share* temp; + share* ifz = z->get_wire_ids_as_share(0); + for (uint32_t tt = 1; tt < bitlen - 1; + tt++) { // -1 -> do not include sign bit + share* tempbit = z->get_wire_ids_as_share(tt); + temp = ifz; + ifz = c->PutORGate(temp, tempbit); + delete tempbit; + delete temp; + } + share* ifzam = c->PutANDGate(ifz, modify); + delete ifz; + + temp = c->PutFPGate(one_gate, z, DIV, bitlen, 1, no_status); + z = c->PutMUXGate(temp, z, ifzam); + delete temp; + + temp = c->PutFPGate(f, z, MUL, bitlen, 1, no_status); + cc = c->PutMUXGate(temp, cc, ifzam); + delete temp; + + temp = c->PutFPGate(h, z, MUL, bitlen, 1, no_status); + s = c->PutMUXGate(temp, s, ifzam); + delete temp; + } + { // f = c * g + s * y; + share* term1 = c->PutFPGate(cc, g, MUL, bitlen, 1, no_status); + share* term2 = c->PutFPGate(s, y, MUL, bitlen, 1, no_status); + share* added = c->PutFPGate(term1, term2, ADD, bitlen, 1, no_status); + share* oldf = f; + f = c->PutMUXGate(added, f, modify); + delete oldf; + delete added; + delete term1; + delete term2; + } + { // x = c * y - s * g; + share* term1 = c->PutFPGate(cc, y, MUL, bitlen, 1, no_status); + share* term2 = c->PutFPGate(s, g, MUL, bitlen, 1, no_status); + share* subbed = c->PutFPGate(term1, term2, SUB, bitlen, 1, no_status); + share* oldx = x; + x = c->PutMUXGate(subbed, x, modify); + delete oldx; + delete subbed; + delete term1; + delete term2; + } + for (jj = 0; jj < nRows; jj++) { + y = new boolshare(a[jj][j]->get_wires(), c); + z = new boolshare(a[jj][i]->get_wires(), c); + { // a[jj][j] = y * c + z * s; + share* term1 = c->PutFPGate(y, cc, MUL, bitlen, 1, no_status); + share* term2 = c->PutFPGate(z, s, MUL, bitlen, 1, no_status); + share* added = + c->PutFPGate(term1, term2, ADD, bitlen, 1, no_status); + share* olda = a[jj][j]; + a[jj][j] = c->PutMUXGate(added, a[jj][j], modify); + delete olda; + delete added; + delete term1; + delete term2; + } + { // a[jj][i] = z * c - y * s; + share* term1 = c->PutFPGate(z, cc, MUL, bitlen, 1, no_status); + share* term2 = c->PutFPGate(y, s, MUL, bitlen, 1, no_status); + share* subbed = + c->PutFPGate(term1, term2, SUB, bitlen, 1, no_status); + share* olda = a[jj][i]; + a[jj][i] = c->PutMUXGate(subbed, a[jj][i], modify); + delete olda; + delete subbed; + delete term1; + delete term2; + } + } + } + for (int l_finder = 0; l_finder < k; l_finder++) { + share* pl_finder = c->PutCONSGate((uint32_t*)&l_finder, 32); + share* found_l = c->PutEQGate(priv_l, pl_finder); + delete pl_finder; + share* flanc = c->PutINVGate(converged); + share* oldflanc = flanc; + flanc = c->PutANDGate(found_l, flanc); + delete oldflanc; + delete found_l; + share* oldr = rv1[l_finder]; + rv1[l_finder] = c->PutMUXGate(zero_gate, rv1[l_finder], flanc); + delete oldr; + delete flanc; + } + share* nconverged = c->PutINVGate(converged); + share* oldr = rv1[k]; + rv1[k] = c->PutMUXGate(f, rv1[k], nconverged); + delete oldr; + share* oldw = w[k]; + w[k] = c->PutMUXGate(x, w[k], nconverged); + delete oldw; + delete nconverged; + } + delete converged; + printf("/"); + fflush(stdout); + } +} + +void BuildAndRunSvd(share*** s_a, int nRows, int nCols, share** s_w, + share*** s_v, BooleanCircuit* c, ABYParty* party, + e_role role, std::function toRawShares, + std::function toShareObjects) { +#if PPL_FLOW == PPL_FLOW_DO // set dx to zero if no error + (void)toRawShares; + (void)toShareObjects; + BuildSvdDO(s_a, nRows, nCols, s_w, s_v, c, party, role); +#elif PPL_FLOW == PPL_FLOW_LOOP_LEAK + BuildAndRunSvdLoopLeak(s_a, nRows, nCols, s_w, s_v, c, party, role, + toRawShares, toShareObjects); +#elif PPL_FLOW == PPL_FLOW_SiSL + BuildSvdOSL(s_a, nRows, nCols, s_w, s_v, c, party, role); +#endif +} + +uint32_t test_fabs_circuit(e_role role, const std::string& address, + uint16_t port, seclvl seclvl, uint32_t nthreads, + e_mt_gen_alg mt_alg, e_sharing sharing, float a) { + uint32_t bitlen = 32; + + uint32_t reservegates = 65536; + const std::string& abycircdir = "../../extern/ABY/bin/circ"; + ABYParty* party = new ABYParty(role, address, port, seclvl, bitlen, nthreads, + mt_alg, reservegates, abycircdir); + + std::vector& sharings = party->GetSharings(); + + Circuit* circ = sharings[sharing]->GetCircuitBuildRoutine(); + + share* s_a; + + if (role == SERVER) { + s_a = circ->PutINGate((uint32_t*)&a, 32, role); + } else { + s_a = circ->PutDummyINGate(32); + } + + share* out; + BuildFabsCircuit(s_a, (BooleanCircuit*)circ); + + out = circ->PutOUTGate(s_a, ALL); + + CLOCK(ExecCircuit); + TIC(ExecCircuit); + party->ExecCircuit(); + TOC(ExecCircuit); + + uint32_t* output; + uint32_t out_bitlen, out_nvals; + + // This method only works for an output length of maximum 64 bits in general, + // if the output length is higher you must use get_clear_value_ptr + out->get_clear_value_vec(&output, &out_bitlen, &out_nvals); + // R[i]->get_clear_value_ptr(); + + // cout << out_bitlen << "jfjf" << endl; + cout << *(float*)output << endl; + + delete party; + // delete[] A; + // delete[] B; + // delete[] s_A; + // delete[] s_B; + // delete[] s_out; + return 0; +} + +uint32_t test_fmax_circuit(e_role role, const std::string& address, + uint16_t port, seclvl seclvl, uint32_t nthreads, + e_mt_gen_alg mt_alg, e_sharing sharing, float a, + float b) { + uint32_t bitlen = 32; + + uint32_t reservegates = 65536; + const std::string& abycircdir = "../../extern/ABY/bin/circ"; + ABYParty* party = new ABYParty(role, address, port, seclvl, bitlen, nthreads, + mt_alg, reservegates, abycircdir); + + std::vector& sharings = party->GetSharings(); + + Circuit* circ = sharings[sharing]->GetCircuitBuildRoutine(); + + share* s_a; + share* s_b; + + if (role == SERVER) { + s_a = circ->PutINGate((uint32_t*)&a, 32, role); + s_b = circ->PutINGate((uint32_t*)&b, 32, role); + } else { + s_a = circ->PutDummyINGate(32); + s_b = circ->PutDummyINGate(32); + } + + share* s_out; + share* out; + s_out = BuildFMAXCircuit(s_a, s_b, (BooleanCircuit*)circ); + + out = circ->PutOUTGate(s_out, ALL); + + CLOCK(ExecCircuit); + TIC(ExecCircuit); + party->ExecCircuit(); + TOC(ExecCircuit); + + uint32_t* output; + uint32_t out_bitlen, out_nvals; + + // This method only works for an output length of maximum 64 bits in general, + // if the output length is higher you must use get_clear_value_ptr + out->get_clear_value_vec(&output, &out_bitlen, &out_nvals); + // R[i]->get_clear_value_ptr(); + + // cout << out_bitlen << "jfjf" << endl; + cout << *(float*)output << endl; + + delete party; + // delete[] A; + // delete[] B; + // delete[] s_A; + // delete[] s_B; + // delete[] s_out; + return 0; +} + +uint32_t test_negative_circuit(e_role role, const std::string& address, + uint16_t port, seclvl seclvl, uint32_t nthreads, + e_mt_gen_alg mt_alg, e_sharing sharing, + float a) { + uint32_t bitlen = 32; + + uint32_t reservegates = 65536; + const std::string& abycircdir = "../../extern/ABY/bin/circ"; + ABYParty* party = new ABYParty(role, address, port, seclvl, bitlen, nthreads, + mt_alg, reservegates, abycircdir); + + std::vector& sharings = party->GetSharings(); + + Circuit* circ = sharings[sharing]->GetCircuitBuildRoutine(); + + share* s_a; + + if (role == SERVER) { + s_a = circ->PutINGate((uint32_t*)&a, 32, role); + } else { + s_a = circ->PutDummyINGate(32); + } + + // share* s_out; + share* out; + BuildNegativeCircuit(s_a, (BooleanCircuit*)circ); + + out = circ->PutOUTGate(s_a, ALL); + + CLOCK(ExecCircuit); + TIC(ExecCircuit); + party->ExecCircuit(); + TOC(ExecCircuit); + + uint32_t* output; + uint32_t out_bitlen, out_nvals; + + // This method only works for an output length of maximum 64 bits in general, + // if the output length is higher you must use get_clear_value_ptr + out->get_clear_value_vec(&output, &out_bitlen, &out_nvals); + // R[i]->get_clear_value_ptr(); + + // cout << out_bitlen << "jfjf" << endl; + cout << *(float*)output << endl; + + delete party; + // delete[] A; + // delete[] B; + // delete[] s_A; + // delete[] s_B; + // delete[] s_out; + return 0; +} diff --git a/src/aby-float-server/svd.h b/src/aby-float-server/svd.h new file mode 100644 index 0000000..41c5d78 --- /dev/null +++ b/src/aby-float-server/svd.h @@ -0,0 +1,45 @@ +#pragma once +#include +#include +#include + +#include "abycore/aby/abyparty.h" +#include "abycore/circuit/arithmeticcircuits.h" +#include "abycore/circuit/booleancircuits.h" +#include "abycore/circuit/circuit.h" +#include "abycore/sharing/sharing.h" + +void BuildFabsCircuit(share* a, BooleanCircuit* c); + +void BuildSignCircuit(share* a, share* b, BooleanCircuit* c); + +share* BuildPythagCircuit(share* a, share* b, BooleanCircuit* c, + uint32_t bitlen); + +// uint32_t arguments must be from PutSharedOUTGate() +// then calling get_clear_value() and circuit->Reset(); +// This function builds/executes multiple circuits. +void RunSvd(uint32_t** a, int nRows, int nCols, uint32_t* w, uint32_t** v, + BooleanCircuit* c, ABYParty* p, e_role role); + +// Wrapper function around RunSvd which +// takes shares instead of secret shares. +// Creates dummy circuits to build secret shares. +void BuildAndRunSvd( + share*** s_a, int nRows, int nCols, share** s_w, share*** s_v, + BooleanCircuit* c, ABYParty* party, e_role role, + std::function toRawShares = []() {}, + std::function toShareObjects = []() {}); + +uint32_t test_fabs_circuit(e_role role, const std::string& address, + uint16_t port, seclvl seclvl, uint32_t nthreads, + e_mt_gen_alg mt_alg, e_sharing sharing, float a); + +uint32_t test_fmax_circuit(e_role role, const std::string& address, + uint16_t port, seclvl seclvl, uint32_t nthreads, + e_mt_gen_alg mt_alg, e_sharing sharing, float a, + float b); + +uint32_t test_negative_circuit(e_role role, const std::string& address, + uint16_t port, seclvl seclvl, uint32_t nthreads, + e_mt_gen_alg mt_alg, e_sharing sharing, float a); diff --git a/src/aby-float-server/trigfuncs.cpp b/src/aby-float-server/trigfuncs.cpp new file mode 100644 index 0000000..5b8a31c --- /dev/null +++ b/src/aby-float-server/trigfuncs.cpp @@ -0,0 +1,98 @@ +#include +#include +#include +#include + +#include "abycore/aby/abyparty.h" +#include "abycore/circuit/arithmeticcircuits.h" +#include "abycore/circuit/booleancircuits.h" +#include "abycore/circuit/circuit.h" +#include "abycore/sharing/sharing.h" + +#include + +using namespace std; + +const uint32_t bitlen = 32; + +share* BuildSinCircuit(share* theta, BooleanCircuit* c) { + float pi = M_PI; + share* pi_gate = c->PutCONSGate((uint32_t*)&pi, bitlen); + share* adjusted = c->PutFPGate(theta, pi_gate, DIV, bitlen, 1, no_status); + share* si = c->PutFPGate( + adjusted, SIN); // sets bitlength to 40 (?), other circuit is 33 (?) + // cout << "sine bitlength " <get_bitlength()<set_bitlength(32); + delete pi_gate; + delete adjusted; + return si; + + // Below uses taylor series approx + ////return x - (pow(x,3)/(3*2)) + (pow(x,5)/(5*4*3*2));// - + ///(pow(x,7)/(7*6*5*4*3*2)); + // share *t1 = c->PutFPGate(theta, theta, MUL, bitlen, 1, no_status); + // t1 = c->PutFPGate(t1, theta, MUL, bitlen, 1, no_status); + // float threeFact = 3*2; + // share* tFGate = c->PutCONSGate((uint32_t*)&threeFact, bitlen); + // t1 = c->PutFPGate(t1, tFGate, DIV, bitlen, 1, no_status); + + // share *t2 = c->PutFPGate(theta, theta, MUL, bitlen, 1, no_status); + // t2 = c->PutFPGate(t2, theta, MUL, bitlen, 1, no_status); + // t2 = c->PutFPGate(t2, theta, MUL, bitlen, 1, no_status); + // t2 = c->PutFPGate(t2, theta, MUL, bitlen, 1, no_status); + // float fiveFact = 5*4*3*2; + // share* fFGate = c->PutCONSGate((uint32_t*)&fiveFact, bitlen); + // t2 = c->PutFPGate(t2, fFGate, DIV, bitlen, 1, no_status); + + // share *res = c->PutFPGate(theta, t1, SUB, bitlen, 1, no_status); + // res = c->PutFPGate(res, t2, ADD, bitlen, 1, no_status); + + //// TODO still has mem leaks + // delete t1; + // delete tFGate; + // delete t2; + // delete fFGate; + + // return res; +} + +share* BuildCosCircuit(share* theta, BooleanCircuit* c) { + float pi = M_PI; + share* pi_gate = c->PutCONSGate((uint32_t*)&pi, bitlen); + share* adjusted = c->PutFPGate(theta, pi_gate, DIV, bitlen, 1, no_status); + share* co = c->PutFPGate( + adjusted, COS); // sets bitlength to 40 (?), other circuit is 33 (?) + // cout << "sine bitlength " <get_bitlength()<set_bitlength(32); + delete pi_gate; + delete adjusted; + return co; + + // Below uses taylor series approx + + ////return 1 - (pow(x,2)/2) + (pow(x,4)/(4*3*2));// - (pow(x,6)/(6*5*4*3*2)); + // share *t1 = c->PutFPGate(theta, theta, MUL, bitlen, 1, no_status); + // float twoFact = 2; + // share* tFGate = c->PutCONSGate((uint32_t*)&twoFact, bitlen); + // t1 = c->PutFPGate(t1, tFGate, DIV, bitlen, 1, no_status); + + // share *t2 = c->PutFPGate(theta, theta, MUL, bitlen, 1, no_status); + // t2 = c->PutFPGate(t2, theta, MUL, bitlen, 1, no_status); + // t2 = c->PutFPGate(t2, theta, MUL, bitlen, 1, no_status); + // float fourFact = 4*3*2; + // share* fFGate = c->PutCONSGate((uint32_t*)&fourFact, bitlen); + // t2 = c->PutFPGate(t2, fFGate, DIV, bitlen, 1, no_status); + + // float one = 1; + // share* oneGate = c->PutCONSGate((uint32_t*)&one, bitlen); + // share *res = c->PutFPGate(oneGate, t1, SUB, bitlen, 1, no_status); + // res = c->PutFPGate(res, t2, ADD, bitlen, 1, no_status); + + //// TODO still has mem leaks + // delete t1; + // delete tFGate; + // delete t2; + // delete fFGate; + + // return res; +} diff --git a/src/aby-float-server/trigfuncs.h b/src/aby-float-server/trigfuncs.h new file mode 100644 index 0000000..e4f7ed7 --- /dev/null +++ b/src/aby-float-server/trigfuncs.h @@ -0,0 +1,15 @@ +#include +#include +#include +#include + +#include "abycore/aby/abyparty.h" +#include "abycore/circuit/arithmeticcircuits.h" +#include "abycore/circuit/booleancircuits.h" +#include "abycore/circuit/circuit.h" +#include "abycore/sharing/sharing.h" + +using namespace std; + +share* BuildSinCircuit(share* theta, BooleanCircuit* c); +share* BuildCosCircuit(share* theta, BooleanCircuit* c); diff --git a/src/aby-float-server/twonormsq.cpp b/src/aby-float-server/twonormsq.cpp new file mode 100644 index 0000000..6b0d883 --- /dev/null +++ b/src/aby-float-server/twonormsq.cpp @@ -0,0 +1,38 @@ +#include +#include +#include +#include "jlog.h" + +#include "abycore/aby/abyparty.h" +#include "abycore/circuit/arithmeticcircuits.h" +#include "abycore/circuit/booleancircuits.h" +#include "abycore/circuit/circuit.h" +#include "abycore/sharing/sharing.h" + +using namespace std; + +/* + An implementation of SVD from Numerical Recipes in C and Mike Erhdmann's + lectures + */ +#include +#include +#include + +const uint32_t bitlen = 32; + +// M mxn (M is secret, m n are public) +// N mmxnn +// res mxnn +share* BuildTwoNormSqCircuit(share* vect[], int sz, BooleanCircuit* c) { + float zero = 0.0; + share* sum = c->PutCONSGate((uint32_t*)&zero, bitlen); + for (int i = 0; i < sz; i++) { + share* temp = c->PutFPGate(vect[i], vect[i], MUL, bitlen, 1, no_status); + share* oldsum = sum; + sum = c->PutFPGate(sum, temp, ADD, bitlen, 1, no_status); + delete temp; + delete oldsum; + } + return sum; +} diff --git a/src/aby-float-server/twonormsq.h b/src/aby-float-server/twonormsq.h new file mode 100644 index 0000000..3bb94cb --- /dev/null +++ b/src/aby-float-server/twonormsq.h @@ -0,0 +1,7 @@ +#include "abycore/aby/abyparty.h" +#include "abycore/circuit/arithmeticcircuits.h" +#include "abycore/circuit/booleancircuits.h" +#include "abycore/circuit/circuit.h" +#include "abycore/sharing/sharing.h" + +share* BuildTwoNormSqCircuit(share* vect[], int sz, BooleanCircuit* c); diff --git a/src/aby-float-server/util.cpp b/src/aby-float-server/util.cpp new file mode 100644 index 0000000..fe211ed --- /dev/null +++ b/src/aby-float-server/util.cpp @@ -0,0 +1,81 @@ +#include +#include +#include + +#include <../ENCRYPTO_utils/src/ENCRYPTO_utils/timer.h> // hack {: +#include +#include +#include "util.h" + +#define BUILD_TIMING 0 +#define EXEC_TIMING 0 + +aby_timings sumTimes[P_LAST - P_FIRST + 1]; +aby_comm sumSend[P_LAST - P_FIRST + 1]; +aby_comm sumRecv[P_LAST - P_FIRST + 1]; + +void collectTiming() { + for (int phase = P_FIRST; phase <= P_LAST; phase++) { + sumTimes[phase].timing += m_tTimes[phase].timing; + } +} + +void collectCommunication() { + for (int phase = P_FIRST; phase <= P_LAST; phase++) { + sumSend[phase].totalcomm += m_tSend[phase].totalcomm; + sumRecv[phase].totalcomm += m_tRecv[phase].totalcomm; + } +} + +void PrintSumTimings(int x) { + std::string unit = " ms"; + std::cout << "Sum Total Timings: " << std::endl; + MSG("SeNtInAl,xy,%s,%s,%d,%g\n", __FUNCTION__, "gngd_total", x, + sumTimes[P_TOTAL].timing); + MSG("SeNtInAl,xy,%s,%s,%d,%g\n", __FUNCTION__, "gngd_init", x, + sumTimes[P_INIT].timing); + MSG("SeNtInAl,xy,%s,%s,%d,%g\n", __FUNCTION__, "gngd_circuit", x, + sumTimes[P_CIRCUIT].timing); + MSG("SeNtInAl,xy,%s,%s,%d,%g\n", __FUNCTION__, "gngd_network", x, + sumTimes[P_NETWORK].timing); + MSG("SeNtInAl,xy,%s,%s,%d,%g\n", __FUNCTION__, "gngd_baseot", x, + sumTimes[P_BASE_OT].timing); + MSG("SeNtInAl,xy,%s,%s,%d,%g\n", __FUNCTION__, "gngd_setup", x, + sumTimes[P_SETUP].timing); + MSG("SeNtInAl,xy,%s,%s,%d,%g\n", __FUNCTION__, "gngd_otext", x, + sumTimes[P_OT_EXT].timing); + MSG("SeNtInAl,xy,%s,%s,%d,%g\n", __FUNCTION__, "gngd_garble", x, + sumTimes[P_GARBLE].timing); + MSG("SeNtInAl,xy,%s,%s,%d,%g\n", __FUNCTION__, "gngd_online", x, + sumTimes[P_ONLINE].timing); + for (int phase = P_FIRST; phase <= P_LAST; phase++) { + sumTimes[phase].timing = 0; + } +} + +void ClearSumTimings() { + for (int phase = P_FIRST; phase <= P_LAST; phase++) { + sumTimes[phase].timing = 0; + } +} + +void PrintSumCommunication() { + std::string unit = " bytes"; + std::cout << "Communication: " << std::endl; + std::cout << "Total Sent / Rcv\t" << m_tSend[P_TOTAL].totalcomm << " " << unit + << " / " << m_tRecv[P_TOTAL].totalcomm << unit << std::endl; + std::cout << "BaseOTs Sent / Rcv\t" << m_tSend[P_BASE_OT].totalcomm << " " + << unit << " / " << m_tRecv[P_BASE_OT].totalcomm << unit + << std::endl; + std::cout << "Setup Sent / Rcv\t" << m_tSend[P_SETUP].totalcomm << " " << unit + << " / " << m_tRecv[P_SETUP].totalcomm << unit << std::endl; + std::cout << "OTExtension Sent / Rcv\t" << m_tSend[P_OT_EXT].totalcomm << " " + << unit << " / " << m_tRecv[P_OT_EXT].totalcomm << unit + << std::endl; + std::cout << "Garbling Sent / Rcv\t" << m_tSend[P_GARBLE].totalcomm << " " + << unit << " / " << m_tRecv[P_GARBLE].totalcomm << unit + << std::endl; + std::cout << "Online Sent / Rcv\t" << m_tSend[P_ONLINE].totalcomm << " " + << unit << " / " << m_tRecv[P_ONLINE].totalcomm << unit + << std::endl; +} diff --git a/src/aby-float-server/util.h b/src/aby-float-server/util.h new file mode 100644 index 0000000..29a9a67 --- /dev/null +++ b/src/aby-float-server/util.h @@ -0,0 +1,14 @@ +#include +#include + +#include +//#include <../ENCRYPTO_utils/src/ENCRYPTO_utils/timer.h> // hack {: + +#define BUILD_TIMING 0 +#define EXEC_TIMING 0 + +void collectTiming(); +void collectCommunication(); +void PrintSumTimings(int x); +void ClearSumTimings(); +void PrintSumCommunication(); diff --git a/src/common/fixed_point.h b/src/common/fixed_point.h new file mode 100644 index 0000000..b0c0434 --- /dev/null +++ b/src/common/fixed_point.h @@ -0,0 +1,1268 @@ +/*******************************************************************************/ +/* */ +/* Copyright (c) 2007-2009: Peter Schregle, */ +/* All rights reserved. */ +/* */ +/* This file is part of the Fixed Point Math Library. */ +/* */ +/* Redistribution of the Fixed Point Math Library and use in source and */ +/* binary forms, with or without modification, are permitted provided that */ +/* the following conditions are met: */ +/* 1. Redistributions of source code must retain the above copyright notice, */ +/* this list of conditions and the following disclaimer. */ +/* 2. Redistributions in binary form must reproduce the above copyright */ +/* notice, this list of conditions and the following disclaimer in the */ +/* documentation and/or other materials provided with the distribution. */ +/* 3. Neither the name of Peter Schregle nor the names of other contributors */ +/* may be used to endorse or promote products derived from this software */ +/* without specific prior written permission. */ +/* */ +/* THIS SOFTWARE IS PROVIDED BY PETER SCHREGLE AND CONTRIBUTORS 'AS IS' AND */ +/* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE */ +/* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + */ +/* ARE DISCLAIMED. IN NO EVENT SHALL PETER SCHREGLE OR CONTRIBUTORS BE LIABLE + */ +/* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + */ +/* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS */ +/* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) */ +/* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, */ +/* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN */ +/* ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ +/* POSSIBILITY OF SUCH DAMAGE. */ +/* */ +/*******************************************************************************/ + +/// \file +/// This file implements a fixed point class, which is hoped to be faster than +/// floating point on some architectures. +//! +//! Paul Dixon helped in making the class compatible with gcc. + +#ifndef __fixed_point_h__ +#define __fixed_point_h__ + +#include +#include +#include +#include +#include +#ifndef _USE_MATH_DEFINES +#define _USE_MATH_DEFINES +#define __FPML_DEFINED_USE_MATH_DEFINES__ +#endif +#include +#include + +/******************************************************************************/ +/* */ +/* fixed_point */ +/* */ +/******************************************************************************/ + +template < + /// The base type. Must be an integer type. + //! + //! If this is a signed type, the fixed_point number will behave signed too, + //! if this is an unsigned type, the fixed_point number will behave + //! unsigned. + typename B, + /// The integer part bit count. + unsigned char I, + /// The fractional part bit count. + unsigned char F = std::numeric_limits::digits - I> +/// A fixed point type. +//! +//! This type is designed to be a plug-in type to replace the floating point +//! types, such as float, double and long double. While it doesn't offer the +//! precision of these types, its operations are all implemented in integer +//! math, and it is therefore hoped that these operations are faster on non- +//! floating-point enabled hardware. +//! +//! The value uses 0/1 bits for the sign, I bits for the integer part and F bits +//! for the fractional part. +//! +//! Here is an example: a signed 8 bit 5:2 fixed_point type would have the +//! following layout: +//! +//! fixed_point +//! +//! sign integer part \ / fractional part +//! | | +//! +----+----+----+----+----+----+----+----+ +//! | S | I4 | I3 | I2 | I1 | I0 | F0 | F1 | +//! +----+----+----+----+----+----+----+----+ +//! +//! where S is the sign-bit, I0 to I4 is the integer part, and F0 to F1 is +//! the fractional part. The range of this type is from -32 to +31.75, the +//! fractional part can encode multiples of 0.25. +//! +//! The class only implements a few operators directly, the others are generated +//! via the ordered_field_operators, unit_steppable and shiftable templates. +//! +//! The ordered_field_operators template takes and generates (==>) the +//! following operators: +//! += ==> + (addable), +//! -= ==> - (subtractable), +//! *= ==> * (multipliable), +//! /= ==> / (dividable), +//! < ==> > , >=, <= (less_than_comparable), +//! == ==> != (equality_comparable). +//! +//! The unit_steppable template takes and generates (==>) the following +//! operators: +//! ++ ==> ++(int), preincrement versus postincrement, (incrementable), +//! -- ==> --(int), predecrement versus postdecrement, (decrementable). +//! +//! The shiftable template takes and generates the following operators: +//! >>= ==> >> (right_shiftable), +//! <<= ==> << (left_shiftable). +class fixed_point : boost::ordered_field_operators< + fixed_point, + boost::unit_steppable< + fixed_point, + boost::shiftable, size_t>>> { + // Only integer types qualify for base type. If this line triggers an error, + // the base type is not an integer type. Note: char does not qualify as an + // integer because of its uncertainty in definition. Use signed char or + // unsigned char to be explicit. + BOOST_CONCEPT_ASSERT((boost::Integer)); + + // Make sure that the bit counts are ok. If this line triggers an error, the + // sum of the bit counts for the fractional and integer parts do not match + // the bit count provided by the base type. The sign bit does not count. + BOOST_STATIC_ASSERT(I + F == std::numeric_limits::digits); + + /// Grant the fixed_point template access to private members. Types with + /// different template parameters are different types and without this + /// declaration they do not have access to private members. + friend class fixed_point; + + /// Grant the numeric_limits specialization for this fixed_point class + /// access to private members. + friend class std::numeric_limits>; + + template < + /// The power. + int P, + /// Make gcc happy. + typename T = void> + /// Calculate 2 to the power of F at compile time. + //! + //! The fixed_point class needs 2 to the power of P in several locations in + //! the code. However, the value depends on compile time constants only and + //! can therefore be calculated at compile time using this template + //! trickery. There is no need to call the function pow(2., P) at runtime to + //! calculate this value. + //! + //! The value is calculated by recursively instantiating the power2 template + //! with successively decrementing P. Finally, 2 to the power of 0 is + //! terminating the recursion and set to 1. + struct power2 { + static const long long value = 2 * power2

::value; + }; + + template < + /// Make gcc happy. + typename P> + /// Calculate 2 to the power of 0 at compile time. + //! + //! The fixed_point class needs 2 to the power of P in several locations in + //! the code. However, the value depends on compile time constants only and + //! can therefore be calculated at compile time using this template + //! trickery. There is no need to call the function pow(2., P) at runtime to + //! calculate this value. + //! + //! The value is calculated by recursively instantiating the power2 template + //! with successively decrementing P. Finally, 2 to the power of 0 is + //! terminating the recursion and set to 1. + struct power2<0, P> { + static const long long value = 1; + }; + + /// Initializing constructor. + //! + //! This constructor takes a value of type B and initializes the internal + //! representation of fixed_point with it. + fixed_point( + /// The internal representation to use for initialization. + B value, + /// This value is not important, it's just here to differentiate from + /// the other constructors that convert its values. + bool) + : value_(value) {} + + public: + /// The base type of this fixed_point class. + typedef B base_type; + + /// The integer part bit count. + static const unsigned char integer_bit_count = I; + + /// The fractional part bit count. + static const unsigned char fractional_bit_count = F; + + /// Default constructor. + //! + //! Just as with built-in types no initialization is done. The value is + //! undetermined after executing this constructor. + fixed_point() {} + + template < + /// The numeric type. Must be integer. + typename T> + /// Converting constructor. + //! + //! This constructor takes a numeric value of type T and converts it to + //! this fixed_point type. + fixed_point( + /// The value to convert. + T value) + : value_((B)value << F) { + BOOST_CONCEPT_ASSERT((boost::Integer)); + } + + /// Converting constructor. + //! + //! This constructor takes a numeric value of type bool and converts it to + //! this fixed_point type. + fixed_point( + /// The value to convert. + bool value) + : value_((B)(value * power2::value)) {} + + /// Converting constructor. + //! + //! This constructor takes a numeric value of type float and converts it to + //! this fixed_point type. + //! + //! The conversion is done by multiplication with 2^F and rounding to the + //! next integer. + fixed_point( + /// The value to convert. + float value) + : value_((B)(value * power2::value + (value >= 0 ? .5 : -.5))) {} + + /// Converting constructor. + //! + //! This constructor takes a numeric value of type double and converts it to + //! this fixed_point type. + fixed_point( + /// The value to convert. + double value) + : value_((B)(value * power2::value + (value >= 0 ? .5 : -.5))) {} + + /// Converting constructor. + //! + //! This constructor takes a numeric value of type long double and converts + //! it to this fixed_point type. + fixed_point( + /// The value to convert. + long double value) + : value_((B)(value * power2::value + (value >= 0 ? .5 : -.5))) {} + + /// Copy constructor. + fixed_point( + /// The right hand side. + fixed_point const& rhs) + : value_(rhs.value_) {} + + template < + /// The other integer part bit count. + unsigned char I2, + /// The other fractional part bit count. + unsigned char F2> + /// Converting copy constructor. + fixed_point( + /// The right hand side. + fixed_point const& rhs) + : value_(rhs.value_) { + if (I - I2 > 0) + value_ >>= I - I2; + if (I2 - I > 0) + value_ <<= I2 - I; + } + + /// Copy assignment operator. + fixed_point& operator=( + /// The right hand side. + fixed_point const& rhs) { + fixed_point temp(rhs); + swap(temp); + return *this; + } + + template < + /// The other integer part bit count. + unsigned char I2, + /// The other fractional part bit count. + unsigned char F2> + /// Converting copy assignment operator. + fixed_point& operator=( + /// The right hand side. + fixed_point const& rhs) { + fixed_point temp(rhs); + swap(temp); + return *this; + } + + /// Exchanges the elements of two fixed_point objects. + void swap( + /// The right hand side. + fixed_point& rhs) { + std::swap(value_, rhs.value_); + } + + /// Less than operator. + //! + //! Through the use of boost::less_than_comparable operator >, operator <= + //! and operator >= are also defined and implemented by calling this + //! operator. + //! + //! /return true if less than, false otherwise. + bool operator<( + /// Right hand side. + fixed_point const& rhs) const { + return value_ < rhs.value_; + } + + /// Equality operator. + //! + //! Through the use of boost::equality_comparable operator != is also + //! defined and implemented by calling this operator. + //! + //! /return true if equal, false otherwise. + bool operator==( + /// Right hand side. + fixed_point const& rhs) const { + return value_ == rhs.value_; + } + + /// Negation operator. + //! + //! /return true if equal to zero, false otherwise. + bool operator!() const { return value_ == 0; } + + /// Unary minus operator. + //! + //! For signed fixed-point types you can apply the unary minus operator to + //! get the additive inverse. For unsigned fixed-point types, this operation + //! is undefined. Also, shared with the integer base type B, the minimum + //! value representable by the type cannot be inverted, since it would yield + //! a positive value that is out of range and cannot be represented. + //! + //! /return The negative value. + fixed_point operator-() const { + fixed_point result; + result.value_ = -value_; + return result; + } + + /// Increment. + //! + //! Through the use of boost::unit_steppable operator ++(int) - + //! postincrement - is also defined and implemented by calling this + //! operator. + //! + //! /return A reference to this object. + fixed_point& operator++() { + value_ += power2::value; + return *this; + } + + /// Decrement. + //! + //! Through the use of boost::unit_steppable operator --(int) - + //! postdecrement - is also defined and implemented by calling this + //! operator. + //! + //! /return A reference to this object. + fixed_point& operator--() { + value_ -= power2::value; + return *this; + } + + /// Addition. + //! + //! Through the use of boost::additive operator+ is also defined and + //! implemented by calling this operator. + //! + //! /return A reference to this object. + fixed_point& operator+=( + /// Summand for addition. + fixed_point const& summand) { + value_ += summand.value_; + return *this; + } + + /// Subtraction. + //! + //! Through the use of boost::additive operator- is also defined and + //! implemented by calling this operator. + //! + //! /return A reference to this object. + fixed_point& operator-=( + /// Diminuend for subtraction. + fixed_point const& diminuend) { + value_ -= diminuend.value_; + return *this; + } + + private: + struct Error_promote_type_not_specialized_for_this_type {}; + + template < + /// Pick up unspecialized types. + typename T, + /// Make gcc happy. + typename U = void> + /// Multiplication and division of fixed_point numbers need type + /// promotion. + //! + //! When two 8 bit numbers are multiplied, a 16 bit result is produced. + //! When two 16 bit numbers are multiplied, a 32 bit result is produced. + //! When two 32 bit numbers are multiplied, a 64 bit result is produced. + //! Since the fixed_point class internally relies on integer + //! multiplication, we need type promotion. After the multiplication we + //! need to adjust the position of the decimal point by shifting the + //! temporary result to the right an appropriate number of bits. + //! However, if the temporary multiplication result is not kept in a big + //! enough variable, overflow errors will occur and lead to wrong + //! results. A similar promotion needs to be done to the divisor in the + //! case of division, but here the divisor needs to be shifted to the + //! left an appropriate number of bits. + //! + //! Unfortunately the integral_promotion class of the boost type_traits + //! library could not be used, since it does not provide a promotion + //! from int/unsigned int (32 bit) to long long/unsigned long long + //! (64 bit). However, this promotion is often needed, because it is + //! quite common to use a 32 bit base type for the fixed_point type. + //! + //! Therefore, the Fixed Point Math Library defines its own promotions + //! here in a set of private classes. + struct promote_type { +#ifdef _MSC_VER + typedef Error_promote_type_not_specialized_for_this_type type; +#endif // #ifdef _MSC_VER + }; + + template < + /// Make gcc happy. + typename U> + /// Promote signed char to signed short. + struct promote_type { + typedef signed short type; + }; + + template < + /// Make gcc happy. + typename U> + /// Promote unsigned char to unsigned short. + struct promote_type { + typedef unsigned short type; + }; + + template < + /// Make gcc happy. + typename U> + /// Promote signed short to signed int. + struct promote_type { + typedef signed int type; + }; + + template < + /// Make gcc happy. + typename U> + /// Promote unsigned short to unsigned int. + struct promote_type { + typedef unsigned int type; + }; + + template < + /// Make gcc happy. + typename U> + /// Promote signed int to signed long long. + struct promote_type { + typedef signed long long type; + }; + + template < + /// Make gcc happy. + typename U> + /// Promote unsigned int to unsigned long long. + struct promote_type { + typedef unsigned long long type; + }; + + // Requires GCC 4.1 or greater + platform support + template + struct promote_type { + typedef __uint128_t type; + }; + + template + struct promote_type { + typedef __int128_t type; + }; + + public: + /// Multiplication. + //! + //! Through the use of boost::multiplicative operator* is also defined and + //! implemented by calling this operator. + //! + //! /return A reference to this object. + fixed_point& operator*=( + /// Factor for mutliplication. + fixed_point const& factor) { + + value_ = + (static_cast< + typename fixed_point::template promote_type::type>( + value_) * + factor.value_) >> + F; + return *this; + } + + /// Division. + //! + //! Through the use of boost::multiplicative operator / is also defined and + //! implemented by calling this operator. + //! + //! /return A reference to this object. + fixed_point& operator/=( + /// Divisor for division. + fixed_point const& divisor) { + value_ = + (static_cast< + typename fixed_point::template promote_type::type>( + value_) + << F) / + divisor.value_; + return *this; + } + + /// Shift right. + //! + //! Through the use of boost::shiftable operator >> is also defined and + //! implemented by calling this operator. + //! + //! /return A reference to this object. + fixed_point& operator>>=( + /// Count of positions to shift. + size_t shift) { + // if (typeid(B) == typeid(int64_t) || typeid(B) == typeid(uint64_t)) + // _mm_srli_epi64(_mm_srli_si128(value_, shift/8), shift%8); + // else + value_ >>= shift; + return *this; + } + + /// Shift left. + //! + //! Through the use of boost::shiftable operator << is also defined and + //! implemented by calling this operator. + //! + //! /return A reference to this object. + fixed_point& operator<<=( + /// Count of positions to shift. + size_t shift) { + // if (typeid(B) == typeid(int64_t) || typeid(B) == typeid(uint64_t)) + // _mm_slli_epi64(_mm_slli_si128(value_, shift/8), shift%8); + // else + value_ <<= shift; + return *this; + } + + /// Convert to char. + //! + //! /return The value converted to char. + operator char() const { + return (char)(value_ >> F); + } + + /// Convert to signed char. + //! + //! /return The value converted to signed char. + operator signed char() const { + return (signed char)(value_ >> F); + } + + /// Convert to unsigned char. + //! + //! /return The value converted to unsigned char. + operator unsigned char() const { + return (unsigned char)(value_ >> F); + } + + /// Convert to short. + //! + //! /return The value converted to short. + operator short() const { + return (short)(value_ >> F); + } + + /// Convert to unsigned short. + //! + //! /return The value converted to unsigned short. + operator unsigned short() const { + return (unsigned short)(value_ >> F); + } + + /// Convert to int. + //! + //! /return The value converted to int. + operator int() const { + return (int)(value_ >> F); + } + + /// Convert to unsigned int. + //! + //! /return The value converted to unsigned int. + operator unsigned int() const { + return (unsigned int)(value_ >> F); + } + + /// Convert to long. + //! + //! /return The value converted to long. + operator long() const { + return (long)(value_ >> F); + } + + /// Convert to unsigned long. + //! + //! /return The value converted to unsigned long. + operator unsigned long() const { + return (unsigned long)(value_ >> F); + } + + /// Convert to long long. + //! + //! /return The value converted to long long. + operator long long() const { + return (long long)(value_ >> F); + } + + /// Convert to unsigned long long. + //! + //! /return The value converted to unsigned long long. + operator unsigned long long() const { + return (unsigned long long)(value_ >> F); + } + + /// Convert to a bool. + //! + //! /return The value converted to a bool. + operator bool() const { + return (bool)value_; + } + + /// Convert to a float. + //! + //! /return The value converted to a float. + operator float() const { + return (float)value_ / power2::value; + } + + /// Convert to a double. + //! + //! /return The value converted to a double. + operator double() const { + return (double)value_ / power2::value; + } + + /// Convert to a long double. + //! + //! /return The value converted to a long double. + operator long double() const { + return (long double)value_ / power2::value; + } + + /**************************************************************************/ + /* */ + /* fabs */ + /* */ + /**************************************************************************/ + + /// Calculates the absolute value. + //! + //! The fabs function computes the absolute value of its argument. + //! + //! /return The absolute value of the argument. + friend fixed_point fabs( + /// The argument to the function. + fixed_point x) { + return x < fixed_point(0) ? -x : x; + } + + /**************************************************************************/ + /* */ + /* ceil */ + /* */ + /**************************************************************************/ + + /// Calculates the ceiling value. + //! + //! The ceil function computes the smallest integral value not less than + //! its argument. + //! + //! /return The smallest integral value not less than the argument. + friend fixed_point ceil( + /// The argument to the function. + fixed_point x) { + fixed_point result; + result.value_ = x.value_ & ~(power2::value - 1); + return result + + fixed_point(x.value_ & (power2::value - 1) ? 1 : 0); + } + + /**************************************************************************/ + /* */ + /* floor */ + /* */ + /**************************************************************************/ + + /// Calculates the floor. + //! + //! The floor function computes the largest integral value not greater than + //! its argument. + //! + //! /return The largest integral value not greater than the argument. + friend fixed_point floor( + /// The argument to the function. + fixed_point x) { + fixed_point result; + result.value_ = x.value_ & ~(power2::value - 1); + return result; + } + + /**************************************************************************/ + /* */ + /* fmod */ + /* */ + /**************************************************************************/ + + /// Calculates the remainder. + //! + //! The fmod function computes the fixed point remainder of x/y. + //! + //! /return The fixed point remainder of x/y. + friend fixed_point fmod( + /// The argument to the function. + fixed_point x, + /// The argument to the function. + fixed_point y) { + fixed_point result; + result.value_ = x.value_ % y.value_; + return result; + } + + /**************************************************************************/ + /* */ + /* modf */ + /* */ + /**************************************************************************/ + + /// Split in integer and fraction parts. + //! + //! The modf function breaks the argument into integer and fraction parts, + //! each of which has the same sign as the argument. It stores the integer + //! part in the object pointed to by ptr. + //! + //! /return The signed fractional part of x/y. + friend fixed_point modf( + /// The argument to the function. + fixed_point x, + /// The pointer to the integer part. + fixed_point* ptr) { + fixed_point integer; + integer.value_ = x.value_ & ~(power2::value - 1); + *ptr = x < fixed_point(0) ? integer + fixed_point(1) + : integer; + + fixed_point fraction; + fraction.value_ = x.value_ & (power2::value - 1); + + return x < fixed_point(0) ? -fraction : fraction; + } + + /**************************************************************************/ + /* */ + /* exp */ + /* */ + /**************************************************************************/ + + /// Calculates the exponential. + //! + //! The function computes the exponential function of x. The algorithm uses + //! the identity e^(a+b) = e^a * e^b. + //! + //! /return The exponential of the argument. + friend fixed_point exp( + /// The argument to the exp function. + fixed_point x) { + // a[x] = exp( (1/2) ^ x ), x: [0 ... 31] + fixed_point a[] = { + 1.64872127070012814684865078781, 1.28402541668774148407342056806, + 1.13314845306682631682900722781, 1.06449445891785942956339059464, + 1.03174340749910267093874781528, 1.01574770858668574745853507208, + 1.00784309720644797769345355976, 1.00391388933834757344360960390, + 1.00195503359100281204651889805, 1.00097703949241653524284529261, + 1.00048840047869447312617362381, 1.00024417042974785493700523392, + 1.00012207776338377107650351967, 1.00006103701893304542177912060, + 1.00003051804379102429545128481, 1.00001525890547841394814004262, + 1.00000762942363515447174318433, 1.00000381470454159186605078771, + 1.00000190735045180306002872525, 1.00000095367477115374544678825, + 1.00000047683727188998079165439, 1.00000023841860752327418915867, + 1.00000011920929665620888994533, 1.00000005960464655174749969329, + 1.00000002980232283178452676169, 1.00000001490116130486995926397, + 1.00000000745058062467940380956, 1.00000000372529030540080797502, + 1.00000000186264515096568050830, 1.00000000093132257504915938475, + 1.00000000046566128741615947508}; + + fixed_point e(2.718281828459045); + + fixed_point y(1); + for (int i = F - 1; i >= 0; --i) { + if (!(x.value_ & 1 << i)) + y *= a[F - i - 1]; + } + + int x_int = (int)(floor(x)); + if (x_int < 0) { + for (int i = 1; i <= -x_int; ++i) + y /= e; + } else { + for (int i = 1; i <= x_int; ++i) + y *= e; + } + + return y; + } + + /**************************************************************************/ + /* */ + /* cos */ + /* */ + /**************************************************************************/ + + /// Calculates the cosine. + //! + //! The algorithm uses a MacLaurin series expansion. + //! + //! First the argument is reduced to be within the range -Pi .. +Pi. Then + //! the MacLaurin series is expanded. The argument reduction is problematic + //! since Pi cannot be represented exactly. The more rounds are reduced the + //! less significant is the argument (every reduction round makes a slight + //! error), to the extent that the reduced argument and consequently the + //! result are meaningless. + //! + //! The argument reduction uses one division. The series expansion uses 3 + //! additions and 4 multiplications. + //! + //! /return The cosine of the argument. + friend fixed_point cos( + /// The argument to the cos function. + fixed_point x) { + fixed_point x_ = fmod(x, fixed_point(M_PI * 2)); + if (x_ > fixed_point(M_PI)) + x_ -= fixed_point(M_PI * 2); + + fixed_point xx = x_ * x_; + + fixed_point y = + -xx * fixed_point(1. / (2 * 3 * 4 * 5 * 6)); + y += fixed_point(1. / (2 * 3 * 4)); + y *= xx; + y -= fixed_point(1. / (2)); + y *= xx; + y += fixed_point(1); + + return y; + } + + /**************************************************************************/ + /* */ + /* sin */ + /* */ + /**************************************************************************/ + + /// Calculates the sine. + //! + //! The algorithm uses a MacLaurin series expansion. + //! + //! First the argument is reduced to be within the range -Pi .. +Pi. Then + //! the MacLaurin series is expanded. The argument reduction is problematic + //! since Pi cannot be represented exactly. The more rounds are reduced the + //! less significant is the argument (every reduction round makes a slight + //! error), to the extent that the reduced argument and consequently the + //! result are meaningless. + //! + //! The argument reduction uses one division. The series expansion uses 3 + //! additions and 5 multiplications. + //! + //! /return The sine of the argument. + friend fixed_point sin( + /// The argument to the sin function. + fixed_point x) { + fixed_point x_ = fmod(x, fixed_point(M_PI * 2)); + if (x_ > fixed_point(M_PI)) + x_ -= fixed_point(M_PI * 2); + + fixed_point xx = x_ * x_; + + fixed_point y = + -xx * fixed_point(1. / (2 * 3 * 4 * 5 * 6 * 7)); + y += fixed_point(1. / (2 * 3 * 4 * 5)); + y *= xx; + y -= fixed_point(1. / (2 * 3)); + y *= xx; + y += fixed_point(1); + y *= x_; + + return y; + } + + /**************************************************************************/ + /* */ + /* sqrt */ + /* */ + /**************************************************************************/ + + /// Calculates the square root. + //! + //! The sqrt function computes the nonnegative square root of its argument. + //! A domain error results if the argument is negative. + //! + //! Calculates an approximation of the square root using an integer + //! algorithm. The algorithm is described in Wikipedia: + //! http://en.wikipedia.org/wiki/Methods_of_computing_square_roots + //! + //! The algorithm seems to have originated in a book on programming abaci by + //! Mr C. Woo. + //! + //! /return The square root of the argument. If the argument is negative, + //! the function returns 0. + friend fixed_point sqrt( + /// The argument to the square root function, a nonnegative fixed-point + /// value. + fixed_point x) { + if (x < fixed_point(0)) { + errno = EDOM; + return 0; + } + + typename fixed_point::template promote_type::type op = + static_cast< + typename fixed_point::template promote_type::type>( + x.value_) + << (I - 1); + typename fixed_point::template promote_type::type res = 0; + + // numeric limits for __int128 not specialized for digits, work around here + int promoted_type_digits; + if (typeid(B) == typeid(int64_t)) + promoted_type_digits = 127; + else if (typeid(B) == typeid(uint64_t)) + promoted_type_digits = 128; + else + promoted_type_digits = std::numeric_limits::template promote_type::type>::digits; + + typename fixed_point::template promote_type::type one = + (typename fixed_point::template promote_type::type)1 + << (promoted_type_digits - 1); + //(std::numeric_limits::template + // promote_type + // ::type>::digits - 1); + + while (one > op) + one >>= 2; + + while (one != 0) { + if (op >= res + one) { + op = op - (res + one); + res = res + (one << 1); + } + res >>= 1; + one >>= 2; + } + + fixed_point root; + root.value_ = static_cast(res); + return root; + } + + // private: + /// The value in fixed point format. + B value_; +}; + +template < + /// The input stream type. + typename S, + /// The base type of the fixed_point type. + typename B, + /// The integer part bit count of the fixed_point type. + unsigned char I, + /// The fractional part bit count of the fixed_point type. + unsigned char F> +/// Stream input operator. +//! +//! A value is first input to type double and then the read value is converted +//! to type fixed_point before it is returned. +//! +//! /return A reference to this input stream. +S& operator>>( + /// The input stream. + S& s, + /// A reference to the value to be read. + fixed_point& v) { + double value = 0.; + s >> value; + if (s) + v = value; + return s; +} + +template < + /// The output stream type. + typename S, + /// The base type of the fixed_point type. + typename B, + /// The integer part bit count of the fixed_point type. + unsigned char I, + /// The fractional part bit count of the fixed_point type. + unsigned char F> +/// Stream output operator. +//! +//! The fixed_point value is first converted to type double and then the output +//! operator for type double is called. +//! +//! /return A reference to this output stream. +S& operator<<( + /// The output stream. + S& s, + /// A const reference to the value to be written. + fixed_point const& v) { + double value = v; + s << value; + return s; +} + +namespace std { + +/******************************************************************************/ +/* */ +/* numeric_limits > */ +/* */ +/******************************************************************************/ + +template < + /// The base type of the fixed_point type. + typename B, + /// The integer part bit count of the fixed_point type. + unsigned char I, + /// The fractional part bit count of the fixed_point type. + unsigned char F> +class numeric_limits> { + public: + /// The fixed_point type. This numeric_limits specialization is specialized + /// for this type. + typedef fixed_point fp_type; + + /// Tests whether a type allows denormalized values. + //! + //! An enumeration value of type const float_denorm_style, indicating + //! whether the type allows denormalized values. The fixed_point class does + //! not have denormalized values. + //! + //! The member is always set to denorm_absent. + static const float_denorm_style has_denorm = denorm_absent; + + /// Tests whether loss of accuracy is detected as a denormalization loss + /// rather than as an inexact result. + //! + //! The fixed_point class does not have denormalized values. + //! + //! The member is always set to false. + static const bool has_denorm_loss = false; + + /// Tests whether a type has a representation for positive infinity. + //! + //! The fixed_point class does not have a representation for positive + //! infinity. + //! + //! The member is always set to false. + static const bool has_infinity = false; + + /// Tests whether a type has a representation for a quiet not a number + /// (NAN), which is nonsignaling. + //! + //! The fixed_point class does not have a quiet NAN. + //! + //! The member is always set to false. + static const bool has_quiet_NaN = false; + + /// Tests whether a type has a representation for signaling not a number + //! (NAN). + //! + //! The fixed_point class does not have a signaling NAN. + //! + //! The member is always set to false. + static const bool has_signaling_NaN = false; + + /// Tests if the set of values that a type may represent is finite. + //! + //! The fixed_point type has a bounded set of representable values. + //! + //! The member is always set to true. + static const bool is_bounded = true; + + /// Tests if the calculations done on a type are free of rounding errors. + //! + //! The fixed_point type is considered exact. + //! + //! The member is always set to true. + static const bool is_exact = true; + + /// Tests if a type conforms to IEC 559 standards. + //! + //! The fixed_point type does not conform to IEC 559 standards. + //! + //! The member is always set to false. + static const bool is_iec559 = false; + + /// Tests if a type has an integer representation. + //! + //! The fixed_point type behaves like a real number and thus has not + //! integer representation. + //! + //! The member is always set to false. + static const bool is_integer = false; + + /// Tests if a type has a modulo representation. + //! + //! A modulo representation is a representation where all results are + //! reduced modulo some value. The fixed_point class does not have a + //! modulo representation. + //! + //! The member is always set to false. + static const bool is_modulo = false; + + /// Tests if a type has a signed representation. + //! + //! The member stores true for a type that has a signed representation, + //! which is the case for all fixed_point types with a signed base type. + //! Otherwise it stores false. + static const bool is_signed = + std::numeric_limits::is_signed; + + /// Tests if a type has an explicit specialization defined in the template + /// class numeric_limits. + //! + //! The fixed_point class has an explicit specialization. + //! + //! The member is always set to true. + static const bool is_specialized = true; + + /// Tests whether a type can determine that a value is too small to + /// represent as a normalized value before rounding it. + //! + //! Types that can detect tinyness were included as an option with IEC 559 + //! floating-point representations and its implementation can affect some + //! results. + //! + //! The member is always set to false. + static const bool tinyness_before = false; + + /// Tests whether trapping that reports on arithmetic exceptions is + //! implemented for a type. + //! + //! The member is always set to false. + static const bool traps = false; + + /// Returns a value that describes the various methods that an + /// implementation can choose for rounding a real value to an integer + /// value. + //! + //! The member is always set to round_toward_zero. + static const float_round_style round_style = round_toward_zero; + + /// Returns the number of radix digits that the type can represent without + /// loss of precision. + //! + //! The member stores the number of radix digits that the type can represent + //! without change. + //! + //! The member is set to the template parameter I (number of integer bits). + static const int digits = I; + + /// Returns the number of decimal digits that the type can represent without + /// loss of precision. + //! + //! The member is set to the number of decimal digits that the type can + //! represent. + static const int digits10 = (int)(digits * 301. / 1000. + .5); + + static const int max_exponent = 0; + static const int max_exponent10 = 0; + static const int min_exponent = 0; + static const int min_exponent10 = 0; + static const int radix = 0; + + /// The minimum value of this type. + //! + //! /return The minimum value representable with this type. + static fp_type(min)() { + fp_type minimum; + minimum.value_ = (std::numeric_limits::min)(); + return minimum; + } + + /// The maximum value of this type. + //! + //! /return The maximum value representable with this type. + static fp_type(max)() { + fp_type maximum; + maximum.value_ = (std::numeric_limits::max)(); + return maximum; + } + + /// The function returns the difference between 1 and the smallest value + /// greater than 1 that is representable for the data type. + //! + //! /return The smallest effective increment from 1.0. + static fp_type epsilon() { + fp_type epsilon; + epsilon.value_ = 1; + return epsilon; + } + + /// Returns the maximum rounding error for the type. + //! + //! The maximum rounding error for the fixed_point type is 0.5. + //! + //! /return Always returns 0.5. + static fp_type round_error() { return (fp_type)(0.5); } + + static fp_type denorm_min() { return (fp_type)(0); } + + static fp_type infinity() { return (fp_type)(0); } + + static fp_type quiet_NaN() { return (fp_type)(0); } + + static fp_type signaling_NaN() { return (fp_type)(0); } +}; + +} // namespace std + +#ifdef __FPML_DEFINED_USE_MATH_DEFINES__ +#undef _USE_MATH_DEFINES +#undef __FPML_DEFINED_USE_MATH_DEFINES__ +#endif + +#endif // __fixed_point_h__ diff --git a/src/common/jlog.h b/src/common/jlog.h new file mode 100644 index 0000000..6096c9b --- /dev/null +++ b/src/common/jlog.h @@ -0,0 +1,265 @@ +#ifndef __JLOG_H +#define __JLOG_H + +//////////////////////// +// Logging Intrinsics // +//////////////////////// +#include + +#define RESET "\033[0m" +#define BLACK "\033[30m" /* Black */ +#define RED "\033[31m" /* Red */ +#define GREEN "\033[32m" /* Green */ +#define YELLOW "\033[33m" /* Yellow */ +#define BLUE "\033[34m" /* Blue */ +#define MAGENTA "\033[35m" /* Magenta */ +#define CYAN "\033[36m" /* Cyan */ +#define WHITE "\033[37m" /* White */ +#define BOLDBLACK "\033[1m\033[30m" /* Bold Black */ +#define BOLDRED "\033[1m\033[31m" /* Bold Red */ +#define BOLDGREEN "\033[1m\033[32m" /* Bold Green */ +#define BOLDYELLOW "\033[1m\033[33m" /* Bold Yellow */ +#define BOLDBLUE "\033[1m\033[34m" /* Bold Blue */ +#define BOLDMAGENTA "\033[1m\033[35m" /* Bold Magenta */ +#define BOLDCYAN "\033[1m\033[36m" /* Bold Cyan */ +#define BOLDWHITE "\033[1m\033[37m" /* Bold White */ + +#define MSG(...) \ + do { \ + printf(__VA_ARGS__); \ + } while (0) + +#define MSGF(enable, ...) \ + do { \ + if (_verbosity & enable) { \ + printf(MAGENTA "[" BOLDMAGENTA "%s" MAGENTA "] " RESET, #enable); \ + printf(__VA_ARGS__); \ + } \ + } while (0) + +#define MSGO(...) \ + do { \ + static uint8_t printed = 0; \ + if (!printed) { \ + printf(__VA_ARGS__); \ + printed = 1; \ + } \ + } while (0) + +#define DEBUG_MSG(...) \ + do { \ + printf(BLUE "(%s:%d - %s)" RESET "\n", __FILE__, __LINE__, __FUNCTION__); \ + printf(__VA_ARGS__); \ + } while (0) + +#define WARNING_MSG(...) \ + do { \ + printf(YELLOW "WARNING! (%s:%d - %s)" RESET "\n", __FILE__, __LINE__, \ + __FUNCTION__); \ + printf(__VA_ARGS__); \ + } while (0) + +#define ERROR_MSG(...) \ + do { \ + fprintf(stderr, RED "ERROR! (%s:%d - %s)" RESET "\n", __FILE__, __LINE__, \ + __FUNCTION__); \ + fprintf(stderr, __VA_ARGS__); \ + } while (0) + +///////////////////////// +// CSV Plotting Tools /// +///////////////////////// + +#include + +// TIC TOC only counts process time, it will not count sleeps or io waits +#define CLOCK(ID_) static clock_t tic##ID_; + +#define TIC(ID_) tic##ID_ = clock(); + +#define TOC(ID_) \ + do { \ + clock_t toc##ID_ = clock(); \ + if (tic##ID_ > toc##ID_) { \ + MSG(BLUE "%s" RESET " - " RED "Timer %s Overflowed" RESET "\n", \ + __FUNCTION__, #ID_); \ + } else { \ + MSG(BLUE "%s" RESET " - " GREEN "Timer %s: %g s" RESET "\n", \ + __FUNCTION__, #ID_, (toc##ID_ - tic##ID_) / (double)CLOCKS_PER_SEC); \ + } \ + } while (0) + +#define TOC_CSV_XY(ID_, X_) \ + do { \ + clock_t toc##ID_ = clock(); \ + if (tic##ID_ > toc##ID_) { \ + ERROR_MSG("SeNtInAl,error,%s,%s,%d,overflow\n", __FUNCTION__, #ID_, X_); \ + } else { \ + MSG("SeNtInAl,xy,%s,%s,%d,%g\n", __FUNCTION__, #ID_, X_, \ + (toc##ID_ - tic##ID_) / (double)CLOCKS_PER_SEC); \ + } \ + } while (0) + +// will stack ID's with same XLABEL_ on top +#define TOC_CSV_BAR(ID_, XLABEL_) \ + do { \ + clock_t toc##ID_ = clock(); \ + if (tic##ID_ > toc##ID_) { \ + ERROR_MSG("SeNtInAl,error,%s,%s,%s,overflow\n", __FUNCTION__, #ID_, \ + XLABEL_); \ + } else { \ + MSG("SeNtInAl,bar,%s,%s,%s,%g\n", __FUNCTION__, #ID_, XLABEL_, \ + (toc##ID_ - tic##ID_) / (double)CLOCKS_PER_SEC); \ + } \ + } while (0) + +#define TOC_CSV_GROUPED_BAR(ID_, XGROUP_) \ + do { \ + clock_t toc##ID_ = clock(); \ + if (tic##ID_ > toc##ID_) { \ + ERROR_MSG("SeNtInAl,error,%s,%s,%s,overflow\n", __FUNCTION__, #ID_, \ + XGROUP_); \ + } else { \ + MSG("SeNtInAl,bar,%s,%s,%s,%g\n", __FUNCTION__, #ID_, XGROUP_, \ + (toc##ID_ - tic##ID_) / (double)CLOCKS_PER_SEC); \ + } \ + } while (0) + +#define TOC_CSV_BOX(ID_) \ + do { \ + clock_t toc##ID_ = clock(); \ + if (tic##ID_ > toc##ID_) { \ + ERROR_MSG("SeNtInAl,error,%s,%s,no-x,overflow\n", __FUNCTION__, #ID_); \ + } else { \ + MSG("SeNtInAl,box,%s,%s,no-x,%g\n", __FUNCTION__, #ID_, \ + (toc##ID_ - tic##ID_) / (double)CLOCKS_PER_SEC); \ + } \ + } while (0) + +// Wall clock measures absolute time + +#include +#include +#include +using namespace std::chrono; + +#define WALL_CLOCK(ID_) static time_point tic##ID_; + +#define WALL_TIC(ID_) tic##ID_ = high_resolution_clock::now(); + +#define WALL_TOC(ID_) \ + do { \ + time_point toc##ID_ = high_resolution_clock::now(); \ + if (tic##ID_ > toc##ID_) { \ + MSG(BLUE "%s" RESET " - " RED "Timer %s Overflowed" RESET "\n", \ + __FUNCTION__, #ID_); \ + } else { \ + MSG(BLUE "%s" RESET " - " GREEN "Timer %s: %g s" RESET "\n", \ + __FUNCTION__, #ID_, \ + std::chrono::duration_cast(toc##ID_ - \ + tic##ID_) \ + .count() / \ + 1000000.0); \ + } \ + } while (0) + +#define WALL_TOC_CSV_XY(ID_, X_) \ + do { \ + time_point toc##ID_ = high_resolution_clock::now(); \ + if (tic##ID_ > toc##ID_) { \ + ERROR_MSG("SeNtInAl,error,%s,%s,%d,overflow\n", __FUNCTION__, #ID_, X_); \ + } else { \ + MSG("SeNtInAl,xy,%s,%s,%d,%g\n", __FUNCTION__, #ID_, X_, \ + std::chrono::duration_cast(toc##ID_ - \ + tic##ID_) \ + .count() / \ + 1000000.0); \ + } \ + } while (0) + +// will stack ID's with same XLABEL on top +#define WALL_TOC_CSV_BAR(ID_, XLABEL_) \ + do { \ + time_point toc##ID_ = high_resolution_clock::now(); \ + if (tic##ID_ > toc##ID_) { \ + ERROR_MSG("SeNtInAl,error,%s,%s,%s,overflow\n", __FUNCTION__, #ID_, \ + XLABEL_); \ + } else { \ + MSG("SeNtInAl,bar,%s,%s,%s,%g\n", __FUNCTION__, #ID_, XLABEL_, \ + std::chrono::duration_cast(toc##ID_ - \ + tic##ID_) \ + .count() / \ + 1000000.0); \ + } \ + } while (0) + +#define WALL_TOC_CSV_GROUPED_BAR(ID_, XGROUP_) \ + do { \ + time_point toc##ID_ = high_resolution_clock::now(); \ + if (tic##ID_ > toc##ID_) { \ + ERROR_MSG("SeNtInAl,error,%s,%s,%s,overflow\n", __FUNCTION__, #ID_, \ + XGROUP_); \ + } else { \ + MSG("SeNtInAl,grouped_bar,%s,%s,%s,%g\n", __FUNCTION__, #ID_, XGROUP_, \ + std::chrono::duration_cast(toc##ID_ - \ + tic##ID_) \ + .count() / \ + 1000000.0); \ + } \ + } while (0) + +#define WALL_TOC_CSV_BOX(ID_) \ + do { \ + static time_point toc##ID_ = \ + high_resolution_clock::now(); \ + if (tic##ID_ > toc##ID_) { \ + ERROR_MSG("SeNtInAl,error,%s,%s,no-x,overflow\n", __FUNCTION__, #ID_); \ + } else { \ + MSG("SeNtInAl,box,%s,%s,no-x,%g\n", __FUNCTION__, #ID_, \ + std::chrono::duration_cast(toc##ID_ - \ + tic##ID_) \ + .count() / \ + 1000000.0); \ + } \ + } while (0) + +// Counter +#define COUNTER(ID_) static uint32_t count##ID_ = 0; + +#define COUNTER_RESET(ID_, VAL_) \ + do { \ + count##ID_ = VAL_; \ + } while (0) + +#define COUNTER_INC(ID_, INC_BY_) \ + do { \ + static uint32_t tmp = count##ID_ + INC_BY_; \ + if (count##ID_ > tmp) { \ + printf("SeNtInAl,error,%s,%s,no-x,overflow\n", __FUNCTION__, #ID_); \ + } \ + count##ID_ = tmp; \ + } while (0) + +#define COUNTER_PRINT(ID_, X_) \ + do { \ + MSG(BLUE "%s" RESET " - " GREEN "Counter %s: \t%d,\t%d" RESET "\n", \ + __FUNCTION__, #ID_, X_, count##ID_); \ + } while (0) + +#define COUNTER_CSV_XY(ID_, X_) \ + do { \ + printf("SeNtInAl,xy,%s,%s,%d,%d\n", __FUNCTION__, #ID_, X_, count##ID_); \ + } while (0) + +#define COUNTER_CSV_BOX(ID_) \ + do { \ + printf("SeNtInAl,box,%s,%s,no-x,%d\n", __FUNCTION__, #ID_, count##ID_); \ + } while (0) + +// Histogram +#define HISTOCSV(ID_, VAL_) \ + do { \ + printf("SeNtInAl,histogram,%s,%s,no-x,%d\n", __FUNCTION__, #ID_, VAL_); \ + } while (0) + +#endif diff --git a/src/common/printutil.h b/src/common/printutil.h new file mode 100644 index 0000000..c07180f --- /dev/null +++ b/src/common/printutil.h @@ -0,0 +1,56 @@ +#pragma once +#include +#include +#include +#include + +template +void printVector(std::string s, T* v, int size) { + printf("%s\n", s.c_str()); + for (int i = 0; i < size; i++) { + if (printints) + printf("%d", *(int*)&v[i]); + else { + // if (typeid(T) == typeid(float) || typeid(T) == typeid(double)) { + // printf("%.8lf ", v[i]); + // } else if (typeid(T) == typeid(fixed_point)) { + // //std::cout.precision(8); + // std::cout << v[i] << " "; + // } else if (typeid(T) == typeid(fixed_point)) { + // //std::cout.precision(8); + // std::cout << v[i] << " "; + // } else { + // std::cout << v[i] << " "; + // } + std::cout << v[i] << " "; + } + } + printf("\n\n"); +} + +template +void printMatrix(std::string s, T** a, int rows, int cols) { + printf("%s\n", s.c_str()); + for (int i = 0; i < rows; i++) { + for (int j = 0; j < cols; j++) { + if (printints) + printf("%d", *(int*)&a[i][j]); + else { + // if (typeid(T) == typeid(float) || typeid(T) == typeid(double)) { + // printf("%.8lf ", a[i][j]); + // } else if (typeid(T) == typeid(fixed_point)) { + // //std::cout.precision(8); + // std::cout << a[i][j] << " "; + // } else if (typeid(T) == typeid(fixed_point)) { + // //std::cout.precision(8); + // std::cout << a[i][j] << " "; + // } else { + // std::cout << a[i][j] << " "; + // } + std::cout << a[i][j] << " "; + } + } + printf("\n"); + } + printf("\n"); +} diff --git a/src/common/privacyconf.h b/src/common/privacyconf.h new file mode 100644 index 0000000..d42f21c --- /dev/null +++ b/src/common/privacyconf.h @@ -0,0 +1,56 @@ +#pragma once +#include + +// Algorithm control flow +// DO = data oblivious running a maximal, fixed upper bound of iterations +// of gradient descent and svd. +#define PPL_FLOW_DO 2 +// Loop Leak is where the number of iterations is revealed to offload servers. +#define PPL_FLOW_LOOP_LEAK 3 +// Single shot localization (SiSL) runs a single optimization iteration. +// It also uses a data oblivious SVD. +#define PPL_FLOW_SiSL 4 +// Fakes localization at the same rate as plaintext for power testing +// on the raspberry pi snail. +#define PPL_FLOW_POWER_TESTING 5 + +#define PPL_FLOW PPL_FLOW_SiSL + +const float GT_MIN_ER = + 1; // (pose - ground truth) L2 norm less than this considered correct + +// Works for snail and hoffs test points for Loop Leak and SiSL +// but broken on aby sisl ctest lm +// const float JACOB_EPSILON = 0x0.0000bfp0; +// const float MIN_ER = 1e-2; +// below works for all tests +const float JACOB_EPSILON = 0x0.0000c3p0; // 25 mpc iterations +const float MIN_ER = 1e-2; + +// ETH3D dataset +// const float JACOB_EPSILON = 0x0.000050p0; +// const float MIN_ER = 5e-2; + +// const float JACOB_EPSILON=0x0.000007p0; // charuco cnail +// const float JACOB_EPSILON = 1e-5; // Hoff's origional parameters + +// maximum number of iterations +// from opencv calibration.cpp - cvFindExtrinsicCameraParams2() = 20 +const int GN_MAX_ITR = 30; +const int LM_MAX_ITR = 30; +const float LM_LAMBDA_INIT = 1e-3; +const float LM_LAMBDA_MAX = 1e5; // opencv default, works with hoff and ETH3D +// const float LM_LAMBDA_MAX = 1e2; // snail with aruco markers (may not be necessary) +const float LM_LAMBDA_MIN = 1e-5; + +// Debugging tools +#define DBG_FLOW 0x01 +#define DBG_PROJECT 0x02 +#define DBG_JACOB 0x04 +#define DBG_POSE_UPDATE 0x08 +#define DBG_ER 0x10 +#define DBG_ARGS 0x20 +#define DBG_LAMBDA 0x40 +const uint32_t _verbosity = 0x00; +const bool printints = + false; // used for ABY, their debug prints all floats as ints diff --git a/src/emp-float-server/CMakeLists.txt b/src/emp-float-server/CMakeLists.txt new file mode 100644 index 0000000..4c3b012 --- /dev/null +++ b/src/emp-float-server/CMakeLists.txt @@ -0,0 +1,22 @@ +cmake_minimum_required(VERSION 3.12) +file(GLOB emp_float_server_src CONFIGURE_DEPENDS "*.h" "*.cpp") +list(FILTER emp_float_server_src EXCLUDE REGEX ".*server-gn\\.cpp$") +list(FILTER emp_float_server_src EXCLUDE REGEX ".*server-lm\\.cpp$") + +# required for emp +set(CMAKE_C_FLAGS "-pthread -Wall -march=native -O3 -maes -mrdseed") +set(CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} -std=c++17") +set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -ggdb -fno-omit-frame-pointer") +set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}") + + +add_library(EmpFloatLocalization SHARED ${emp_float_server_src} ) +target_include_directories(EmpFloatLocalization PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/../common/") +target_compile_options(EmpFloatLocalization PRIVATE "-Wall" "-Wextra" "-fPIC") +target_link_libraries(EmpFloatLocalization ${EMP-SH2PC_LIBRARIES} ${OpenCV_LIBS}) + + +add_executable(lm_emp_float_server "${CMAKE_CURRENT_SOURCE_DIR}/server-lm.cpp" ${emp_float_server_src}) +target_include_directories(lm_emp_float_server PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/../common/") +target_compile_options(lm_emp_float_server PRIVATE "-Wall" "-Wextra") +target_link_libraries(lm_emp_float_server ${EMP-SH2PC_LIBRARIES} ${OpenCV_LIBS}) diff --git a/src/emp-float-server/fixed_point_emp.h b/src/emp-float-server/fixed_point_emp.h new file mode 100644 index 0000000..e878307 --- /dev/null +++ b/src/emp-float-server/fixed_point_emp.h @@ -0,0 +1,798 @@ +#pragma once +#include +#include +#include +#include +#include +#ifndef _USE_MATH_DEFINES +#define _USE_MATH_DEFINES +#define __FPML_DEFINED_USE_MATH_DEFINES__ +#endif +#include +#include + +/******************************************************************************/ +/* */ +/* fixed_point_emp */ +/* */ +/******************************************************************************/ + +template < + /// The integer part bit count. + unsigned char I, + /// The fractional part bit count. + unsigned char F> +/// A fixed point type. +//! +//! This type is designed to be a plug-in type to replace the floating point +//! types, such as float, double and long double. While it doesn't offer the +//! precision of these types, its operations are all implemented in integer +//! math, and it is therefore hoped that these operations are faster on non- +//! floating-point enabled hardware. +//! +//! The value uses 0/1 bits for the sign, I bits for the integer part and F bits +//! for the fractional part. +//! +//! Here is an example: a signed 8 bit 5:2 fixed_point_emp type would have the +//! following layout: +//! +//! fixed_point_emp +//! +//! sign integer part \ / fractional part +//! | | +//! +----+----+----+----+----+----+----+----+ +//! | S | I4 | I3 | I2 | I1 | I0 | F0 | F1 | +//! +----+----+----+----+----+----+----+----+ +//! +//! where S is the sign-bit, I0 to I4 is the integer part, and F0 to F1 is +//! the fractional part. The range of this type is from -32 to +31.75, the +//! fractional part can encode multiples of 0.25. +//! +//! The class only implements a few operators directly, the others are generated +//! via the ordered_field_operators, unit_steppable and shiftable templates. +//! +//! The ordered_field_operators template takes and generates (==>) the +//! following operators: +//! += ==> + (addable), +//! -= ==> - (subtractable), +//! *= ==> * (multipliable), +//! /= ==> / (dividable), +//! < ==> > , >=, <= (less_than_comparable), +//! == ==> != (equality_comparable). +//! +//! The unit_steppable template takes and generates (==>) the following +//! operators: +//! ++ ==> ++(int), preincrement versus postincrement, (incrementable), +//! -- ==> --(int), predecrement versus postdecrement, (decrementable). +//! +//! The shiftable template takes and generates the following operators: +//! >>= ==> >> (right_shiftable), +//! <<= ==> << (left_shiftable). +class fixed_point_emp : public Swappable>, + public Comparable> { + /// Grant the fixed_point_emp template access to private members. Types with + /// different template parameters are different types and without this + /// declaration they do not have access to private members. + friend class fixed_point_emp; + + template < + /// The power. + int P, + /// Make gcc happy. + typename T = void> + /// Calculate 2 to the power of F at compile time. + //! + //! The fixed_point_emp class needs 2 to the power of P in several locations + //! in the code. However, the value depends on compile time constants only and + //! can therefore be calculated at compile time using this template + //! trickery. There is no need to call the function pow(2., P) at runtime to + //! calculate this value. + //! + //! The value is calculated by recursively instantiating the power2 template + //! with successively decrementing P. Finally, 2 to the power of 0 is + //! terminating the recursion and set to 1. + struct power2 { + static const long long value = 2 * power2

::value; + }; + + template < + /// Make gcc happy. + typename P> + /// Calculate 2 to the power of 0 at compile time. + //! + //! The fixed_point_emp class needs 2 to the power of P in several locations + //! in the code. However, the value depends on compile time constants only and + //! can therefore be calculated at compile time using this template + //! trickery. There is no need to call the function pow(2., P) at runtime to + //! calculate this value. + //! + //! The value is calculated by recursively instantiating the power2 template + //! with successively decrementing P. Finally, 2 to the power of 0 is + //! terminating the recursion and set to 1. + struct power2<0, P> { + static const long long value = 1; + }; + + /// Initializing constructor. + //! + //! This constructor takes a value of type Integer and initializes the + //! internal representation of fixed_point_emp with it. + fixed_point_emp( + /// The internal representation to use for initialization. + Integer value, + /// This value is not important, it's just here to differentiate from + /// the other constructors that convert its values. + bool) + : value_(value) {} + + public: + /// The base type of this fixed_point_emp class. + typedef Integer base_type; + + /// The integer part bit count. + static const unsigned char integer_bit_count = I; + + /// The fractional part bit count. + static const unsigned char fractional_bit_count = F; + + /// Default constructor. + //! + //! Just as with built-in types no initialization is done. The value is + //! undetermined after executing this constructor. + fixed_point_emp() {} + + template < + /// The numeric type. Must be integer. + typename T> + /// Converting constructor. + //! + //! This constructor takes a numeric value of type T and converts it to + //! this fixed_point_emp type. + fixed_point_emp( + /// The value to convert. + T value, + // Who owns the value + int party = PUBLIC) + : value_(I + F, value << F, party) {} + + /// Converting constructor. + //! + //! This constructor takes a numeric value of type bool and converts it to + //! this fixed_point_emp type. + fixed_point_emp( + /// The value to convert. + bool value, + // Who owns the value + int party = PUBLIC) + : value_(I + F, value * power2::value, party) {} + + /// Converting constructor. + //! + //! This constructor takes a numeric value of type float and converts it to + //! this fixed_point_emp type. + //! + //! The conversion is done by multiplication with 2^F and rounding to the + //! next integer. + fixed_point_emp( + /// The value to convert. + float value, + // Who owns the value + int party = PUBLIC) + : value_(I + F, value * power2::value + (value >= 0 ? .5 : -.5), + party) {} + + /// Converting constructor. + //! + //! This constructor takes a numeric value of type double and converts it to + //! this fixed_point_emp type. + fixed_point_emp( + /// The value to convert. + double value, + // Who owns the value + int party = PUBLIC) + : value_(I + F, value * power2::value + (value >= 0 ? .5 : -.5), + party) {} + + /// Copy constructor. + fixed_point_emp(fixed_point_emp const& rhs) : value_(rhs.value_) {} + + /// Copy constructor. + fixed_point_emp(Integer const& rhs) : value_(rhs) {} + + template < + /// The other integer part bit count. + unsigned char I2, + /// The other fractional part bit count. + unsigned char F2> + /// Converting copy constructor. + fixed_point_emp( + /// The right hand side. + fixed_point_emp const& rhs) + : value_(rhs.value_) { + if (I - I2 > 0) + value_ = value_ >> (I - I2); + if (I2 - I > 0) + value_ = value_ << (I2 - I); + } + + /// Copy assignment operator. + fixed_point_emp& operator=( + /// The right hand side. + fixed_point_emp const& rhs) { + fixed_point_emp temp(rhs); + swap(temp); + return *this; + } + + template < + /// The other integer part bit count. + unsigned char I2, + /// The other fractional part bit count. + unsigned char F2> + /// Converting copy assignment operator. + fixed_point_emp& operator=( + /// The right hand side. + fixed_point_emp const& rhs) { + fixed_point_emp temp(rhs); + swap(temp); + return *this; + } + + /// Exchanges the elements of two fixed_point_emp objects. + void swap( + /// The right hand side. + fixed_point_emp& rhs) { + std::swap(value_, rhs.value_); + } + + // Comparable + Bit geq(const fixed_point_emp& rhs) const { + return value_.geq(rhs.value_); + } + + Bit equal(const fixed_point_emp& rhs) const { + return value_.equal(rhs.value_); + } + + // Swappable + fixed_point_emp select(const Bit& sel, + const fixed_point_emp& rhs) const { + return value_.select(sel, rhs.value_); + } + + fixed_point_emp operator^(const fixed_point_emp& rhs) const { + return value_ ^ rhs.value_; + } + + fixed_point_emp operator^=(const fixed_point_emp& rhs) { + return value_ ^= rhs.value_; + } + + /// Negation operator. + //! + //! /return true if equal to zero, false otherwise. + Bit operator!() const { return value_ == Integer(value_.size(), 0, PUBLIC); } + + /// Unary minus operator. + fixed_point_emp operator-() const { + fixed_point_emp result; + result.value_ = -value_; + return result; + } + + /// Increment. + // fixed_point_emp & operator ++() + //{ + // value_ = value_ + Integer(value_.size(), power2::value, PUBLIC); + // return *this; + // } + + ///// Decrement. + // fixed_point_emp & operator --() + //{ + // value_ = value_ - Integer(value_.size(), power2::value, PUBLIC); + // return *this; + // } + + /// Addition. + fixed_point_emp& operator+=(fixed_point_emp const& summand) { + assert(value_.size() == summand.value_.size()); + size_t size = I + F; + + value_.resize(size * 2, true); + vector summand_bits = summand.value_.bits; + summand_bits.resize(size * 2, true); + add_full(value_.bits.data(), nullptr, value_.bits.data(), + summand_bits.data(), nullptr, size * 2); + value_.resize(size); + + return *this; + } + fixed_point_emp operator+(fixed_point_emp const& summand) { + fixed_point_emp res(*this); + res += summand; + return res; + } + + /// Subtraction. + fixed_point_emp& operator-=(fixed_point_emp const& diminuend) { + assert(value_.size() == diminuend.value_.size()); + size_t size = I + F; + value_.resize(size * 2, true); + vector diminuend_bits = diminuend.value_.bits; + diminuend_bits.resize(size * 2, true); + sub_full(value_.bits.data(), nullptr, value_.bits.data(), + diminuend_bits.data(), nullptr, size * 2); + // Bit MSB = value_.bits[(size*2)-1]; + value_.resize(size); + // value_.bits[size-1] = MSB; + return *this; + } + fixed_point_emp operator-(fixed_point_emp const& diminuend) { + fixed_point_emp res(*this); + res -= diminuend; + return res; + } + + /// Multiplication. + fixed_point_emp& operator*=(fixed_point_emp const& factor) { + int size = (I + F); + + // std::cout << value_.reveal() << endl; + // std::cout << "*" << endl; + // std::cout << factor.value_.reveal() << endl; + // + // //// do mult by hand instead of resize() then using * operator + // //// to take advantage of operators only being size bits (not + // 2*size). Bit * sum = new Bit[size*2]; Bit * temp = new Bit[size]; + // for(int i = 0; i < size*2; ++i) + // sum[i]=false; + // for(int i=0;i vsum(sum, sum + (2*size)); + // Integer isum(vsum); + // std::cout << isum.reveal() << endl; + // } + // memcpy(value_.bits.data(), sum+F, sizeof(Bit)*size); + // delete[] sum; + // delete[] temp; + // return *this; + + Integer fc(factor.value_); + value_.resize(size * 2, true); + fc.resize(size * 2, true); + // value_ = (value_ * fc) >> F; + value_ = value_ * fc; + // right shift with sign extend in place + for (int i = F; i < size; ++i) + value_.bits[i - F] = value_.bits[i]; + for (int i = size - F; i < size; ++i) + value_.bits[i] = value_.bits[size - 1]; + value_.resize(size); + return *this; + } + fixed_point_emp operator*(fixed_point_emp const& factor) { + fixed_point_emp res(*this); + res *= factor; + return res; + } + + /// Division. + fixed_point_emp& operator/=( + fixed_point_emp const& divisor) // divisor cannot be const + { + Integer dc(divisor.value_); // non const copy + value_.resize((I + F) * 2, true); + dc.resize((I + F) * 2, true); + // fixed_point_emp res((value_ << F) / dc); + value_ = (value_ << F) / dc; + value_.resize(I + F, true); + return *this; + } + fixed_point_emp operator/( + fixed_point_emp const& divisor) // divisor cannot be const + { + fixed_point_emp res(*this); + res /= divisor; + return res; + } + + /// Shift right. + fixed_point_emp& operator>>=( + /// Count of positions to shift. + size_t shift) { + value_ = value_ >> shift; + return *this; + } + fixed_point_emp& operator>>( + /// Count of positions to shift. + size_t shift) { + fixed_point_emp res(*this); + res >>= shift; + return res; + } + + /// Shift left. + fixed_point_emp& operator<<=( + /// Count of positions to shift. + size_t shift) { + value_ = value_ << shift; + return *this; + } + fixed_point_emp& operator<<( + /// Count of positions to shift. + size_t shift) { + fixed_point_emp res(*this); + res <<= shift; + return res; + } + + double reveal(int party = PUBLIC) const { + if (I + F == 32) { + int32_t ct = value_.reveal(party); + return ((double)ct) / power2::value; + } else if (I + F == 64) { + int64_t ct = value_.reveal(party); + return ((double)ct) / power2::value; + } + } + + /**************************************************************************/ + /* */ + /* fabs */ + /* */ + /**************************************************************************/ + + /// Calculates the absolute value. + //! + //! The fabs function computes the absolute value of its argument. + //! + //! /return The absolute value of the argument. + friend fixed_point_emp fabs( + /// The argument to the function. + fixed_point_emp x) { + return x.value_.abs(); + } + + /**************************************************************************/ + /* */ + /* fmod */ + /* */ + /**************************************************************************/ + + /// Calculates the remainder. + //! + //! The fmod function computes the fixed point remainder of x/y. + //! + //! /return The fixed point remainder of x/y. + friend fixed_point_emp fmod( + /// The argument to the function. + fixed_point_emp x, + /// The argument to the function. + fixed_point_emp y) { + fixed_point_emp result; + result.value_ = x.value_ % y.value_; + return result; + } + + /**************************************************************************/ + /* */ + /* modf */ + /* */ + /**************************************************************************/ + + ///// Split in integer and fraction parts. + ////! + ////! The modf function breaks the argument into integer and fraction parts, + ////! each of which has the same sign as the argument. It stores the integer + ////! part in the object pointed to by ptr. + ////! + ////! /return The signed fractional part of x/y. + // friend fixed_point_emp modf( + // /// The argument to the function. + // fixed_point_emp x, + // /// The pointer to the integer part. + // fixed_point_emp * ptr) + //{ + // fixed_point_emp integer; + // integer.value_ = x.value_ & ~(power2::value-1); + // *ptr = x < fixed_point_emp(0) ? + // integer + fixed_point_emp(1) : integer; + + // fixed_point_emp fraction; + // fraction.value_ = x.value_ & (power2::value-1); + + // return x < fixed_point_emp(0) ? -fraction : fraction; + //} + + /**************************************************************************/ + /* */ + /* exp */ + /* */ + /**************************************************************************/ + + ///// Calculates the exponential. + ////! + ////! The function computes the exponential function of x. The algorithm uses + ////! the identity e^(a+b) = e^a * e^b. + ////! + ////! /return The exponential of the argument. + // friend fixed_point_emp exp( + // /// The argument to the exp function. + // fixed_point_emp x) + //{ + // // a[x] = exp( (1/2) ^ x ), x: [0 ... 31] + // fixed_point_emp a[] = { + // 1.64872127070012814684865078781, + // 1.28402541668774148407342056806, + // 1.13314845306682631682900722781, + // 1.06449445891785942956339059464, + // 1.03174340749910267093874781528, + // 1.01574770858668574745853507208, + // 1.00784309720644797769345355976, + // 1.00391388933834757344360960390, + // 1.00195503359100281204651889805, + // 1.00097703949241653524284529261, + // 1.00048840047869447312617362381, + // 1.00024417042974785493700523392, + // 1.00012207776338377107650351967, + // 1.00006103701893304542177912060, + // 1.00003051804379102429545128481, + // 1.00001525890547841394814004262, + // 1.00000762942363515447174318433, + // 1.00000381470454159186605078771, + // 1.00000190735045180306002872525, + // 1.00000095367477115374544678825, + // 1.00000047683727188998079165439, + // 1.00000023841860752327418915867, + // 1.00000011920929665620888994533, + // 1.00000005960464655174749969329, + // 1.00000002980232283178452676169, + // 1.00000001490116130486995926397, + // 1.00000000745058062467940380956, + // 1.00000000372529030540080797502, + // 1.00000000186264515096568050830, + // 1.00000000093132257504915938475, + // 1.00000000046566128741615947508 }; + + // fixed_point_emp e(2.718281828459045); + + // fixed_point_emp y(1); + // for (int i=F-1; i>=0; --i) + // { + // if (!(x.value_ & 1< cos( + /// The argument to the cos function. + fixed_point_emp x) { + fixed_point_emp x_ = fmod(x, fixed_point_emp(M_PI * 2)); + + // if (x_ > fixed_point_emp(M_PI)) + // x_ -= fixed_point_emp(M_PI * 2); + x_ = x_.If(x_ > fixed_point_emp(M_PI), + x_ - fixed_point_emp(M_PI * 2)); + + fixed_point_emp xx = x_ * x_; + + fixed_point_emp y = + -xx * fixed_point_emp(1. / (2 * 3 * 4 * 5 * 6)); + y += fixed_point_emp(1. / (2 * 3 * 4)); + y *= xx; + y -= fixed_point_emp(1. / (2)); + y *= xx; + y += fixed_point_emp(1); + + return y; + } + + /**************************************************************************/ + /* */ + /* sin */ + /* */ + /**************************************************************************/ + + /// Calculates the sine. + //! + //! The algorithm uses a MacLaurin series expansion. + //! + //! First the argument is reduced to be within the range -Pi .. +Pi. Then + //! the MacLaurin series is expanded. The argument reduction is problematic + //! since Pi cannot be represented exactly. The more rounds are reduced the + //! less significant is the argument (every reduction round makes a slight + //! error), to the extent that the reduced argument and consequently the + //! result are meaningless. + //! + //! The argument reduction uses one division. The series expansion uses 3 + //! additions and 5 multiplications. + //! + //! /return The sine of the argument. + friend fixed_point_emp sin( + /// The argument to the sin function. + fixed_point_emp x) { + fixed_point_emp x_ = fmod(x, fixed_point_emp(M_PI * 2)); + // if (x_ > fixed_point_emp(M_PI)) + // x_ -= fixed_point_emp(M_PI * 2); + x_ = x_.If(x_ > fixed_point_emp(M_PI), + x_ - fixed_point_emp(M_PI * 2)); + + fixed_point_emp xx = x_ * x_; + + fixed_point_emp y = + -xx * fixed_point_emp(1. / (2 * 3 * 4 * 5 * 6 * 7)); + y += fixed_point_emp(1. / (2 * 3 * 4 * 5)); + y *= xx; + y -= fixed_point_emp(1. / (2 * 3)); + y *= xx; + y += fixed_point_emp(1); + y *= x_; + + return y; + } + + /**************************************************************************/ + /* */ + /* sqrt */ + /* */ + /**************************************************************************/ + + /// Calculates the square root. + //! + //! The sqrt function computes the nonnegative square root of its argument. + //! + //! Calculates an approximation of the square root using an integer + //! algorithm. The algorithm is described in Wikipedia: + //! http://en.wikipedia.org/wiki/Methods_of_computing_square_roots + //! + //! The algorithm seems to have originated in a book on programming abaci by + //! Mr C. Woo. + //! + //! /return The square root of the argument. If the argument is negative, + //! the function returns 0. + friend fixed_point_emp sqrt(fixed_point_emp x) { + int size = (I + F); + Integer op(x.value_); + op.resize(size * 2, true); + op = op << I - 1; + + Integer res(size * 2, 0, PUBLIC); + + int64_t o = 1; + o <<= ((size * 2) - 1 - 1); // additional -1 for sign bit + Integer one(size * 2, o, PUBLIC); + // std::cout << one.reveal() << endl; + + // while (one > op) + // one >>= 2; + for (int i = 0; i < size; i++) + one = one.If(one > op, one >> 2); + + // std::cout << one.reveal() << endl; + // std::cout << op.reveal() << endl; + + // while (one != 0) + for (int i = 0; i < size * 2; i++) // is 2* required? + { + // if (op >= res + one) + //{ + // op = op - (res + one); + // res = res + (one << 1); + // } + Bit ogtrpo = (op >= (res + one)); + op = op.If(ogtrpo, op - (res + one)); + res = res.If(ogtrpo, res + (one << 1)); + // res >>= 1; + res = res >> 1; + // one >>= 2; + one = one >> 2; + std::cout << res.reveal() << endl; + } + std::cout << res.reveal() << endl; + + fixed_point_emp root(res); + root.value_.resize(I + F, true); + return root; + } + + // private: + /// The value in fixed point format. + Integer value_; +}; + +template < + /// The input stream type. + typename S, + /// The integer part bit count of the fixed_point_emp type. + unsigned char I, + /// The fractional part bit count of the fixed_point_emp type. + unsigned char F> +/// Stream input operator. +//! +//! A value is first input to type double and then the read value is converted +//! to type fixed_point_emp before it is returned. +//! +//! /return A reference to this input stream. +S& operator>>( + /// The input stream. + S& s, + /// A reference to the value to be read. + fixed_point_emp& v) { + double value = 0.; + s >> value; + if (s) + v = value; + return s; +} + +template < + /// The output stream type. + typename S, + /// The integer part bit count of the fixed_point_emp type. + unsigned char I, + /// The fractional part bit count of the fixed_point_emp type. + unsigned char F> +/// Stream output operator. +//! +//! The fixed_point_emp value is first converted to type double and then the +//! output operator for type double is called. +//! +//! /return A reference to this output stream. +S& operator<<( + /// The output stream. + S& s, + /// A const reference to the value to be written. + fixed_point_emp const& v) { + double value; + value = v.reveal(); + s << value; + return s; +} + +#ifdef __FPML_DEFINED_USE_MATH_DEFINES__ +#undef _USE_MATH_DEFINES +#undef __FPML_DEFINED_USE_MATH_DEFINES__ +#endif diff --git a/src/emp-float-server/gaussnewtonlocalization.cpp b/src/emp-float-server/gaussnewtonlocalization.cpp new file mode 100644 index 0000000..c1247fd --- /dev/null +++ b/src/emp-float-server/gaussnewtonlocalization.cpp @@ -0,0 +1,226 @@ +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "emp-sh2pc/emp-sh2pc.h" +#include "emp-tool/emp-tool.h" + +#include +#include +#include +#include +#include +#include + +using namespace emp; +using namespace std; + +// Note: All 2D matrix inputs passed as single dimension array +// Note: the constant 1's can be ignored (but space must be allocated) +// +// 3D World Points (4xnumPts) which look like: +// _ _ +// | x1 x2 x3 | +// | y1 y2 y3 ... | +// | z1 z2 z3 | +// |_ 1 1 1 _| +// must be pass as single dimension: +// [x1, x2, ...; y1, y2, ...; z1, z2, ...; 1, 1, ... ] +// +// 2D Image Points (3xnumPts) which look like: +// _ _ +// | u1 u2 u3 | +// | v1 v2 v3 ... | +// |_ 1 1 1 _| +// must be passed as single dimension: +// [ u1, u2, ...; v1, v2, ...; 1, 1, ... ] +// +// x (initial guess) : 6x1 [ rotation angles, translations ] +std::pair BuildGaussNewton(Float threeDPts[], Float y0[], int numPts, + Float f, Float cx, Float cy, Float x[]) { + // cout << "y0:\n"; + // printFloatVector(y0, 2*numPts); + + // add constant ones to threeDPts + Float one_gate = Float(1.0, PUBLIC); + for (int i = 0; i < numPts; i++) { + threeDPts[3 * numPts + i] = one_gate; + } + + // Camera params (3x3) + Float K[9] = {f, + Float(0.0, PUBLIC), + cx, + Float(0.0, PUBLIC), + f, + cy, + Float(0.0, PUBLIC), + Float(0.0, PUBLIC), + Float(1.0, PUBLIC)}; + + float jacob_epsilon = + JACOB_EPSILON; // this is needed, can't &JACOB_EPSILON directly + + Float min_er = Float(MIN_ER, PUBLIC); + int i; + for (i = 0; i < GN_MAX_ITR; i++) { + + // project points using x guess + // share **yHomog = new share*[3*numPts]; + Float* yHomog = + static_cast(operator new[](3 * numPts * sizeof(Float))); + BuildProjectPointsCircuit(threeDPts, x, K, yHomog, numPts, true); + + // throw away last "row" of result (constant ones), + // interleave, and transpose y into 2nx1 vector + // e.g. [x1; y1; x2; y2 ...] + // share **y = new share*[2*numPts]; + Float* y = static_cast(operator new[](2 * numPts * sizeof(Float))); + for (int p = 0; p < numPts; ++p) { + y[p * 2] = yHomog[p]; + y[(p * 2) + 1] = yHomog[p + numPts]; + } + delete[] yHomog; + // cout << "y:\n"; + // printFloatVector(y, 2*numPts); + + // calculate jacobian (real 2D matrix not linearized) + // _ _ + // | du1/dr1 du1/dr2 du1/dr3 du1/t1 du1/t2 du1/t3 | + // | dv1/dr1 dv1/dr2 dv1/dr3 dv1/t1 dv1/t2 dv1/t3 | + // | du2/dr1 du2/dr2 du2/dr3 du2/t1 du2/t2 du2/t3 | + // | . | + // | . | + // |_ . (2n) _| + Float** jacob = new Float*[numPts * 2]; + for (int p = 0; p < numPts * 2; p++) { + jacob[p] = static_cast(operator new[](6 * sizeof(Float))); + // for(int pp=0; pp<6; pp++) { + // jacob[p][pp] = Float(0.0, PUBLIC); + // } + } + for (int j = 0; j < 6; j++) { // for each DOF + // perturb x + Float temp = x[j]; + x[j] = x[j] + Float(jacob_epsilon, PUBLIC); + + // project with epsilon + Float* ytempHomog = + static_cast(operator new[](3 * numPts * sizeof(Float))); + // TODO - initialize? + BuildProjectPointsCircuit(threeDPts, x, K, ytempHomog, numPts, true); + + // throw away last "row" of result, + // reshape, and transpose y into 2nx1 vector + // e.g. [x1; y1; x2; y2 ...] + Float* ytemp = + static_cast(operator new[](2 * numPts * sizeof(Float))); + for (int p = 0; p < numPts; ++p) { + ytemp[p * 2] = ytempHomog[p]; + ytemp[(p * 2) + 1] = ytempHomog[p + numPts]; + } + delete[] ytempHomog; + + // put into jacob + for (int p = 0; p < 2 * numPts; ++p) { + jacob[p][j] = (ytemp[p] - y[p]) / Float(jacob_epsilon, PUBLIC); + } + + delete[] ytemp; + + // un-perturb x + x[j] = temp; + } + // printFloatMatrix(jacob, 2*numPts, 6); + + // calculate error + // dy = y0 - y; + Float* dy = static_cast(operator new[](2 * numPts * sizeof(Float))); + for (int p = 0; p < 2 * numPts; p++) { + dy[p] = y0[p] - y[p]; + } + // cout << "dy:\n"; + // printFloatVector(dy, 2*numPts); + delete[] y; + + // compute pseudo inverse to solve: + // dy = J dx -> dx = Jdagger dy + // Note: invert requires real + // 2D arrays not linearized versions + Float** jacobI = new Float*[6]; // 6x2n + for (int p = 0; p < 6; p++) { + jacobI[p] = + static_cast(operator new[](2 * numPts * sizeof(Float))); + // for (int pp=0; pp<2*numPts; pp++) { + // jacobI[p][pp] = Float(0.0, PUBLIC); + // } + } + + BuildInvertCircuit(jacob, 2 * numPts, 6, jacobI); + // printFloatMatrix(jacobI, 6, 2*numPts); + + // linearize for matmult + Float* jacobILinear = + static_cast(operator new[](6 * 2 * numPts * sizeof(Float))); + for (int p = 0; p < 6; p++) { + for (int pp = 0; pp < 2 * numPts; pp++) { + jacobILinear[(p * 2 * numPts) + pp] = jacobI[p][pp]; + } + } + Float* dx = static_cast(operator new[](6 * sizeof(Float))); + BuildMatmultCircuit(jacobILinear, 6, 2 * numPts, dy, 2 * numPts, + 1, // column vector + dx); + // cout << "dx\n"; + // printFloatVector(dx, 6); + delete[] dy; + + // cleanup jacobian + for (int p = 0; p < numPts * 2; p++) { + // individual jacob shares are deleted in invert->svd() + delete[] jacob[p]; + } + delete[] jacob; + + Float norm = BuildTwoNormSqCircuit(dx, 6); + Float absnorm = norm.abs(); + + for (int p = 0; p < 6; p++) { + delete[] jacobI[p]; + } + delete[] jacobI; + delete[] jacobILinear; + + Bit er_flag = absnorm.less_than(min_er); + + // update pose - note: this is usually done after checking error + for (int p = 0; p < 6; p++) { +#if PPL_FLOW == PPL_FLOW_DO // set dx to zero if no error + for (int bi = 0; bi < dx[p].size(); bi++) + dx[p][i] = dx[p][i] & er_flag; +#endif + x[p] = x[p] + dx[p]; + } + delete[] dx; + +#if PPL_FLOW != PPL_FLOW_DO + // okay to reveal how many iterations it took + // return if error under threshold + // abs(norm(dy, cv::NORM_L2SQR)) < MIN_ER + if (er_flag.reveal()) { + break; + } +#endif + } + + if (_verbosity & DBG_FLOW) + cout << "Did " << i << " GN iterations\n"; + return {i < GN_MAX_ITR, i + 1}; +} diff --git a/src/emp-float-server/gaussnewtonlocalization.h b/src/emp-float-server/gaussnewtonlocalization.h new file mode 100644 index 0000000..58b9227 --- /dev/null +++ b/src/emp-float-server/gaussnewtonlocalization.h @@ -0,0 +1,15 @@ +#include + +#include +#include +#include +#include + +#include "emp-sh2pc/emp-sh2pc.h" +#include "emp-tool/emp-tool.h" + +using namespace emp; +using namespace std; + +std::pair BuildGaussNewton(Float threeDPts[], Float y0[], int numPts, + Float f, Float cx, Float cy, Float x[]); diff --git a/src/emp-float-server/invert.cpp b/src/emp-float-server/invert.cpp new file mode 100644 index 0000000..787e86d --- /dev/null +++ b/src/emp-float-server/invert.cpp @@ -0,0 +1,84 @@ +#include +#include +#include +#include + +// #include +#include +#include +#include +#include + +#include "emp-sh2pc/emp-sh2pc.h" +#include "emp-tool/emp-tool.h" + +#include +#include +#include + +using namespace emp; +using namespace std; + +const uint32_t bitlen = 32; + +void BuildInvertCircuit(Float* in[], int m, int n, Float* res[]) { + assert(m >= n); + Float zero_gate = Float(0.0, PUBLIC); + + Float* w = static_cast(operator new[](n * sizeof(Float))); + for (int i = 0; i < n; i++) { + w[i] = Float(0.0, PUBLIC); + } + Float** v = new Float*[n]; + for (int i = 0; i < n; i++) { + v[i] = static_cast(operator new[](n * sizeof(Float))); + for (int j = 0; j < n; j++) { + v[i][j] = Float(0.0, PUBLIC); + } + } + + BuildSvdCircuit(in, m, n, w, v); + + Float** u = in; + + // cout << "w\n"; + // printFloatVector(w, m); + + // cout << "v\n"; + // printFloatMatrix(v, n, n); + // cout << endl; + + // cout << "u\n"; + // printFloatMatrix(u, m, n); + // cout << endl; + + // res = inv(in) = v*inv(w)*uT + // inv(w)*uT + for (int j = 0; j < n; j++) { + // if (w[j]) { + Bit ifw = w[j].equal(zero_gate); + + for (int i = 0; i < m; i++) { + Float temp = u[i][j] / w[j]; + u[i][j] = temp.If(ifw, Float(0.0, PUBLIC)); + } + //} + } + + // v*(w*uT) (don't use matmult so we can do transpose ourselves) + for (int j = 0; j < n; j++) { + for (int jj = 0; jj < m; jj++) { + res[j][jj] = Float(0.0, PUBLIC); + for (int k = 0; k < n; k++) { + res[j][jj] = + res[j][jj] + (v[j][k] * u[jj][k]); // note u indices do transpose + } + } + } + + delete[] w; + for (int i = 0; i < n; i++) { + delete[] v[i]; + } + delete[] v; +} diff --git a/src/emp-float-server/invert.h b/src/emp-float-server/invert.h new file mode 100644 index 0000000..416ca4a --- /dev/null +++ b/src/emp-float-server/invert.h @@ -0,0 +1,7 @@ +#include "emp-sh2pc/emp-sh2pc.h" +#include "emp-tool/emp-tool.h" + +using namespace emp; +using namespace std; + +void BuildInvertCircuit(Float* in[], int m, int n, Float* res[]); diff --git a/src/emp-float-server/lmlocalization.cpp b/src/emp-float-server/lmlocalization.cpp new file mode 100644 index 0000000..24481c9 --- /dev/null +++ b/src/emp-float-server/lmlocalization.cpp @@ -0,0 +1,459 @@ +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "emp-sh2pc/emp-sh2pc.h" +#include "emp-tool/emp-tool.h" + +#include + +#include +#include +#include +#include +#include +#include +#include + +using namespace emp; +using namespace std; + +Float LMIteration(Float threeDPts[], Float y0[], int numPts, Float& f, + Float& cx, Float& cy, Float& lambda, Float x[]) { + // add constant ones to threeDPts + Float one_gate = Float(1.0, PUBLIC); + for (int i = 0; i < numPts; i++) { + threeDPts[3 * numPts + i] = one_gate; + } + + // Camera params (3x3) + Float K[9] = {f, + Float(0.0, PUBLIC), + cx, + Float(0.0, PUBLIC), + f, + cy, + Float(0.0, PUBLIC), + Float(0.0, PUBLIC), + Float(1.0, PUBLIC)}; + + float jacob_epsilon = + JACOB_EPSILON; // this is needed, can't &JACOB_EPSILON directly + Float jacob_epsilon_gate = Float(jacob_epsilon, PUBLIC); + + if (_verbosity & DBG_POSE_UPDATE) { + cout << "x\n"; + printFloatVector(x, 6); + } + + // project points using x guess + // share **yHomog = new share*[3*numPts]; + Float* yHomog = + static_cast(operator new[](3 * numPts * sizeof(Float))); + BuildProjectPointsCircuit(threeDPts, x, K, yHomog, numPts, true); + if (_verbosity & DBG_PROJECT) { + cout << "projected y\n"; + printFloatVector(yHomog, 2 * numPts); + } + + // throw away last "row" of result (constant ones), + // interleave, and transpose y into 2nx1 vector + // e.g. [x1; y1; x2; y2 ...] + // share **y = new share*[2*numPts]; + Float* y = static_cast(operator new[](2 * numPts * sizeof(Float))); + for (int p = 0; p < numPts; ++p) { + y[p * 2] = yHomog[p]; + y[(p * 2) + 1] = yHomog[p + numPts]; + } + delete[] yHomog; + if (_verbosity & DBG_PROJECT) { + cout << "reshapen y\n"; + printFloatVector(y, 2 * numPts); + } + // cout << "y:\n"; + // printFloatVector(y, 2*numPts); + + // calculate jacobian (real 2D matrix not linearized) + // _ _ + // | du1/dr1 du1/dr2 du1/dr3 du1/t1 du1/t2 du1/t3 | + // | dv1/dr1 dv1/dr2 dv1/dr3 dv1/t1 dv1/t2 dv1/t3 | + // | du2/dr1 du2/dr2 du2/dr3 du2/t1 du2/t2 du2/t3 | + // | . | + // | . | + // |_ . (2n) _| + Float** jacob = new Float*[numPts * 2]; + for (int p = 0; p < numPts * 2; p++) { + jacob[p] = static_cast(operator new[](6 * sizeof(Float))); + // for(int pp=0; pp<6; pp++) { + // jacob[p][pp] = Float(0.0, PUBLIC); + // } + } + for (int j = 0; j < 6; j++) { // for each DOF + // perturb x + Float temp = x[j]; + x[j] = x[j] + jacob_epsilon_gate; + + // project with epsilon + Float* ytempHomog = + static_cast(operator new[](3 * numPts * sizeof(Float))); + BuildProjectPointsCircuit(threeDPts, x, K, ytempHomog, numPts, true); + + // throw away last "row" of result, + // reshape, and transpose y into 2nx1 vector + // e.g. [x1; y1; x2; y2 ...] + Float* ytemp = + static_cast(operator new[](2 * numPts * sizeof(Float))); + for (int p = 0; p < numPts; ++p) { + ytemp[p * 2] = ytempHomog[p]; + ytemp[(p * 2) + 1] = ytempHomog[p + numPts]; + } + delete[] ytempHomog; + + // put into jacob + for (int p = 0; p < 2 * numPts; ++p) { + jacob[p][j] = (ytemp[p] - y[p]) / jacob_epsilon_gate; + } + + delete[] ytemp; + + // un-perturb x + x[j] = temp; + } + if (_verbosity & DBG_JACOB) { + cout << "jacob\n"; + printFloatMatrix(jacob, 2 * numPts, 6); + } + + // calculate error + // dy = y0 - y; + Float* dy = static_cast(operator new[](2 * numPts * sizeof(Float))); + for (int p = 0; p < 2 * numPts; p++) { + dy[p] = y0[p] - y[p]; + } + if (_verbosity & DBG_ER) { + cout << "dy\n"; + printFloatVector(dy, 2 * numPts); + } + delete[] y; + + // LM with fletcher improvement + // inv(Jt*J + lambda*diag(Jt*J)) * Jt * dy + Float** JtJ = new Float*[6]; // 6x6 + for (int p = 0; p < 6; p++) { + JtJ[p] = static_cast(operator new[](6 * sizeof(Float))); + } + BuildMatmult2DwTransposeCircuit(jacob, numPts * 2, 6, true, jacob, numPts * 2, + 6, false, + JtJ); // must use 2D arrays, not 1D + // add lambda * diag(Jt*J) in-place + for (int p = 0; p < 6; p++) { + // add fletcher column-wise to JtJ + Float fletcher = lambda * JtJ[p][p]; + + // for (int pp=0; pp<6; pp++) { + // JtJ[pp][p] = JtJ[pp][p] + fletcher; + // } + JtJ[p][p] = JtJ[p][p] + fletcher; + } + + Float** JtJ_i = new Float*[6]; // 6x6 + for (int p = 0; p < 6; p++) { + JtJ_i[p] = static_cast(operator new[](6 * sizeof(Float))); + } + BuildInvertCircuit(JtJ, 6, 6, JtJ_i); + for (int p = 0; p < 6; p++) { + delete[] JtJ[p]; + } + delete[] JtJ; + + Float** JtJ_i_Jt = new Float*[6]; // 6x2n + for (int p = 0; p < 6; p++) { + JtJ_i_Jt[p] = + static_cast(operator new[](numPts * 2 * sizeof(Float))); + } + BuildMatmult2DwTransposeCircuit(JtJ_i, 6, 6, false, jacob, numPts * 2, 6, + true, JtJ_i_Jt); + for (int p = 0; p < numPts * 2; p++) + delete[] jacob[p]; + delete[] jacob; + for (int p = 0; p < 6; p++) + delete[] JtJ_i[p]; + delete[] JtJ_i; + + // linearize for matmult + Float* JtJ_i_Jt_linear = + static_cast(operator new[](6 * numPts * 2 * sizeof(Float))); + for (int p = 0; p < 6; p++) { + for (int pp = 0; pp < numPts * 2; pp++) { + JtJ_i_Jt_linear[(p * numPts * 2) + pp] = JtJ_i_Jt[p][pp]; + } + } + Float* dx = static_cast(operator new[](6 * sizeof(Float))); + BuildMatmultCircuit(JtJ_i_Jt_linear, 6, numPts * 2, dy, numPts * 2, + 1, // column vector + dx); + + if (_verbosity & DBG_POSE_UPDATE) { + cout << "dx\n"; + printFloatVector(dx, 6); + } + + delete[] dy; + for (int p = 0; p < 6; p++) + delete[] JtJ_i_Jt[p]; + delete[] JtJ_i_Jt; + delete[] JtJ_i_Jt_linear; + + // abs(norm(dy, cv::NORM_L2SQR)) < MIN_ER + Float norm = BuildTwoNormSqCircuit(dx, 6); + Float errNorm = norm.abs(); + if (_verbosity & DBG_ER) + cout << "errNorm\n" << errNorm.reveal() << endl; + +#if PPL_FLOW == PPL_FLOW_DO // error threshold + Bit er_flag = Float(MIN_ER, PUBLIC).less_than(errNorm); +#endif + + // update pose + for (int p = 0; p < 6; p++) { +#if PPL_FLOW == PPL_FLOW_DO // set dx to zero if no error + for (int bi = 0; bi < dx[p].size(); bi++) + dx[p][bi] = dx[p][bi] & er_flag; +#endif + x[p] = x[p] + dx[p]; + } + delete[] dx; + return errNorm; +} + +// Note: All 2D matrix inputs passed as single dimension array +// Note: the constant 1's can be ignored (but space must be allocated) +// +// 3D World Points (4xnumPts) which look like: +// _ _ +// | x1 x2 x3 | +// | y1 y2 y3 ... | +// | z1 z2 z3 | +// |_ 1 1 1 _| +// must be pass as single dimension: +// [x1, x2, ...; y1, y2, ...; z1, z2, ...; 1, 1, ... ] +// +// 2D Image Points (3xnumPts) which look like: +// _ _ +// | u1 u2 u3 | +// | v1 v2 v3 ... | +// |_ 1 1 1 _| +// must be passed as single dimension: +// [ u1, u2, ...; v1, v2, ...; 1, 1, ... ] +// +// x (initial guess) : 6x1 [ rotation angles, translations ] +std::pair BuildLM(Float threeDPts[], Float y0[], int numPts, Float f, + Float cx, Float cy, Float x[]) { + if (_verbosity & DBG_FLOW) + cout << "Levenberg–Marquardt\n"; + + if (_verbosity & DBG_ARGS) { + cout << "P\n"; + printFloatVector(threeDPts, 3 * numPts); + cout << "y0\n"; + printFloatVector(y0, 2 * numPts); + cout << "f: " << f.reveal() << endl; + cout << "cx: " << cx.reveal() << endl; + cout << "cy: " << cy.reveal() << endl; + } + + Float lambdaBoundTop = Float(LM_LAMBDA_MAX, PUBLIC); + Float lambdaBoundBottom = Float(LM_LAMBDA_MIN, PUBLIC); + Float lambda = Float(LM_LAMBDA_INIT, PUBLIC); + Float errNorm = Float(std::numeric_limits::max(), PUBLIC); + + int i; + for (i = 0; i < LM_MAX_ITR; i++) { + if (_verbosity & DBG_FLOW) + cout << RED << "\n\nLM Iteration " << i << RESET << endl; + + Float prevErrNorm = errNorm; + errNorm = LMIteration(threeDPts, y0, numPts, f, cx, cy, lambda, x); + + Float biggerLambda = lambda * Float(10, PUBLIC); + Float smallerLambda = lambda / Float(10, PUBLIC); + + Bit errWentDown = errNorm.less_than(prevErrNorm); + lambda = biggerLambda.If(errWentDown, smallerLambda); + + Bit lambdaTooBig = lambdaBoundTop.less_than(lambda); + lambda = lambda.If(lambdaTooBig, lambdaBoundTop); + Bit lambdaTooSmall = lambda.less_than(lambdaBoundBottom); + lambda = lambda.If(lambdaTooSmall, lambdaBoundBottom); + + if (_verbosity & DBG_LAMBDA) { + cout << "lambda " << lambda.reveal() << endl; + } + +#if PPL_FLOW == PPL_FLOW_SiSL || PPL_FLOW == PPL_FLOW_LOOP_LEAK + // okay to reveal how many iterations it took. + // If loop leak, this is done server side. + // If SiSL, this would be done client side (not revealed to servers) + // but for benchmarking purposes do it here. +#if PPL_FLOW == PPL_FLOW_SiSL + static bool warning_printed = false; + if (!warning_printed) { + cout << "WARNING: BuildLM() when compiling with SiSL is for testing " + "only. Do not use in prod.\n"; + warning_printed = true; + } +#endif + Bit er_flag = prevErrNorm.less_than(Float(MIN_ER, PUBLIC)); + if (er_flag.reveal()) { + if (_verbosity & DBG_ER) { + cout << "errNorm is smaller than " << MIN_ER << endl; + } + break; + } +#endif + } + if (_verbosity & DBG_FLOW) + cout << "Did " << i + 1 << " LM iterations\n"; + return {i < LM_MAX_ITR, i + 1}; +} + +std::pair test_lm_circuit( + int party, NetIO* io, vector threeDPts, + vector twoDPts, float f, float cx, float cy, + float* x /* initial guess for { r1, r2, r3, t1, t2, t3 } */) { + + setup_semi_honest(io, party); + + int numPts = threeDPts.size(); + + Float* s_threeDPts = + static_cast(operator new[](4 * numPts * sizeof(Float))); + Float* s_twoDPts = + static_cast(operator new[](3 * numPts * sizeof(Float))); + for (int i = 0; i < numPts; i++) { + // [x1, x2... ; y1, y2... ; z1, z2...] + s_threeDPts[i] = Float(threeDPts[i].x, ALICE); + s_threeDPts[numPts + i] = Float(threeDPts[i].y, ALICE); + s_threeDPts[2 * numPts + i] = Float(threeDPts[i].z, ALICE); + // s_threeDPts[3*numPts + i] = Float(1.0, PUBLIC); + + // [x1, y1; x2, y2; ...] + s_twoDPts[2 * i] = Float(twoDPts[i].x, ALICE); + s_twoDPts[2 * i + 1] = Float(twoDPts[i].y, ALICE); + // s_twoDPts[2*numPts + i] = Float(1.0, PUBLIC); + } + + Float s_f = Float(f, ALICE); + Float s_cx = Float(cx, ALICE); + Float s_cy = Float(cy, ALICE); + + Float* s_x = static_cast(operator new[](6 * sizeof(Float))); + for (int i = 0; i < 6; i++) { + s_x[i] = Float(x[i], ALICE); + } + + auto start = clock_start(); + auto res = BuildLM(s_threeDPts, s_twoDPts, numPts, s_f, s_cx, s_cy, s_x); + double interval = time_from(start); + + for (int i = 0; i < 6; i++) { + x[i] = s_x[i].reveal(); + s_x[i].~Float(); + } + delete[] s_x; + delete[] s_threeDPts; + delete[] s_twoDPts; + + uint64_t numand = CircuitExecution::circ_exec->num_and(); + cout << "number of and gates: " << numand << endl; + cout << "garbling speed : " << numand / interval + << " million gate per second\n"; + + finalize_semi_honest(); + + return res; +} + +void lm_server(int party, NetIO* io, NetIO* ttpio) { + uint32_t numPts; + ttpio->recv_data(&numPts, sizeof(numPts)); + MSG("Performing LM with %d points\n", numPts); + + setup_semi_honest(io, party, 1024 * 16, ttpio); + MSG("setup done\n"); + int stack_corruption[100]; + + Float* s_threeDPts = + static_cast(operator new[](4 * numPts * sizeof(Float) + 100)); + Float* s_twoDPts = + static_cast(operator new[](3 * numPts * sizeof(Float) + 100)); + for (uint32_t i = 0; i < numPts; i++) { + // [x1, x2... ; y1, y2... ; z1, z2...] + s_threeDPts[i] = Float(0, TTP); + s_threeDPts[numPts + i] = Float(0, TTP); + s_threeDPts[2 * numPts + i] = Float(0, TTP); + // s_threeDPts[3*numPts + i] = Float(0, PUBLIC); + + // [x1, y1; x2, y2; ...] + s_twoDPts[2 * i] = Float(0, TTP); + s_twoDPts[2 * i + 1] = Float(0, TTP); + // s_twoDPts[2*numPts + i] = Float(1.0, PUBLIC); + } + + Float s_f = Float(0, TTP); + Float s_cx = Float(0, TTP); + Float s_cy = Float(0, TTP); + + Float* s_x = static_cast(operator new[](6 * sizeof(Float) + 100)); + for (int i = 0; i < 6; i++) { + s_x[i] = Float(0, TTP); + } + + auto start = clock_start(); +#if PPL_FLOW == PPL_FLOW_LOOP_LEAK || PPL_FLOW == PPL_FLOW_DO + auto [converged, num_iterations] = + BuildLM(s_threeDPts, s_twoDPts, numPts, s_f, s_cx, s_cy, s_x); + if (converged) + cout << "converged in " << num_iterations << " iterations\n"; + else + cout << "did not converge\n"; +#elif PPL_FLOW == PPL_FLOW_SiSL + Float s_lambda = Float(0, TTP); + Float s_errNorm = LMIteration(s_threeDPts, s_twoDPts, numPts, s_f, s_cx, s_cy, + s_lambda, s_x); +#elif PPL_FLOW == PPL_FLOW_POWER_TESTING + Float s_lambda = Float(0, TTP); + Float s_errNorm = Float(MIN_ER, PUBLIC); +#endif + double interval = time_from(start); + + cout << "revealing pose to ttp\n"; + for (int i = 0; i < 6; i++) { + s_x[i].reveal(TTP); // ignore output, it goes to ttp + s_x[i].~Float(); + } +#if PPL_FLOW == PPL_FLOW_SiSL || PPL_FLOW == PPL_FLOW_POWER_TESTING + cout << "revealing errnorm to ttp\n"; + s_errNorm.reveal(TTP); +#endif + io->flush(); + ttpio->flush(); + delete[] s_x; + delete[] s_threeDPts; + delete[] s_twoDPts; + + uint64_t numand = CircuitExecution::circ_exec->num_and(); + cout << "number of and gates: " << numand << endl; + cout << "garbling speed : " << numand / interval + << " million gate per second\n"; + cout << "time: " << interval << "us\n"; + + finalize_semi_honest(); +} diff --git a/src/emp-float-server/lmlocalization.h b/src/emp-float-server/lmlocalization.h new file mode 100644 index 0000000..40140f6 --- /dev/null +++ b/src/emp-float-server/lmlocalization.h @@ -0,0 +1,21 @@ +#include +#include + +#include +#include +#include +#include + +#include "emp-sh2pc/emp-sh2pc.h" +#include "emp-tool/emp-tool.h" + +using namespace emp; +using namespace std; + +std::pair BuildLM(Float threeDPts[], Float y0[], int numPts, Float f, + Float cx, Float cy, Float x[]); + +Float LMIteration(Float threeDPts[], Float y0[], int numPts, Float& f, + Float& cx, Float& cy, Float& lambda, Float x[]); + +void lm_server(int party, NetIO* io, NetIO* ttpio); diff --git a/src/emp-float-server/matmult.cpp b/src/emp-float-server/matmult.cpp new file mode 100644 index 0000000..0f7075d --- /dev/null +++ b/src/emp-float-server/matmult.cpp @@ -0,0 +1,55 @@ +#include +#include +#include +#include + +#include "emp-sh2pc/emp-sh2pc.h" +#include "emp-tool/emp-tool.h" + +using namespace emp; +using namespace std; + +void BuildMatmultCircuit(Float M[], int m, int n, Float N[], int mm, int nn, + Float res[]) { + (void)mm; + assert(n == mm); + for (int i = 0; i < m; i++) { // row + for (int j = 0; j < nn; j++) { // col + // res[i*nn +j] = Float(0.0, ALICE); + new (&res[i * nn + j]) Float(0.0, PUBLIC); + for (int k = 0; k < n; k++) { + Float temp = M[i * n + k] * N[k * nn + j]; + res[i * nn + j] = res[i * nn + j] + temp; + } + } + } +} + +void BuildMatmult2DwTransposeCircuit(Float** M, int m, int n, + const bool transposeM, Float** N, int mm, + int nn, const bool transposeN, + Float** res) { + if (transposeM) + std::swap(m, n); + if (transposeN) + std::swap(mm, nn); + assert(n == mm); + + for (int i = 0; i < m; i++) { // row + for (int j = 0; j < nn; j++) { // col + new (&res[i][j]) Float(0.0, PUBLIC); + for (int k = 0; k < n; k++) { + Float temp = Float(0.0, PUBLIC); + if (!transposeM && !transposeN) + temp = M[i][k] * N[k][j]; + else if (!transposeM && transposeN) + temp = M[i][k] * N[j][k]; + else if (transposeM && !transposeN) + temp = M[k][i] * N[k][j]; + else if (transposeM && transposeN) + temp = M[k][i] * N[j][k]; + res[i][j] = res[i][j] + temp; + } + } + } +} diff --git a/src/emp-float-server/matmult.h b/src/emp-float-server/matmult.h new file mode 100644 index 0000000..88f1ef1 --- /dev/null +++ b/src/emp-float-server/matmult.h @@ -0,0 +1,15 @@ +#include "emp-sh2pc/emp-sh2pc.h" +#include "emp-tool/emp-tool.h" + +using namespace emp; + +// M mxn (M is secret, m n are public) +// N mmxnn +// res mxnn +void BuildMatmultCircuit(Float M[], int m, int n, Float N[], int mm, int nn, + Float res[]); + +void BuildMatmult2DwTransposeCircuit(Float** M, int m, int n, + const bool transposeM, Float** N, int mm, + int nn, const bool transposeN, + Float** res); diff --git a/src/emp-float-server/projectpoints.cpp b/src/emp-float-server/projectpoints.cpp new file mode 100644 index 0000000..ff739ba --- /dev/null +++ b/src/emp-float-server/projectpoints.cpp @@ -0,0 +1,72 @@ +#include +#include +#include +#include + +//#include +#include +#include +#include +#include + +#include "emp-sh2pc/emp-sh2pc.h" +#include "emp-tool/emp-tool.h" + +#include +#include +#include + +using namespace emp; +using namespace std; + +// Note: All 2D matrix inputs passed as single dimension array +// P=4xnumPoints [x1, x2, ...; y1, y2, ...; z1, z2, ...; 1, 1, ... ] +// x=6x1 [ rotation angles; translation] (column) +// K=3x3 camera intrinsic +// result=3xnumPoints preallocated, only first two rows are x,y. +// last row needed for intermediate calculation (homogeneous) +// [ u1, u2, ...; v1, v2, ...; 1, 1, ... ] +void BuildProjectPointsCircuit(Float _P[], Float _x[], Float _K[], Float _res[], + int numPoints, bool skipOneGate = false) { + Float one_gate = Float(1.0, PUBLIC); + + void* raw_memoryR = operator new[](12 * sizeof(Float)); + Float* R = static_cast(raw_memoryR); + + BuildRodriguesCircuit(_x, R); + + // copy translation into last column of R + R[3] = _x[3]; + R[7] = _x[4]; + R[11] = _x[5]; + + // cout << "x:"<< endl; + // printFloatVector(_x, 6); + // cout <<"R:"<< endl; + // printFloatVector(R, 12); + // cout << endl; + + // res = K*R*P; + void* raw_memoryscratch = operator new[](3 * 4 * sizeof(Float)); + Float* scratch = static_cast(raw_memoryscratch); + BuildMatmultCircuit(_K, 3, 3, R, 3, 4, scratch); + BuildMatmultCircuit(scratch, 3, 4, _P, 4, numPoints, _res); + + // homogeneous coords to x,y + for (int i = 0; i < numPoints; i++) { + _res[i] = _res[i] / _res[2 * numPoints + i]; + _res[numPoints + i] = _res[numPoints + i] / _res[2 * numPoints + i]; + if (!skipOneGate) { + _res[2 * numPoints + i] = one_gate; // can be excluded for efficiency + } + } + + for (int i = 0; i < 12; i++) { + R[i].~Float(); + } + delete[] R; + for (int i = 0; i < 3 * 4; i++) { + scratch[i].~Float(); + } + delete[] scratch; +} diff --git a/src/emp-float-server/projectpoints.h b/src/emp-float-server/projectpoints.h new file mode 100644 index 0000000..07e20e8 --- /dev/null +++ b/src/emp-float-server/projectpoints.h @@ -0,0 +1,15 @@ +#include "emp-sh2pc/emp-sh2pc.h" +#include "emp-tool/emp-tool.h" + +using namespace emp; +using namespace std; + +// Note: All 2D matrix inputs passed as single dimension array +// P=4xnumPoints [x1, x2, ...; y1, y2, ...; z1, z2, ...; 1, 1, ... ] +// x=6x1 [ rotation angles; translation] (column) +// K=3x3 camera intrinsic +// result=3xnumPoints preallocated, only first two rows are x,y. +// last row needed for intermediate calculation (homogeneous) +// [ u1, u2, ...; v1, v2, ...; 1, 1, ... ] +void BuildProjectPointsCircuit(Float _P[], Float _x[], Float _K[], Float _res[], + int numPoints, bool skipOneGate); diff --git a/src/emp-float-server/rodrigues.cpp b/src/emp-float-server/rodrigues.cpp new file mode 100644 index 0000000..012a360 --- /dev/null +++ b/src/emp-float-server/rodrigues.cpp @@ -0,0 +1,71 @@ +#include +#include +#include +#include + +//#include +#include +#include +#include +#include + +#include "emp-sh2pc/emp-sh2pc.h" +#include "emp-tool/emp-tool.h" + +#include +#include + +using namespace emp; +using namespace std; + +// r=3x1, R=3x4 (only 3x3 is used here) +// values are secret, sizes are public +void BuildRodriguesCircuit(Float r[], Float R[]) { + Float one_gate = Float(1.0, PUBLIC); + Float zero_gate = Float(0.0, PUBLIC); + + // calculate theta + Float rzerosq = r[0] * r[0]; + Float ronesq = r[1] * r[1]; + Float rtwosq = r[2] * r[2]; + Float theta1 = rzerosq + ronesq; + Float theta2 = theta1 + rtwosq; + Float theta = theta2.sqrt(); + + Float co = BuildCosCircuit(theta); + Float si = BuildSinCircuit(theta); + + Float c1 = one_gate - co; + + // if theta == 0 then itheta = 0 + Bit thetaiszero = theta.equal(zero_gate); + Float itheta = (one_gate / theta).If(thetaiszero, zero_gate); + + Float x = r[0] * itheta; + Float y = r[1] * itheta; + Float z = r[2] * itheta; + + Float xx = x * x; + Float yy = y * y; + Float zz = z * z; + + Float xy = x * y; + Float xz = x * z; + Float yz = y * z; + + Float sx = si * x; + Float sy = si * y; + Float sz = si * z; + + R[0] = (c1 * xx) + co; + R[1] = (c1 * xy) - sz; + R[2] = (c1 * xz) + sy; + + R[4] = (c1 * xy) + sz; + R[5] = (c1 * yy) + co; + R[6] = (c1 * yz) - sx; + + R[8] = (c1 * xz) - sy; + R[9] = (c1 * yz) + sx; + R[10] = (c1 * zz) + co; +} diff --git a/src/emp-float-server/rodrigues.h b/src/emp-float-server/rodrigues.h new file mode 100644 index 0000000..2f399e5 --- /dev/null +++ b/src/emp-float-server/rodrigues.h @@ -0,0 +1,9 @@ +#include "emp-sh2pc/emp-sh2pc.h" +#include "emp-tool/emp-tool.h" + +using namespace emp; +using namespace std; + +// r=3x1, R=3x4 (only 3x3 is used here) +// values are secret, sizes are public +void BuildRodriguesCircuit(Float r[], Float R[]); diff --git a/src/emp-float-server/server-lm.cpp b/src/emp-float-server/server-lm.cpp new file mode 100644 index 0000000..e82e377 --- /dev/null +++ b/src/emp-float-server/server-lm.cpp @@ -0,0 +1,76 @@ +#include +#include +#include +#include +#include "jlog.h" + +#include "invert.h" +#include "lmlocalization.h" +#include "matmult.h" +#include "projectpoints.h" +#include "rodrigues.h" +#include "svd.h" +#include "trigfuncs.h" +#include "twonormsq.h" +#include "util.h" + +// #include +#include +#include +#include +#include + +#include "emp-sh2pc/emp-sh2pc.h" + +#include +void signal_callback_handler(int signum) { + cout << "Caught signal " << signum << endl; + // Terminate program + exit(signum); +} + +using namespace emp; +using namespace std; + +int main(int argc, char** argv) { + if (argc != 2) { + MSG("Usage: %s \n", argv[0]); + return 1; + } + + while (true) { + int party = atoi(argv[1]) + 1; + int port = 8080; + cout << "party: " << party << " port: " << port << endl; + NetIO* io = new NetIO(party == ALICE ? nullptr : "127.0.0.1", port); + + int ttpport = port + party * 17; + cout << "ttp port: " << ttpport << endl; + NetIO* ttpio = new NetIO(nullptr, ttpport); + + while (true) { + auto t1 = chrono::steady_clock ::now(); + auto sec_passed = chrono::duration_cast( + chrono::steady_clock ::now() - t1) + .count(); + int bytes_ready = 0; + while (!bytes_ready && sec_passed < 3) { + sec_passed = chrono::duration_cast( + chrono::steady_clock ::now() - t1) + .count(); + ioctl(ttpio->consocket, FIONREAD, &bytes_ready); + } + if (sec_passed >= 10) { + cout << "waited for 10 seconds with no data, restarting server\n"; + break; + } + + cout << "running pplm server\n"; + lm_server(party, io, ttpio); + } + delete io; + delete ttpio; + } + + return 0; +} diff --git a/src/emp-float-server/svd.cpp b/src/emp-float-server/svd.cpp new file mode 100644 index 0000000..ab72e8e --- /dev/null +++ b/src/emp-float-server/svd.cpp @@ -0,0 +1,611 @@ +#include +#include +#include +#include +#include +#include + +#include "emp-sh2pc/emp-sh2pc.h" +#include "emp-tool/emp-tool.h" + +#include +#include + +static int iminarg1, iminarg2; +#define IMIN(a, b) \ + (iminarg1 = (a), iminarg2 = (b), \ + (iminarg1 < (iminarg2) ? (iminarg1) : iminarg2)) + +using namespace emp; +using namespace std; + +// EMP NOTE - new value = falseValue.If(bool, trueValue) + +void BuildSignCircuit(Float* a, Float* b) { + // WARNING: + // cannot simply set msb of a to msb of b because + // of the case when b is 0.0 + Float zero_gate = Float(0.0, PUBLIC); + + // share* cmp = c->PutFPGate(b, zero_gate, CMP); + Bit bgtzero = zero_gate.less_than(*b); + // share* tmp = c->PutINVGate(cmp); + // a->set_wire_id(31, tmp->get_wire_id(0)); + a->value[31] = !bgtzero; +} + +Float BuildPythagCircuit(Float* a, Float* b) { + Float aa = a->sqr(); + Float bb = b->sqr(); + Float sum = aa + bb; + Float res = sum.sqrt(); + return res; +} + +int BuildSvdCircuit(Float** a, int nRows, int nCols, Float* w, Float** v) { + Float zero_gate = Float(0.0, PUBLIC); + // Float one_gate = Float(1.0, PUBLIC); + // Float two_gate = Float(2.0, PUBLIC); + + int flag, i, its, j, jj, k, l, nm; + Float anorm = Float(0.0, PUBLIC); + Float c = Float(0.0, PUBLIC); + Float f = Float(0.0, PUBLIC); + Float g = Float(0.0, PUBLIC); + Float h = Float(0.0, PUBLIC); + Float s = Float(0.0, PUBLIC); + Float scale = Float(0.0, PUBLIC); + Float x = Float(0.0, PUBLIC); + Float y = Float(0.0, PUBLIC); + Float z = Float(0.0, PUBLIC); + Float* rv1 = (Float*)malloc(sizeof(Float) * nCols); + if (rv1 == NULL) { + printf("BuildSvdCircuit(): Unable to allocate vector\n"); + return (-1); + } + + // Householder reduction to bidiagonal form + // g = scale = anorm = 0.0; + for (i = 0; i < nCols; i++) { + l = i + 1; + rv1[i] = scale * g; + g = Float(0.0, PUBLIC); + s = Float(0.0, PUBLIC); + scale = Float(0.0, PUBLIC); + if (i < nRows) { + for (k = i; k < nRows; k++) { + scale = scale + a[k][i].abs(); + } + /*if (scale)*/ { + Bit ifscale = scale.equal(zero_gate); + for (k = i; k < nRows; k++) { + Float temp = a[k][i] / scale; + a[k][i] = temp.If(ifscale, a[k][i]); + s = s + a[k][i].sqr(); + // s = s + (a[k][i] * a[k][i]); + } + // if (i == 2) return 0; // good here + // if (i==2) cout << "a[i][i] " << a[i][i].reveal() << endl; + // good + f = a[i][i]; // no .If okay + Float temp = s.sqrt(); + BuildSignCircuit(&temp, &f); + temp = -temp; + g = temp.If(ifscale, g); + h = f * g - s; // no .If okay + temp = f - g; + a[i][i] = temp.If(ifscale, a[i][i]); + // if (i==2) cout << "f " << f.reveal() << endl; good + // if (i==2) cout << "g " << g.reveal() << endl; good + // if (i==2) cout << "h " << h.reveal() << endl; good + // if (i==2) cout << "a[i][i] " << a[i][i].reveal() << endl; + // good + for (j = l; j < nCols; j++) { + s = Float(0.0); + for (k = i; k < nRows; k++) { + // if (i==2) cout << "a[k][i] " << a[k][i].reveal() << endl; + // if (i==2) cout << "a[k][j] " << a[k][j].reveal() << endl; + s = s + (a[k][i] * a[k][j]); + // if (i==2) cout << "mid s " << s.reveal() << endl; + } + f = s / h; + for (k = i; k < nRows; k++) { + Float temp = a[k][j] + (f * a[k][i]); + a[k][j] = temp.If(ifscale, a[k][j]); + } + } + for (k = i; k < nRows; k++) { + Float temp = a[k][i] * scale; + a[k][i] = temp.If(ifscale, a[k][i]); + } + } + } + // if (i == 2) return 0; // bad here + // discrepancy in A here because of underflow? + w[i] = scale * g; + g = Float(0.0, PUBLIC); + s = Float(0.0, PUBLIC); + scale = Float(0.0, PUBLIC); + if (i < nRows && i != nCols - 1) { + for (k = l; k < nCols; k++) + scale = scale + a[i][k].abs(); + /*if (scale)*/ { + Bit ifscale = scale.equal(zero_gate); + for (k = l; k < nCols; k++) { + Float temp = a[i][k] / scale; + a[i][k] = temp.If(ifscale, a[i][k]); + s = s + a[i][k].sqr(); + // s = s + (a[i][k] * a[i][k]); + } + f = a[i][l]; // no .If okay + Float temp = s.sqrt(); + BuildSignCircuit(&temp, &f); + temp = -temp; + g = temp.If(ifscale, g); + temp = f * g - s; + h = temp.If(ifscale, Float(1.0, PUBLIC)); // h is used to set rv1 if + // scale==0, set to 1 for noop + temp = f - g; + a[i][l] = temp.If(ifscale, a[i][l]); + for (k = l; k < nCols; k++) + rv1[k] = a[i][k] / h; // no mux because h is one ifscale + for (j = l; j < nRows; j++) { + for (s = Float(0.0, PUBLIC), k = l; k < nCols; k++) + s = s + (a[j][k] * a[i][k]); + for (k = l; k < nCols; k++) { + Float temp = a[j][k] + (s * rv1[k]); + a[j][k] = temp.If(ifscale, a[j][k]); + } + } + for (k = l; k < nCols; k++) { + Float temp = a[i][k] * scale; + a[i][k] = temp.If(ifscale, a[i][k]); + } + } + } + // anorm = FMAX(anorm, (fabs(w[i]) + fabs(rv1[i]))); + Float temp = w[i].abs() + rv1[i].abs(); + Bit cmp = anorm.less_than(temp); + anorm = anorm.If(cmp, temp); + + printf("."); + fflush(stdout); + } + + // accumulation of right hand transformations + for (i = nCols - 1; i >= 0; i--) { + if (i < nCols - 1) { + /*if (g)*/ { + Bit ifg = g.equal(zero_gate); + for (j = l; j < nCols; j++) { + Float temp = (a[i][j] / a[i][l]) / g; + v[j][i] = temp.If(ifg, v[j][i]); + } + for (j = l; j < nCols; j++) { + s = Float(0.0, PUBLIC); + for (k = l; k < nCols; k++) + s = s + (a[i][k] * v[k][j]); + for (k = l; k < nCols; k++) { + Float temp = v[k][j] + (s * v[k][i]); + v[k][j] = temp.If(ifg, v[k][j]); + } + } + } + for (j = l; j < nCols; j++) { + v[i][j] = Float(0.0, PUBLIC); + v[j][i] = Float(0.0, PUBLIC); + } + } + v[i][i] = Float(1.0, PUBLIC); + g = rv1[i]; + l = i; + printf(":"); + fflush(stdout); + } + + // accumulation of left hand transformations + for (i = IMIN(nRows, nCols) - 1; i >= 0; i--) { + l = i + 1; + g = w[i]; + for (j = l; j < nCols; j++) + a[i][j] = Float(0.0, PUBLIC); + /*if (g)*/ { + Bit ifg = g.equal(zero_gate); + g = Float(1.0, PUBLIC) / g; + for (j = l; j < nCols; j++) { + s = Float(0.0, PUBLIC); + for (k = l; k < nRows; k++) { + s = s + (a[k][i] * a[k][j]); // no .If okay + } + f = (s / a[i][i]) * g; // no .If okay + for (k = i; k < nRows; k++) { + Float temp = a[k][j] + (f * a[k][i]); + a[k][j] = temp.If(ifg, a[k][j]); + } + } + for (j = i; j < nRows; j++) { + Float temp = a[j][i] * g; + a[j][i] = temp.If(ifg, Float(0.0, PUBLIC)); + } + /*} else { else - done in for loop above with mux */ + // for (j = i; j < nRows; j++) { + // a[j][i] = a[j][i].If(ifg, zero_gate); + // } + } + a[i][i] = a[i][i] + Float(1.0, PUBLIC); + printf("|"); + fflush(stdout); + } + // NOTE - ignore the fact that last row may be different here. + // Cause is floats running out of precision + + // Diagonalization of the bidiagonal form: loop over singular + // values and over allowed iterations +#if PPL_FLOW == PPL_FLOW_DO + for (k = nCols - 1; k >= 0; k--) { + Bit converged = Bit(false, PUBLIC); + for (its = 0; its < 30; its++) { + cout << k << flush; + Bit flag = Bit(true, PUBLIC); + Bit l_found = Bit(false, PUBLIC); + Integer priv_l = Integer(32, 0, PUBLIC); + + for (l = k; l >= 0; l--) { // test for splitting + nm = l - 1; // note rv1[0] is always zero + + Integer this_l = Integer(32, l, PUBLIC); + + Bit temp = (rv1[l] + anorm).equal(anorm); + flag = flag & !(temp & !l_found); + priv_l = priv_l.If(temp & (!l_found), this_l); + l_found = l_found | temp; + + if (nm >= 0) { + temp = (w[nm].abs() + anorm).equal(anorm); // TODO need abs here? + priv_l = priv_l.If(temp & (!l_found), this_l); + l_found = l_found | temp; + } + } + + Integer priv_nm = priv_l - Integer(32, 1, PUBLIC); + { // if(flag) - cancellation of rv1(l), note l is always >= 1 + c = Float(0.0, PUBLIC); // no If needed + s = Float(1.0, PUBLIC); // no If needed + for (int i = 1; i <= k; i++) { // start at lowest l (which is 1) + Bit igel = Integer(32, i, PUBLIC).geq(priv_l); + Bit modify = flag & igel & (!converged); + + f = s * rv1[i]; + rv1[i] = rv1[i].If(modify, c * rv1[i]); + + Bit breaked = (f.abs() + anorm).equal(anorm); + modify = modify & (!breaked); + + g = w[i]; + h = BuildPythagCircuit(&f, &g); + w[i] = w[i].If(modify, h); + h = Float(1.0, PUBLIC) / h; + c = c.If(modify, g * h); + s = s.If(modify, -f * h); + for (j = 0; j < nRows; j++) { + for (int nm_finder = 0; nm_finder < k; nm_finder++) { + Bit found_nm = priv_nm.equal(Integer(32, nm_finder, PUBLIC)); + Bit mafnm = modify & found_nm; + y = a[j][nm_finder]; + z = a[j][i]; + a[j][nm_finder] = a[j][nm_finder].If(mafnm, y * c + z * s); + a[j][i] = a[j][i].If(mafnm, z * c - y * s); + } + } + } + } + z = w[k]; + Bit newconverged = priv_l.equal(Integer(32, k, PUBLIC)); + /*if (l == k)*/ { // convergence + /*if (z < 0.0)*/ { // singular value is made nonnegative + Bit ifzlt = z.less_than(zero_gate); + Bit modify = ifzlt & newconverged & (!converged); + w[k] = w[k].If(modify, -z); + for (j = 0; j < nCols; j++) { + v[j][k] = v[j][k].If(modify, -v[j][k]); + } + } + } + converged = converged | newconverged; + if (k == 0) + break; // if k == 0 we are done, code below segfaults from nm + + // shift from bottom 2-by-2 minor + for (int l_finder = 0; l_finder < k; l_finder++) { + Bit found_l = priv_l.equal(Integer(32, l_finder, PUBLIC)); + x = x.If(found_l & (!converged), w[l_finder]); + } + nm = k - 1; + y = w[nm]; + g = rv1[nm]; + h = rv1[k]; + f = ((y - z) * (y + z) + (g - h) * (g + h)) / + (Float(2.0, PUBLIC) * h * y); + Float one_gate = Float(1.0, PUBLIC); + g = BuildPythagCircuit(&f, &one_gate); + BuildSignCircuit(&g, &f); // modifies g + f = ((x - z) * (x + z) + h * ((y / (f + g)) - h)) / x; + // Next QR transformation + c = Float(1.0, PUBLIC); + s = Float(1.0, PUBLIC); + + for (j = 0; j <= nm; j++) { + Integer priv_j = Integer(32, j, PUBLIC); + Bit jgtl = priv_j.geq(priv_l); + Bit modify = jgtl & (!converged); + + i = j + 1; + g = rv1[i]; + y = w[i]; + h = s * g; + g = c * g; + z = BuildPythagCircuit(&f, &h); + rv1[j] = rv1[j].If(modify, z); + c = c.If(modify, f / z); + s = s.If(modify, h / z); + f = f.If(modify, x * c + g * s); + g = g * c - x * s; + h = y * s; + y = y * c; + for (jj = 0; jj < nCols; jj++) { + x = x.If(modify, v[jj][j]); + // z = z.If(modify, v[jj][i]); + // x = v[jj][j]; + z = v[jj][i]; + // if (k==1 && modify.reveal()) { + // printf("v[%d][%d] = %d\n", jj, j, x.reveal()); + // printf("v[%d][%d] = %d\n", jj, i, z.reveal()); + // } + v[jj][j] = v[jj][j].If(modify, x * c + z * s); + v[jj][i] = v[jj][i].If(modify, z * c - x * s); + } + z = BuildPythagCircuit(&f, &h); + w[j] = w[j].If(modify, z); // rotation can be artibitrary if z=0 + /*if (z)*/ { + Bit ifz = !(z.equal(zero_gate)); + Bit ifzam = ifz & modify; + z = Float(1.0, PUBLIC) / z; // no If needed + c = c.If(ifzam, f * z); + s = s.If(ifzam, h * z); + } + f = f.If(modify, c * g + s * y); + x = x.If(modify, c * y - s * g); + + for (jj = 0; jj < nRows; jj++) { + y = a[jj][j]; + z = a[jj][i]; + a[jj][j] = a[jj][j].If(modify, y * c + z * s); + a[jj][i] = a[jj][i].If(modify, z * c - y * s); + } + } + for (int l_finder = 0; l_finder < k; l_finder++) { + Bit found_l = priv_l.equal(Integer(32, l_finder, PUBLIC)); + rv1[l_finder] = + rv1[l_finder].If(found_l & (!converged), Float(0.0, PUBLIC)); + } + rv1[k] = rv1[k].If(!converged, f); + w[k] = w[k].If(!converged, x); + } + printf("/"); + fflush(stdout); + } + +#elif PPL_FLOW == PPL_FLOW_LOOP_LEAK + for (k = nCols - 1; k >= 0; k--) { + for (its = 0; its < 30; its++) { + cout << k; + flag = 1; + for (l = k; l >= 0; l--) { // test for splitting + nm = l - 1; // note rv1(1) is always zero + Bit temp = (rv1[l] + anorm).equal(anorm); + if (temp.reveal(PUBLIC)) { + flag = 0; + break; + } + assert(nm >= 0); // sanity check, should never happen since rv1[0] = 0 + temp = (w[nm].abs() + anorm).equal(anorm); + if (temp.reveal(PUBLIC)) { + break; + } + } + + if (flag) { // cancellation of rv1(l), if l > 1 + c = Float(0.0, PUBLIC); + s = Float(1.0, PUBLIC); + for (i = l; i <= k; i++) { + f = s * rv1[i]; + rv1[i] = c * rv1[i]; + Bit temp = (f.abs() + anorm).equal(anorm); + if (temp.reveal(PUBLIC)) { + break; + } + g = w[i]; + h = BuildPythagCircuit(&f, &g); + w[i] = h; + h = Float(1.0, PUBLIC) / h; + c = g * h; + s = -f * h; + for (j = 0; j < nRows; j++) { + y = a[j][nm]; + z = a[j][i]; + a[j][nm] = y * c + z * s; + a[j][i] = z * c - y * s; + } + } + } + z = w[k]; + if (l == k) { // convergence + /*if (z < 0.0)*/ { // singular value is made nonnegative + Bit ifzlt = z.less_than(zero_gate); + Float nz = -z; + w[k] = w[k].If(ifzlt, nz); + for (j = 0; j < nCols; j++) { + Float nv = -v[j][k]; + v[j][k] = v[j][k].If(ifzlt, nv); + } + } + break; + } + if (its == 29) + printf("no convergence in 30 svdcmp iterations\n"); + // shift from bottom 2-by-2 minor + x = w[l]; + nm = k - 1; + y = w[nm]; + g = rv1[nm]; + h = rv1[k]; + f = ((y - z) * (y + z) + (g - h) * (g + h)) / + (Float(2.0, PUBLIC) * h * y); + Float one_gate = Float(1.0, PUBLIC); + g = BuildPythagCircuit(&f, &one_gate); + BuildSignCircuit(&g, &f); // modifies g + f = ((x - z) * (x + z) + h * ((y / (f + g)) - h)) / x; + // Next QR transformation + c = Float(1.0, PUBLIC); + s = Float(1.0, PUBLIC); + for (j = l; j <= nm; j++) { + i = j + 1; + g = rv1[i]; + y = w[i]; + h = s * g; + g = c * g; + z = BuildPythagCircuit(&f, &h); + rv1[j] = z; + c = f / z; + s = h / z; + f = x * c + g * s; + g = g * c - x * s; + h = y * s; + y = y * c; + for (jj = 0; jj < nCols; jj++) { + x = v[jj][j]; + z = v[jj][i]; + v[jj][j] = x * c + z * s; + v[jj][i] = z * c - x * s; + } + z = BuildPythagCircuit(&f, &h); + w[j] = z; // rotation can be artibitrary if z=0 + /*if (z)*/ { + Bit ifz = z.equal(zero_gate); + Float temp = Float(1.0, PUBLIC) / z; + z = temp.If(ifz, z); + temp = f * z; + c = temp.If(ifz, c); + temp = h * z; + s = temp.If(ifz, s); + } + f = c * g + s * y; + x = c * y - s * g; + for (jj = 0; jj < nRows; jj++) { + y = a[jj][j]; + z = a[jj][i]; + a[jj][j] = y * c + z * s; + a[jj][i] = z * c - y * s; + } + } + rv1[l] = Float(0.0, PUBLIC); + rv1[k] = f; + w[k] = x; + } + printf("/"); + fflush(stdout); + } + +#elif PPL_FLOW == PPL_FLOW_SiSL + for (k = nCols - 1; k >= 0; k--) { + for (its = 0; its < 2; its++) { + cout << k; + l = 0; + nm = -1; + flag = 0; + + z = w[k]; + if (l == k) { // convergence + /*if (z < 0.0)*/ { // singular value is made nonnegative + Bit ifzlt = z.less_than(zero_gate); + Float nz = -z; + w[k] = w[k].If(ifzlt, nz); + for (j = 0; j < nCols; j++) { + Float nv = -v[j][k]; + v[j][k] = v[j][k].If(ifzlt, nv); + } + } + break; + } + + if (k == 0) + break; + + // shift from bottom 2-by-2 minor + x = w[l]; + nm = k - 1; + y = w[nm]; + g = rv1[nm]; + h = rv1[k]; + f = ((y - z) * (y + z) + (g - h) * (g + h)) / + (Float(2.0, PUBLIC) * h * y); + Float one_gate = Float(1.0, PUBLIC); + g = BuildPythagCircuit(&f, &one_gate); + BuildSignCircuit(&g, &f); // modifies g + f = ((x - z) * (x + z) + h * ((y / (f + g)) - h)) / x; + // Next QR transformation + c = Float(1.0, PUBLIC); + s = Float(1.0, PUBLIC); + for (j = l; j <= nm; j++) { + i = j + 1; + g = rv1[i]; + y = w[i]; + h = s * g; + g = c * g; + z = BuildPythagCircuit(&f, &h); + rv1[j] = z; + c = f / z; + s = h / z; + f = x * c + g * s; + g = g * c - x * s; + h = y * s; + y = y * c; + for (jj = 0; jj < nCols; jj++) { + x = v[jj][j]; + z = v[jj][i]; + v[jj][j] = x * c + z * s; + v[jj][i] = z * c - x * s; + } + z = BuildPythagCircuit(&f, &h); + w[j] = z; // rotation can be artibitrary if z=0 + /*if (z)*/ { + Bit ifz = z.equal(zero_gate); + Float temp = Float(1.0, PUBLIC) / z; + z = temp.If(ifz, z); + temp = f * z; + c = temp.If(ifz, c); + temp = h * z; + s = temp.If(ifz, s); + } + f = c * g + s * y; + x = c * y - s * g; + for (jj = 0; jj < nRows; jj++) { + y = a[jj][j]; + z = a[jj][i]; + a[jj][j] = y * c + z * s; + a[jj][i] = z * c - y * s; + } + } + rv1[l] = Float(0.0, PUBLIC); + rv1[k] = f; + w[k] = x; + } + printf("/"); + fflush(stdout); + } +#endif + printf("\n"); + + free(rv1); + return 0; +} diff --git a/src/emp-float-server/svd.h b/src/emp-float-server/svd.h new file mode 100644 index 0000000..043904f --- /dev/null +++ b/src/emp-float-server/svd.h @@ -0,0 +1,11 @@ +#pragma once +#include "emp-sh2pc/emp-sh2pc.h" +#include "emp-tool/emp-tool.h" + +using namespace emp; + +void BuildSignCircuit(Float* a, Float* b); + +Float BuildPythagCircuit(Float* a, Float* b); + +int BuildSvdCircuit(Float** a, int nRows, int nCols, Float* w, Float** v); diff --git a/src/emp-float-server/trigfuncs.cpp b/src/emp-float-server/trigfuncs.cpp new file mode 100644 index 0000000..8b84e7a --- /dev/null +++ b/src/emp-float-server/trigfuncs.cpp @@ -0,0 +1,22 @@ +#include +#include +#include +#include + +#include "emp-sh2pc/emp-sh2pc.h" +#include "emp-tool/emp-tool.h" + +#include + +using namespace emp; +using namespace std; + +Float BuildSinCircuit(Float a) { + Float pi = Float(M_PI, PUBLIC); + return (a / pi).sin(); // circuit mutliplies by pi for some reason +} + +Float BuildCosCircuit(Float a) { + Float pi = Float(M_PI, PUBLIC); + return (a / pi).cos(); // circuit mutliplies by pi for some reason +} diff --git a/src/emp-float-server/trigfuncs.h b/src/emp-float-server/trigfuncs.h new file mode 100644 index 0000000..6966c86 --- /dev/null +++ b/src/emp-float-server/trigfuncs.h @@ -0,0 +1,15 @@ +#include +#include +#include +#include + +#include "emp-sh2pc/emp-sh2pc.h" +#include "emp-tool/emp-tool.h" + +using namespace emp; +using namespace std; + +// r=3x1, R=3x4 (only 3x3 is used here) +// values are secret, sizes are public +Float BuildSinCircuit(Float a); +Float BuildCosCircuit(Float a); diff --git a/src/emp-float-server/twonormsq.cpp b/src/emp-float-server/twonormsq.cpp new file mode 100644 index 0000000..3c47b8e --- /dev/null +++ b/src/emp-float-server/twonormsq.cpp @@ -0,0 +1,29 @@ +#include +#include +#include +#include +#include +#include "jlog.h" + +//#include +#include +#include +#include +#include + +#include "emp-sh2pc/emp-sh2pc.h" +#include "emp-tool/emp-tool.h" + +using namespace emp; +using namespace std; + +// M mxn (M is secret, m n are public) +// N mmxnn +// res mxnn +Float BuildTwoNormSqCircuit(Float vect[], int sz) { + Float sum = Float(0.0, PUBLIC); + for (int i = 0; i < sz; i++) { + sum = sum + (vect[i] * vect[i]); + } + return sum; +} diff --git a/src/emp-float-server/twonormsq.h b/src/emp-float-server/twonormsq.h new file mode 100644 index 0000000..0dbdf50 --- /dev/null +++ b/src/emp-float-server/twonormsq.h @@ -0,0 +1,7 @@ +#include "emp-sh2pc/emp-sh2pc.h" +#include "emp-tool/emp-tool.h" + +using namespace emp; +using namespace std; + +Float BuildTwoNormSqCircuit(Float vect[], int sz); diff --git a/src/emp-float-server/util.cpp b/src/emp-float-server/util.cpp new file mode 100644 index 0000000..d38becf --- /dev/null +++ b/src/emp-float-server/util.cpp @@ -0,0 +1,63 @@ +#include +#include +#include + +#include "emp-sh2pc/emp-sh2pc.h" +#include "emp-tool/emp-tool.h" + +#include +#include "util.h" + +using namespace emp; +using namespace std; + +#define BUILD_TIMING 0 +#define EXEC_TIMING 0 + +void print_float32_bits(Float a, bool mute) { + for (int i = 31; i >= 0; i--) { + if (mute) { + a[i].reveal(); + } else { + printf("%d", a[i].reveal()); + } + } + if (!mute) { + printf("\n\n"); + } +} + +// prints an arbitrary size vector to the standard output +void printFloatVector(Float* v, int size, bool mute) { + int i; + + for (i = 0; i < size; i++) { + if (mute) { + v[i].reveal(); + } else { + printf("%.8lf ", v[i].reveal(PUBLIC)); + } + } + if (!mute) { + printf("\n\n"); + } +} + +// prints an arbitrary size matrix to the standard output +void printFloatMatrix(Float** a, int rows, int cols, bool mute) { + int i, j; + + for (i = 0; i < rows; i++) { + for (j = 0; j < cols; j++) { + if (mute) { + a[i][j].reveal(); + } else { + printf("%.8lf ", a[i][j].reveal(PUBLIC)); + } + } + if (!mute) { + printf("\n"); + } + } + printf("\n"); +} diff --git a/src/emp-float-server/util.h b/src/emp-float-server/util.h new file mode 100644 index 0000000..9ae0221 --- /dev/null +++ b/src/emp-float-server/util.h @@ -0,0 +1,15 @@ +#pragma once + +#include +#include + +#include "emp-sh2pc/emp-sh2pc.h" +#include "emp-tool/emp-tool.h" + +using namespace emp; + +void print_float32_bits(Float a, bool mute = false); + +void printFloatMatrix(Float** a, int rows, int cols, bool mute = false); + +void printFloatVector(Float* v, int size, bool mute = false); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt new file mode 100644 index 0000000..29f91f7 --- /dev/null +++ b/test/CMakeLists.txt @@ -0,0 +1,5 @@ +add_subdirectory(common-test/) +add_subdirectory(cleartext-fixed/) +add_subdirectory(aby-float/) +add_subdirectory(emp-float/) +add_subdirectory(emp-fixed/) diff --git a/test/aby-float/CMakeLists.txt b/test/aby-float/CMakeLists.txt new file mode 100644 index 0000000..3e6b125 --- /dev/null +++ b/test/aby-float/CMakeLists.txt @@ -0,0 +1,26 @@ +cmake_minimum_required(VERSION 3.12) +set(common_test_dir "${CMAKE_CURRENT_SOURCE_DIR}/../common-test/") +file(GLOB_RECURSE common_test CONFIGURE_DEPENDS "${common_test_dir}/*.cpp" "${common_test_dir}/*.hpp") +list(FILTER common_test EXCLUDE REGEX ".*eth3d_tester\\.cpp$") +list(FILTER common_test EXCLUDE REGEX ".*kitti_tester\\.cpp$") + + +add_executable(aby_float_test + "${CMAKE_CURRENT_SOURCE_DIR}/test.cpp" + ${common_test}) +target_include_directories(aby_float_test PUBLIC + ${common_test_dir} + "${CMAKE_CURRENT_SOURCE_DIR}") +target_compile_options(aby_float_test PRIVATE "-Wall" "-Wextra") +target_link_libraries(aby_float_test ABY::aby ${OpenCV_LIBS} AbyFloatLocalization Catch2::Catch2WithMain) +catch_discover_tests(aby_float_test PROPERTIES TIMEOUT 4000) + + +add_executable(aby_float_eth3d_bench + "${CMAKE_CURRENT_SOURCE_DIR}/eth3d_bench.cpp" + ${common_test}) +target_include_directories(aby_float_eth3d_bench PUBLIC + ${common_test_dir} + "${CMAKE_CURRENT_SOURCE_DIR}") +target_compile_options(aby_float_eth3d_bench PRIVATE "-Wall" "-Wextra") +target_link_libraries(aby_float_eth3d_bench ABY::aby ${OpenCV_LIBS} AbyFloatLocalization) \ No newline at end of file diff --git a/test/aby-float/eth3d_bench.cpp b/test/aby-float/eth3d_bench.cpp new file mode 100644 index 0000000..4d67c16 --- /dev/null +++ b/test/aby-float/eth3d_bench.cpp @@ -0,0 +1,169 @@ +#include + +#include // dirname +#include // PATH_MAX +#include +#include +#include +#include // readlink + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +//#include +#include +#include +#include +#include +#include + +using std::vector; + +static_assert(std::is_same_v, + "ABY GN localizer function sig does not match what " + "localize_wrapper expects. Tests will fail."); + +static_assert(std::is_same_v, + "ABY LM localizer function sig does not match what " + "localize_wrapper expects. Tests will fail."); + +// These tests require cleartext functions to have this signature +typedef bool (*cleartext_localizer)( + vector objectPointsSubset, + vector imagePointsSubset, float f, float cx, float xy, + float* x, /* initial guess for { r1, r2, r3, t1, t2, t3 } */ + string logprint); + +static_assert( + std::is_same_v), cleartext_localizer>, + "cleartext GN localizer function sig does not match what " + "this test expects."); + +static_assert(std::is_same_v), cleartext_localizer>, + "cleartext LM localizer function sig does not match what " + "this test expects."); + +int main(int argc, char** argv) { + if (argc != 5) { + std::cout << "Usage: " << argv[0] + << " \n"; + return 1; + } + int num_frames = atoi(argv[2]); + int num_trials = atoi(argv[3]); + int max_num_pts = atoi(argv[4]); + + std::string lm_str("lm"); + cleartext_localizer clear_func = + lm_str == argv[1] ? lm : gaussNewton; + aby_localizer secure_func = + lm_str == argv[1] ? BuildAndRunLM : BuildAndRunGaussNewton; + std::string log_str = lm_str == argv[1] ? "lm" : "gn"; + bool silent = false; + + eth3d_test_harness( + num_frames, num_trials, max_num_pts, + [&](cv::Mat rvec, cv::Mat tvec, cv::Mat cameraMatrix, cv::Mat distCoeffs, + vector objectPointsSubset, + vector imagePointsSubset, vector& res, + const vector& groundTruthRes) { + vector initialGuess(6); + for (int i = 0; i < 3; ++i) { + initialGuess[i] = rvec.at(i); + initialGuess[i + 3] = tvec.at(i); + } + float f = cameraMatrix.at(0, 0); + float cx = cameraMatrix.at(0, 2); + float cy = cameraMatrix.at(1, 2); + + bool out = clear_func(objectPointsSubset, imagePointsSubset, f, cx, cy, + &initialGuess[0], "clearlm"); + for (int i = 0; i < 6; ++i) { + res[i] = initialGuess[i]; + } + return out; + }, + [&](cv::Mat rvec, cv::Mat tvec, cv::Mat cameraMatrix, cv::Mat distCoeffs, + vector objectPointsSubset, + vector imagePointsSubset, vector& res, + const vector& groundTruthRes) { + bool result_matches = true; + int num_pts = objectPointsSubset.size(); + + for (auto& share_type : ctypes) { + if (!silent) { + std::cout << "testing secure " << cnames[share_type] << '\n'; + } + + std::thread bob([&]() { + vector res(6, 0); + rvec = cv::Mat::zeros(3, 1, cv::DataType::type); + tvec = cv::Mat::zeros(3, 1, cv::DataType::type); + tvec.at(2) = 1; // cleartext has bug where it fails if z=0 + aby_localize_wrapper(CLIENT, share_type, rvec, tvec, cameraMatrix, + distCoeffs, objectPointsSubset, + imagePointsSubset, res, secure_func); + }); + + vector res(6, 0); + rvec = cv::Mat::zeros(3, 1, cv::DataType::type); + tvec = cv::Mat::zeros(3, 1, cv::DataType::type); + tvec.at(2) = 1; // cleartext has bug where it fails if z=0 + + time_point tic = high_resolution_clock::now(); + + aby_localize_wrapper(SERVER, share_type, rvec, tvec, cameraMatrix, + distCoeffs, objectPointsSubset, + imagePointsSubset, res, secure_func); + bob.join(); + + time_point toc = high_resolution_clock::now(); + + if (!silent) { + cout << "secure result: "; + for (auto const& f : res) + cout << f << ' '; + } + + for (int i = 0; i < 6; ++i) { + result_matches &= withinRel(res[i], groundTruthRes[i]); + } + + if (result_matches) { + if (!silent) { + cout << "aby converged!\n"; + } + std::string timer_name = "aby_" + cnames[share_type] + "_float_" + + log_str + "_time_vs_points"; +#if PPL_FLOW == PPL_FLOW_DO + timer_name += "_dataobl"; +#endif + MSG("SeNtInAl,xy,%s,%s,%d,%g\n", __FUNCTION__, timer_name.c_str(), + num_pts, + std::chrono::duration_cast(toc - tic) + .count() / + 1000000.0); + // MSG("SeNtInAl,xy,%s,%s,%d,%g\n", __FUNCTION__, + // (timer_name + "_per_loc_itr").c_str(), num_pts, + // std::chrono::duration_cast(toc - + // tic) + // .count() / + // (1000000.0 * num_loc_iterations)); + + } else { + if (!silent) { + MSG("Not printing timing - not converged.\n"); + } + } + } + return result_matches; + }, + silent); +} \ No newline at end of file diff --git a/test/aby-float/localize_wrapper.hpp b/test/aby-float/localize_wrapper.hpp new file mode 100644 index 0000000..c66afc8 --- /dev/null +++ b/test/aby-float/localize_wrapper.hpp @@ -0,0 +1,163 @@ +#include + +#include +#include +#include +#include +#include + +using std::vector; + +const std::string address = "127.0.0.1"; +constexpr const int port = 7766; +constexpr const uint32_t nthreads = 1; +constexpr const e_mt_gen_alg mt_alg = MT_OT; +const seclvl slvl = LT; // get_sec_lvl(secparam); +const vector ctypes = {/*S_ARITH,*/ S_BOOL, + S_YAO}; // no floats with arith +std::map cnames = { + {S_ARITH, "arith"}, + {S_BOOL, "bool"}, + {S_YAO, "yao"}, +}; +constexpr const uint32_t reservegates = 65536; +constexpr const uint32_t bitlen = 32; + +// TODO(jc): should return pair +typedef void (*aby_localizer)(share* s_threeDPts[], share* s_twoDPts[], + int numPts, share* s_f, share* s_cx, share* s_cy, + share* s_x[], BooleanCircuit* c, ABYParty* party, + e_role role); + +static std::string get_circuit_dir() { + char result[PATH_MAX]; + ssize_t count = readlink("/proc/self/exe", result, PATH_MAX); + const char* path; + if (count != -1) { + path = dirname(result); + } else { + std::cerr << "cant find circuit path\n"; + assert(false); + } + std::string base_path = string(path) + "/../../extern/ABY/bin/circ"; + return base_path; +} + +void aby_localize_wrapper(e_role role, e_sharing sharing, cv::Mat rvec, + cv::Mat tvec, cv::Mat cameraMatrix, + cv::Mat distCoeffs [[maybe_unused]], + vector objectPoints, + vector imagePoints, vector& res, + auto secure_localize_func) { + float f = cameraMatrix.at(0, 0); + float cx = cameraMatrix.at(0, 2); + float cy = cameraMatrix.at(1, 2); + + ABYParty* party = new ABYParty(role, address, port, slvl, bitlen, nthreads, + mt_alg, reservegates, get_circuit_dir()); + std::vector& sharings = party->GetSharings(); + Circuit* circ = sharings[sharing]->GetCircuitBuildRoutine(); + +#if PPL_FLOW == PPL_FLOW_LOOP_LEAK || PPL_FLOW == PPL_FLOW_SiSL + // always use boolean shares (not yao) so BuildAndRunLM can get raw shares. + // this is because you cant use Y2B shares on Yao input gates. + Circuit* bc = sharings[S_BOOL]->GetCircuitBuildRoutine(); +#else + Circuit* bc = circ; +#endif + + assert(objectPoints.size() == imagePoints.size()); + int numPts = objectPoints.size(); + + // Allocate space for shares + share** s_objectPoints = new share*[4 * numPts]; + share** s_imagePoints = new share*[3 * numPts]; + share* s_f; + share* s_cx; + share* s_cy; + share** s_x = new share*[6]; + + // Prepare inputs + if (role == SERVER) { + // float one=1.0; + for (int p = 0; p < numPts; p++) { + s_objectPoints[p] = + bc->PutINGate((uint32_t*)&objectPoints[p].x, bitlen, role); + } + for (int p = 0; p < numPts; p++) { + s_objectPoints[numPts + p] = + bc->PutINGate((uint32_t*)&objectPoints[p].y, bitlen, role); + } + for (int p = 0; p < numPts; p++) { + s_objectPoints[(2 * numPts) + p] = + bc->PutINGate((uint32_t*)&objectPoints[p].z, bitlen, role); + } + // for(int p=0; pPutINGate((uint32_t*) &one, bitlen, + // role); + // } + for (int p = 0; p < numPts; p++) { + s_imagePoints[p] = + bc->PutINGate((uint32_t*)&imagePoints[p].x, bitlen, role); + } + for (int p = 0; p < numPts; p++) { + s_imagePoints[numPts + p] = + bc->PutINGate((uint32_t*)&imagePoints[p].y, bitlen, role); + } + // for(int p=0; pPutINGate((uint32_t*) &one, bitlen, + // role); + // } + s_f = bc->PutINGate((uint32_t*)&f, bitlen, role); + s_cx = bc->PutINGate((uint32_t*)&cx, bitlen, role); + s_cy = bc->PutINGate((uint32_t*)&cy, bitlen, role); + for (int p = 0; p < 3; p++) { + s_x[p] = bc->PutINGate((uint32_t*)&rvec.at(p), bitlen, role); + } + for (int p = 0; p < 3; p++) { + s_x[p + 3] = bc->PutINGate((uint32_t*)&tvec.at(p), bitlen, role); + } + } else { + for (int p = 0; p < 3 * numPts; p++) { // ignore constant 1s + s_objectPoints[p] = bc->PutDummyINGate(bitlen); + } + for (int p = 0; p < 2 * numPts; p++) { // ignore constant 1s + s_imagePoints[p] = bc->PutDummyINGate(bitlen); + } + s_f = bc->PutDummyINGate(bitlen); + s_cx = bc->PutDummyINGate(bitlen); + s_cy = bc->PutDummyINGate(bitlen); + for (int p = 0; p < 6; p++) { + s_x[p] = bc->PutDummyINGate(bitlen); + } + } + + secure_localize_func(s_objectPoints, s_imagePoints, numPts, s_f, s_cx, s_cy, + s_x, (BooleanCircuit*)circ, party, role); + + for (int i = 0; i < 6; i++) { + share* temp = s_x[i]; + s_x[i] = bc->PutOUTGate(s_x[i], ALL); + delete temp; + } + + party->ExecCircuit(); + + if (role == SERVER) { + cout << "secure result: "; + } + for (int i = 0; i < 6; i++) { + uint32_t* output; + uint32_t out_bitlen, out_nvals; + s_x[i]->get_clear_value_vec(&output, &out_bitlen, &out_nvals); + if (role == SERVER) { + cout << *(float*)output << " "; + } + res[i] = *(float*)output; + } + if (role == SERVER) { + cout << '\n'; + } + + delete party; +} \ No newline at end of file diff --git a/test/aby-float/test.cpp b/test/aby-float/test.cpp new file mode 100644 index 0000000..ed250eb --- /dev/null +++ b/test/aby-float/test.cpp @@ -0,0 +1,985 @@ +#include + +#include +#include +#include +#include +#include +#include +#include + +#include // dirname +#include // PATH_MAX +#include +#include +#include +#include +#include // readlink +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using Catch::Matchers::WithinAbs; +using Catch::Matchers::WithinRel; +using std::cout; +using std::map; +using std::vector; + +enum class TRIG_FUNC : uint8_t { + SIN, + COS, +}; + +void aby_trig(e_role role, e_sharing sharing, float angle, TRIG_FUNC tf) { + ABYParty* party = new ABYParty(role, address, port, slvl, bitlen, nthreads, + mt_alg, reservegates, get_circuit_dir()); + std::vector& sharings = party->GetSharings(); + Circuit* circ = sharings[sharing]->GetCircuitBuildRoutine(); + + share* s_in; + if (role == SERVER) { + s_in = circ->PutINGate((uint32_t*)&angle, 32, role); + } else { + s_in = circ->PutDummyINGate(32); + } + + share* s_res = NULL; + share* res = NULL; // stores plaintext + float cleartext_res; + if (tf == TRIG_FUNC::SIN) { + s_res = BuildSinCircuit(s_in, (BooleanCircuit*)circ); + cleartext_res = sin(angle); + } else if (tf == TRIG_FUNC::COS) { + s_res = BuildCosCircuit(s_in, (BooleanCircuit*)circ); + cleartext_res = cos(angle); + } else { + assert(false); + } + + res = circ->PutOUTGate(s_res, ALL); + party->ExecCircuit(); + assert(res != NULL); + + uint32_t* output; + uint32_t out_bitlen, out_nvals; + res->get_clear_value_vec(&output, &out_bitlen, &out_nvals); + + REQUIRE(out_bitlen == 32); + REQUIRE(out_nvals == 1); + REQUIRE_THAT(*(float*)output, WithinAbs(cleartext_res, float_test_epsilon)); + + delete s_in; + delete s_res; + delete res; + delete party; +} + +TEST_CASE("ABY trig functions are computed", "[aby_trig]") { + float angle = .2; + + std::thread bob([angle]() { + for (auto tfunc : {TRIG_FUNC::SIN, TRIG_FUNC::COS}) { + for (auto ctype : ctypes) { + aby_trig(CLIENT, ctype, angle, tfunc); + aby_trig(CLIENT, ctype, angle, tfunc); + } + } + }); + + for (auto tfunc : {TRIG_FUNC::SIN, TRIG_FUNC::COS}) { + for (auto ctype : ctypes) { + aby_trig(SERVER, ctype, angle, tfunc); + aby_trig(SERVER, ctype, angle, tfunc); + } + } + bob.join(); +} + +void aby_matmult(e_role role, e_sharing sharing, cv::Mat A, cv::Mat B) { + ABYParty* party = new ABYParty(role, address, port, slvl, bitlen, nthreads, + mt_alg, reservegates, get_circuit_dir()); + std::vector& sharings = party->GetSharings(); + Circuit* circ = sharings[sharing]->GetCircuitBuildRoutine(); + + int m = A.rows, n = A.cols, mm = B.rows, nn = B.cols; + share *s_A[m * n], *s_B[mm * nn]; + + for (int i = 0; i < m * n; ++i) { + if (role == SERVER) + s_A[i] = circ->PutINGate((uint32_t*)&A.at(i), 32, role); + else + s_A[i] = circ->PutDummyINGate(32); + } + + for (int i = 0; i < mm * nn; ++i) { + if (role == SERVER) + s_B[i] = circ->PutINGate((uint32_t*)&B.at(i), 32, role); + else + s_B[i] = circ->PutDummyINGate(32); + } + + share* res[m * nn]; + share* s_out[m * nn]; // stores plaintext + BuildMatmultCircuit(s_A, m, n, s_B, mm, nn, res, (BooleanCircuit*)circ); + + for (int i = 0; i < m * nn; ++i) { + s_out[i] = circ->PutOUTGate(res[i], ALL); + } + + party->ExecCircuit(); + + cv::Mat C = cv::Mat::zeros(m, nn, cv::DataType::type); + matmult(A.ptr(), A.rows, A.cols, B.ptr(), B.rows, B.cols, + C.ptr()); + + uint32_t* output; + uint32_t out_bitlen, out_nvals; + for (int i = 0; i < m * nn; i++) { + s_out[i]->get_clear_value_vec(&output, &out_bitlen, &out_nvals); + REQUIRE(out_bitlen == 32); + REQUIRE(out_nvals == 1); + REQUIRE_THAT(*(float*)output, + WithinAbs(C.ptr()[i], float_test_epsilon)); + } + delete party; +} + +TEST_CASE("ABY matrix multiply is computed", "[aby_mat]") { + cv::Mat A = cv::Mat::ones(3, 5, cv::DataType::type); + cv::Mat B = cv::Mat::ones(5, 4, cv::DataType::type) * 2; + + std::thread bob([A, B]() { + for (auto ctype : ctypes) { + aby_matmult(CLIENT, ctype, A, B); + } + }); + + for (auto ctype : ctypes) { + aby_matmult(SERVER, ctype, A, B); + } + bob.join(); +} + +void aby_rod(e_role role, e_sharing sharing, float* r) { + ABYParty* party = new ABYParty(role, address, port, slvl, bitlen, nthreads, + mt_alg, reservegates, get_circuit_dir()); + std::vector& sharings = party->GetSharings(); + Circuit* circ = sharings[sharing]->GetCircuitBuildRoutine(); + + share* s_r[3]; + + if (role == SERVER) { + s_r[0] = circ->PutINGate((uint32_t*)&r[0], 32, role); + s_r[1] = circ->PutINGate((uint32_t*)&r[1], 32, role); + s_r[2] = circ->PutINGate((uint32_t*)&r[2], 32, role); + } else { + s_r[0] = circ->PutDummyINGate(32); + s_r[1] = circ->PutDummyINGate(32); + s_r[2] = circ->PutDummyINGate(32); + } + + share* s_R[12]; + share* R[12]; // stores plaintext + BuildRodriguesCircuit(s_r, s_R, (BooleanCircuit*)circ); + + for (int i = 0; i < 12; ++i) { + if (i == 3 || i == 7 || i == 11) + continue; + R[i] = circ->PutOUTGate(s_R[i], ALL); + } + + party->ExecCircuit(); + + cv::Mat clear_res = cv::Mat::zeros(3, 4, cv::DataType::type); + rodrigues(r, clear_res.ptr()); + + uint32_t* output; + uint32_t out_bitlen, out_nvals; + for (int i = 0; i < 12; i++) { + if (i == 3 || i == 7 || i == 11) + continue; + assert(R[i] != NULL); + R[i]->get_clear_value_vec(&output, &out_bitlen, &out_nvals); + REQUIRE(out_bitlen == 32); + REQUIRE(out_nvals == 1); + REQUIRE_THAT(*(float*)output, + WithinAbs(clear_res.ptr()[i], float_test_epsilon)); + } + + delete party; +} + +TEST_CASE("ABY rodriguez transformation is computed", "[aby_rod]") { + float r[3] = {.2, .4, .2}; // 3x1 + + std::thread bob([&r]() { + for (auto ctype : ctypes) { + aby_rod(CLIENT, ctype, r); + } + }); + + for (auto ctype : ctypes) { + aby_rod(SERVER, ctype, r); + } + bob.join(); +} + +// From points, first estimate the pose using opencv. +// Then reproject using the estimated pose +// and compare to the origional points. +void aby_proj(e_role role, e_sharing sharing, cv::Mat cameraMatrix, + cv::Mat distCoeffs, vector imagePoints, + vector objectPoints) { + ABYParty* party = new ABYParty(role, address, port, slvl, bitlen, nthreads, + mt_alg, reservegates, get_circuit_dir()); + std::vector& sharings = party->GetSharings(); + Circuit* circ = sharings[sharing]->GetCircuitBuildRoutine(); + int numPoints = imagePoints.size(); + + // Find rotation and translation + cv::Mat rvec = cv::Mat::zeros(3, 1, cv::DataType::type); + cv::Mat tvec = cv::Mat::zeros(3, 1, cv::DataType::type); + cv::solvePnP(objectPoints, imagePoints, cameraMatrix, distCoeffs, rvec, tvec, + false, cv::SOLVEPNP_ITERATIVE); + + cv::Mat x = cv::Mat(6, 1, cv::DataType::type); + rvec.copyTo(x(cv::Range(0, 3), cv::Range(0, 1))); + tvec.copyTo(x(cv::Range(3, 6), cv::Range(0, 1))); + + // cleartext projection + cv::Mat matOP = cv::Mat(objectPoints, false); + matOP = matOP.reshape(1, matOP.total()); + matOP.convertTo(matOP, cv::DataType::type); + cv::Mat P = cv::Mat::ones(4, numPoints, cv::DataType::type); + P(cv::Range(0, 3), cv::Range(0, numPoints)) = + matOP.t(); // copy points into new matrix + cv::Mat projected = cv::Mat(3, numPoints, cv::DataType::type); + projectPoints(P.ptr(), x.ptr(), cameraMatrix.ptr(), + projected.ptr(), numPoints); + for (int i = 0; i < numPoints; ++i) { + REQUIRE_THAT(imagePoints[i].x, + WithinAbs(projected.at(0, i), reprojection_tol)); + REQUIRE_THAT(imagePoints[i].y, + WithinAbs(projected.at(1, i), reprojection_tol)); + } + + share* s_P[4 * numPoints]; + share* s_x[6]; + share* s_K[9]; + + float one = 1; + share* one_gate = circ->PutCONSGate((uint32_t*)&one, bitlen); + + if (role == SERVER) { + for (int i = 0; i < 9; i++) { + s_K[i] = circ->PutINGate((uint32_t*)&cameraMatrix.at(i), 32, role); + } + for (int i = 0; i < 6; i++) { + s_x[i] = circ->PutINGate((uint32_t*)&x.at(i), 32, role); + } + for (int i = 0; i < 3 * numPoints; i++) { + s_P[i] = circ->PutINGate((uint32_t*)&P.at(i), 32, role); + } + // in gate for contant 1s + for (int i = 3 * numPoints; i < 4 * numPoints; i++) { + s_P[i] = circ->PutCONSGate((uint32_t*)&one, bitlen); + } + } else { + for (int i = 0; i < 9; i++) { + s_K[i] = circ->PutDummyINGate(32); + } + for (int i = 0; i < 6; i++) { + s_x[i] = circ->PutDummyINGate(32); + } + for (int i = 0; i < 3 * numPoints; i++) { + s_P[i] = circ->PutDummyINGate(32); + } + // in gate for contant 1s + for (int i = 3 * numPoints; i < 4 * numPoints; i++) { + s_P[i] = circ->PutCONSGate((uint32_t*)&one, bitlen); + } + } + + share* + s_res[3 * numPoints]; // dont forget extra space for homog constant 1's + share* res[3 * numPoints]; // stores plaintext + BuildProjectPointsCircuit(s_P, s_x, s_K, s_res, numPoints, + (BooleanCircuit*)circ); + + for (int i = 0; i < 2 * numPoints; ++i) { + res[i] = circ->PutOUTGate(s_res[i], ALL); + } + + party->ExecCircuit(); + + uint32_t* output; + uint32_t out_bitlen, out_nvals; + for (int i = 0; i < numPoints; ++i) { + assert(res[i] != NULL); + res[i]->get_clear_value_vec(&output, &out_bitlen, &out_nvals); + REQUIRE(out_bitlen == 32); + REQUIRE(out_nvals == 1); + REQUIRE_THAT(*(float*)output, + WithinAbs(imagePoints[i].x, reprojection_tol)); + + assert(res[i + numPoints] != NULL); + res[i + numPoints]->get_clear_value_vec(&output, &out_bitlen, &out_nvals); + REQUIRE(out_bitlen == 32); + REQUIRE(out_nvals == 1); + REQUIRE_THAT(*(float*)output, + WithinAbs(imagePoints[i].y, reprojection_tol)); + } + + delete party; +} + +TEST_CASE("ABY point projection is computed", "[aby_proj]") { + float f = 715; + float cx = 354; + float cy = 245; + float _cM[] = {f, 0, cx, 0, f, cy, 0, 0, 1}; + cv::Mat cameraMatrix = cv::Mat(3, 3, cv::DataType::type, _cM); + cv::Mat distCoeffs = cv::Mat::zeros(4, 1, cv::DataType::type); + cv::Mat rvec(3, 1, cv::DataType::type); + cv::Mat tvec(3, 1, cv::DataType::type); + vector imagePoints; + vector objectPoints; + Hoffs2DPoints(imagePoints); + Hoffs3DPoints(objectPoints); + + std::thread bob([&]() { + for (auto ctype : ctypes) { + aby_proj(CLIENT, ctype, cameraMatrix, distCoeffs, imagePoints, + objectPoints); + } + }); + + for (auto ctype : ctypes) { + aby_proj(SERVER, ctype, cameraMatrix, distCoeffs, imagePoints, + objectPoints); + } + bob.join(); +} + +void aby_svd_sign(e_role role, e_sharing sharing, float a, float b) { + ABYParty* party = new ABYParty(role, address, port, slvl, bitlen, nthreads, + mt_alg, reservegates, get_circuit_dir()); + std::vector& sharings = party->GetSharings(); + Circuit* circ = sharings[sharing]->GetCircuitBuildRoutine(); + + share* s_a; + share* s_b; + if (role == SERVER) { + s_a = circ->PutINGate((uint32_t*)&a, 32, role); + s_b = circ->PutINGate((uint32_t*)&b, 32, role); + } else { + s_a = circ->PutDummyINGate(32); + s_b = circ->PutDummyINGate(32); + } + + share *out_a, *out_b; + BuildSignCircuit(s_a, s_b, (BooleanCircuit*)circ); + out_a = circ->PutOUTGate(s_a, ALL); + out_b = circ->PutOUTGate(s_b, ALL); + party->ExecCircuit(); + + uint32_t* output; + uint32_t out_bitlen, out_nvals; + + float cleartext_res = mysign(a, b); + + out_a->get_clear_value_vec(&output, &out_bitlen, &out_nvals); + REQUIRE(out_bitlen == 32); + REQUIRE(out_nvals == 1); + REQUIRE_THAT(*(float*)output, WithinAbs(cleartext_res, float_test_epsilon)); + + out_b->get_clear_value_vec(&output, &out_bitlen, &out_nvals); + REQUIRE(out_bitlen == 32); + REQUIRE(out_nvals == 1); + REQUIRE_THAT(*(float*)output, WithinAbs(b, float_test_epsilon)); + + delete party; +} + +TEST_CASE("ABY SVD sign function is computed", "[aby_svd]") { + float a = 2.40, b = 0; + std::thread bob([&]() { + for (auto ctype : ctypes) { + aby_svd_sign(CLIENT, ctype, a, b); + } + }); + + for (auto ctype : ctypes) { + aby_svd_sign(SERVER, ctype, a, b); + } + bob.join(); +} + +void aby_svd_pythag(e_role role, e_sharing sharing, float a, float b) { + ABYParty* party = new ABYParty(role, address, port, slvl, bitlen, nthreads, + mt_alg, reservegates, get_circuit_dir()); + std::vector& sharings = party->GetSharings(); + Circuit* circ = sharings[sharing]->GetCircuitBuildRoutine(); + + share* s_a; + share* s_b; + + if (role == SERVER) { + s_a = circ->PutINGate((uint32_t*)&a, 32, role); + s_b = circ->PutINGate((uint32_t*)&b, 32, role); + } else { + s_a = circ->PutDummyINGate(32); + s_b = circ->PutDummyINGate(32); + } + + share* s_out; + share* out; + s_out = BuildPythagCircuit(s_a, s_b, (BooleanCircuit*)circ, bitlen); + + out = circ->PutOUTGate(s_out, ALL); + + party->ExecCircuit(); + + float cleartext_res = mypythag(a, b); + + uint32_t* output; + uint32_t out_bitlen, out_nvals; + out->get_clear_value_vec(&output, &out_bitlen, &out_nvals); + REQUIRE(out_bitlen == 32); + REQUIRE(out_nvals == 1); + REQUIRE_THAT(*(float*)output, WithinAbs(cleartext_res, float_test_epsilon)); + + delete party; +} + +TEST_CASE("ABY SVD pythag function is computed", "[aby_svd]") { + float a = -2.40, b = 96.0; + std::thread bob([&]() { + for (auto ctype : ctypes) { + aby_svd_pythag(CLIENT, ctype, a, b); + } + }); + + for (auto ctype : ctypes) { + aby_svd_pythag(SERVER, ctype, a, b); + } + bob.join(); +} + +void aby_svd(e_role role, e_sharing sharing, float** in, int m, int n) { + // OpenCV SVD - requires linearize + float* cvin = new float[m * n]; + for (int i = 0; i < m; i++) { + for (int j = 0; j < n; j++) { + cvin[i * n + j] = in[i][j]; + } + } + cv::Mat cvinM = cv::Mat(m, n, cv::DataType::type, in); + cv::SVD cvsvd(cvinM, cv::SVD::FULL_UV); // constructor + if (role == SERVER) { + cout << "opencv result\n" + << cvsvd.u << '\n' + << cvsvd.w << '\n' + << cvsvd.vt << '\n'; + } + delete[] cvin; + + ABYParty* party = new ABYParty(role, address, port, slvl, bitlen, nthreads, + mt_alg, reservegates, get_circuit_dir()); + std::vector& sharings = party->GetSharings(); + Circuit* circ = sharings[sharing]->GetCircuitBuildRoutine(); + +#if PPL_FLOW == PPL_FLOW_LOOP_LEAK + // always use boolean shares (not yao) so BuildAndRunSvd can get raw shares. + // this is because you cant use Y2B shares on Yao input gates. + Circuit* bc = sharings[S_BOOL]->GetCircuitBuildRoutine(); +#else + Circuit* bc = circ; +#endif + + // cleartext SVD - svd overwrites with u matrix, so copy + float** a = new float*[m]; + for (int i = 0; i < m; i++) { + a[i] = new float[n]; + for (int j = 0; j < n; j++) { + a[i][j] = in[i][j]; + } + } + float* w = new float[n](); // aka sigma, only diag + float** v = new float*[n]; // nxn + for (int i = 0; i < n; i++) + v[i] = new float[n](); + + svdcmp(a, m, n, w, v); + if (role == SERVER) { + cout << "cleartext" << '\n'; + printMatrix("a", a, m, n); + printVector("w", w, n); + printMatrix("v", v, n, n); + } + + share*** s_a = new share**[m]; + for (int i = 0; i < m; i++) { + s_a[i] = new share*[n]; + } + share** s_w = new share*[n]; + share*** s_v = new share**[n]; + for (int i = 0; i < n; i++) { + s_v[i] = new share*[n]; + } + + // create shared output from plaintext input + // initialize input shares + // always use boolean shares (not yao) so BuildAndRunSvd can get raw shares. + // this is because you cant use Y2B shares on Yao input gates. + if (role == SERVER) { + for (int i = 0; i < m; i++) { + for (int j = 0; j < n; j++) { + s_a[i][j] = bc->PutINGate((uint32_t*)&in[i][j], 32, role); + } + } + } else { + for (int i = 0; i < m; i++) { + for (int j = 0; j < n; j++) { + s_a[i][j] = bc->PutDummyINGate(32); + } + } + } + + BuildAndRunSvd(s_a, m, n, s_w, s_v, (BooleanCircuit*)circ, party, role); + + // shared outputs from circuit + // again always use boolean shares (not yao) because output is coming + // directly from a boolean circuit ->PutSharedInGate() + for (int i = 0; i < m; i++) { + for (int j = 0; j < n; j++) { + share* temp = s_a[i][j]; + s_a[i][j] = circ->PutOUTGate(temp, ALL); + delete temp; + } + } + for (int i = 0; i < n; i++) { + share* temp = s_w[i]; + s_w[i] = circ->PutOUTGate(temp, ALL); + delete temp; + } + for (int i = 0; i < n; i++) { + for (int j = 0; j < n; j++) { + share* temp = s_v[i][j]; + s_v[i][j] = circ->PutOUTGate(temp, ALL); + delete temp; + } + } + // run the circuit + party->ExecCircuit(); + + // get cleartext output + for (int i = 0; i < m; i++) { + for (int j = 0; j < n; j++) { + // compare cleartext to opencv + // REQUIRE_THAT(a[i][j], WithinAbs(cvsvd.u.at(i,j), svd_tol)); + + // compare secure to cleartext + float* valptr = (float*)s_a[i][j]->get_clear_value_ptr(); + REQUIRE_THAT(*(float*)valptr, WithinAbs(a[i][j], svd_tol)); + // in[i][j] = *valptr; + free(valptr); + delete s_a[i][j]; + } + delete[] s_a[i]; + } + delete[] s_a; + for (int i = 0; i < n; i++) { + // compare cleartext to opencv + // REQUIRE_THAT(w[i], WithinAbs(cvsvd.w.at(i), svd_tol)); + + float* valptr = (float*)s_w[i]->get_clear_value_ptr(); + REQUIRE_THAT(*(float*)valptr, WithinAbs(w[i], svd_tol)); + // w[i] = *valptr; + free(valptr); + delete s_w[i]; + } + delete[] s_w; + for (int i = 0; i < n; i++) { + for (int j = 0; j < n; j++) { + // compare cleartext to opencv + // REQUIRE_THAT(v[i][j], WithinAbs(cvsvd.vt.at(i,j), svd_tol)); + + float* valptr = (float*)s_v[i][j]->get_clear_value_ptr(); + REQUIRE_THAT(*(float*)valptr, WithinAbs(v[i][j], svd_tol)); + // v[i][j] = *valptr; + free(valptr); + delete s_v[i][j]; + } + delete[] s_v[i]; + } + delete[] s_v; + delete[] w; + delete[] v; + delete party; +} + +TEST_CASE("ABY large SVD function is computed", "[aby_svd]") { + int m = 12, n = 6; + float one_d_M[] = { + -9.1552734, 149.53613, -59.509277, 18.310547, 0, 3.0517578, + -164.79492, -64.086914, -47.302246, 0, 19.836426, 1.5258789, + 27.46582, 77.819824, 50.354004, 22.888184, 0, 1.5258789, + -56.45752, -42.724609, -53.405762, 0, 22.888184, 3.0517578, + 45.776367, 79.345703, 79.345703, 21.362305, 0, 0, + -32.806396, -13.73291, -20.599365, 0, 24.414062, 4.5776367, + -21.362305, 108.3374, -93.078613, 16.784668, 0, 3.0517578, + -152.58789, -45.776367, -10.681152, 0, 19.836426, 3.0517578, + 6.1035156, 39.672852, 3.0517578, 21.362305, 0, 0, + -39.672852, -21.362305, -16.784668, 0, 22.888184, 1.5258789, + 24.414062, 48.828125, 27.46582, 24.414062, 0, 0, + -13.73291, 6.1035156, 12.207031, 0, 22.888184, 1.5258789}; + float** in = new float*[m]; + for (int i = 0; i < m; i++) { + in[i] = new float[n](); + for (int j = 0; j < n; j++) { + in[i][j] = one_d_M[i * n + j]; + } + } + + std::thread bob([&]() { + for (auto ctype : ctypes) { + aby_svd(CLIENT, ctype, in, m, n); + } + }); + + for (auto ctype : ctypes) { + aby_svd(SERVER, ctype, in, m, n); + } + bob.join(); +} + +void aby_invert(e_role role, e_sharing sharing, float* in, int m, int n) { + cv::Mat cvM = cv::Mat(m, n, cv::DataType::type, in); + cv::Mat cvres; + invert(cvM, cvres, cv::DECOMP_SVD); + cout << "opencv result:\n" << cvres << endl; + + float** M = new float*[m]; + for (int i = 0; i < m; i++) { + M[i] = new float[n]; + for (int j = 0; j < n; j++) { + M[i][j] = in[i * n + j]; + } + } + + // res is nxm (not mxn) + float** res = new float*[n]; + for (int i = 0; i < n; i++) { + res[i] = new float[m]; + } + + myinvert(M, m, n, res); + printMatrix("cleartext result:", res, n, m); + + // check cleartext vs opencv + for (int i = 0; i < n; i++) { + for (int j = 0; j < m; j++) { + REQUIRE_THAT(cvres.at(i, j), + WithinAbs(res[i][j], float_test_epsilon)); + } + } + + ABYParty* party = new ABYParty(role, address, port, slvl, bitlen, nthreads, + mt_alg, reservegates, get_circuit_dir()); + std::vector& sharings = party->GetSharings(); + Circuit* circ = sharings[sharing]->GetCircuitBuildRoutine(); + +#if PPL_FLOW == PPL_FLOW_LOOP_LEAK + // always use boolean shares (not yao) so BuildAndRunSvd can get raw shares. + // this is because you cant use Y2B shares on Yao input gates. + Circuit* bc = sharings[S_BOOL]->GetCircuitBuildRoutine(); +#else + Circuit* bc = circ; +#endif + + // share arrays + share*** s_in = new share**[m]; + for (int i = 0; i < m; i++) { + s_in[i] = new share*[n]; + } + + // initialize input shares + // always use boolean shares (not yao) so BuildAndRunSvd (called in + // BuildInvertCircuit) can get raw shares. This is because you cant use Y2B + // shares on Yao input gates. + if (role == SERVER) { + for (int i = 0; i < m; i++) { + for (int j = 0; j < n; j++) { + s_in[i][j] = bc->PutINGate((uint32_t*)&M[i][j], 32, role); + } + } + } else { + for (int i = 0; i < m; i++) { + for (int j = 0; j < n; j++) { + s_in[i][j] = bc->PutDummyINGate(32); + } + } + } + + // res is nxm not mxn + share*** s_res = new share**[n]; + for (int i = 0; i < n; i++) { + s_res[i] = new share*[m]; + } + + BuildInvertCircuit(s_in, m, n, s_res, (BooleanCircuit*)circ, party, role); + + for (int i = 0; i < n; i++) { + for (int j = 0; j < m; j++) { + share* temp = s_res[i][j]; + s_res[i][j] = circ->PutOUTGate(s_res[i][j], ALL); + delete temp; + } + } + + party->ExecCircuit(); + + for (int i = 0; i < n; i++) { + for (int j = 0; j < m; j++) { + uint32_t* output; + uint32_t out_bitlen, out_nvals; + s_res[i][j]->get_clear_value_vec(&output, &out_bitlen, &out_nvals); + REQUIRE(out_bitlen == 32); + REQUIRE(out_nvals == 1); + REQUIRE_THAT(*(float*)output, WithinAbs(res[i][j], float_test_epsilon)); + } + } + + delete party; +} + +TEST_CASE("ABY small invert function is computed", "[aby_small_invert]") { + int m = 7, n = 3; + float one_d_M[] = {3, 0, + 1, // toy example + 0, 0, 0, 0, 4, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3, 0, 0, 0}; + std::thread bob([&]() { + for (auto ctype : ctypes) { + aby_invert(CLIENT, ctype, one_d_M, m, n); + } + }); + + for (auto ctype : ctypes) { + aby_invert(SERVER, ctype, one_d_M, m, n); + } + bob.join(); +} + +TEST_CASE("ABY large invert function is computed", "[aby_large_invert]") { + int m = 12, n = 6; + float one_d_M[] = { + -9.1552734, 149.53613, -59.509277, 18.310547, 0, 3.0517578, + -164.79492, -64.086914, -47.302246, 0, 19.836426, 1.5258789, + 27.46582, 77.819824, 50.354004, 22.888184, 0, 1.5258789, + -56.45752, -42.724609, -53.405762, 0, 22.888184, 3.0517578, + 45.776367, 79.345703, 79.345703, 21.362305, 0, 0, + -32.806396, -13.73291, -20.599365, 0, 24.414062, 4.5776367, + -21.362305, 108.3374, -93.078613, 16.784668, 0, 3.0517578, + -152.58789, -45.776367, -10.681152, 0, 19.836426, 3.0517578, + 6.1035156, 39.672852, 3.0517578, 21.362305, 0, 0, + -39.672852, -21.362305, -16.784668, 0, 22.888184, 1.5258789, + 24.414062, 48.828125, 27.46582, 24.414062, 0, 0, + -13.73291, 6.1035156, 12.207031, 0, 22.888184, 1.5258789}; + std::thread bob([&]() { + for (auto ctype : ctypes) { + aby_invert(CLIENT, ctype, one_d_M, m, n); + } + }); + + for (auto ctype : ctypes) { + aby_invert(SERVER, ctype, one_d_M, m, n); + } + bob.join(); +} + +void aby_twonorm(e_role role, e_sharing sharing, float* vect, int sz) { + ABYParty* party = new ABYParty(role, address, port, slvl, bitlen, nthreads, + mt_alg, reservegates, get_circuit_dir()); + std::vector& sharings = party->GetSharings(); + Circuit* circ = sharings[sharing]->GetCircuitBuildRoutine(); + + share** s_vect = new share*[sz]; + + for (int i = 0; i < sz; ++i) { + if (role == SERVER) + s_vect[i] = circ->PutINGate((uint32_t*)&vect[i], 32, role); + else + s_vect[i] = circ->PutDummyINGate(32); + } + + share* s_res = BuildTwoNormSqCircuit(s_vect, sz, (BooleanCircuit*)circ); + + share* s_out = circ->PutOUTGate(s_res, ALL); + + party->ExecCircuit(); + + float cleartext_res = twonormsq(vect, sz); + + uint32_t* output; + uint32_t out_bitlen, out_nvals; + s_out->get_clear_value_vec(&output, &out_bitlen, &out_nvals); + + REQUIRE(out_bitlen == 32); + REQUIRE(out_nvals == 1); + REQUIRE_THAT(*(float*)output, WithinAbs(cleartext_res, float_test_epsilon)); + + delete party; + delete[] s_vect; + delete s_out; +} + +TEST_CASE("ABY two norm is computed", "[aby_twonorm]") { + int sz = 5; + float vec[] = {3.1, 5, 2, 4, 1}; + std::thread bob([&]() { + for (auto ctype : ctypes) { + aby_twonorm(CLIENT, ctype, vec, sz); + } + }); + + for (auto ctype : ctypes) { + aby_twonorm(SERVER, ctype, vec, sz); + } + bob.join(); +} + +void aby_localize(e_role role, e_sharing sharing, cv::Mat rvec, cv::Mat tvec, + cv::Mat cameraMatrix, cv::Mat distCoeffs, + vector objectPoints, + vector imagePoints, auto cleartext_localize_func, + auto secure_localize_func) { + cv::Mat cvrvec = rvec.clone(); + cv::Mat cvtvec = tvec.clone(); + cv::solvePnP(objectPoints, imagePoints, cameraMatrix, distCoeffs, cvrvec, + cvtvec, false, cv::SOLVEPNP_ITERATIVE); + if (role == SERVER) { + cout << "opencv res: " << cvrvec.t() << cvtvec.t() << '\n'; + } + + float rt[6] = {rvec.at(0), rvec.at(1), rvec.at(2), + tvec.at(0), tvec.at(1), tvec.at(2)}; + float f = cameraMatrix.at(0, 0); + float cx = cameraMatrix.at(0, 2); + float cy = cameraMatrix.at(1, 2); + cleartext_localize_func(objectPoints, imagePoints, f, cx, cy, rt, ""); + if (role == SERVER) { + printVector("cleartext res", rt, 6); + } + + // check cleartext vs opencv + for (int i = 0; i < 3; i++) { + REQUIRE_THAT(cvrvec.at(i), + WithinRel(rt[i], localization_tol_rel) || + WithinAbs(rt[i], localization_tol_abs)); + REQUIRE_THAT(cvtvec.at(i), + WithinRel(rt[i + 3], localization_tol_rel) || + WithinAbs(rt[i + 3], localization_tol_abs)); + } + + vector res(6, 0); + aby_localize_wrapper(role, sharing, rvec, tvec, cameraMatrix, distCoeffs, + objectPoints, imagePoints, res, secure_localize_func); + + if (role == SERVER) { + cout << "secure result: "; + for (auto const& f : res) + cout << f << " "; + cout << '\n'; + } + for (int i = 0; i < 6; i++) { + REQUIRE_THAT(res[i], WithinRel(rt[i], localization_tol_rel) || + WithinAbs(rt[i], localization_tol_abs)); + } +} + +//#if PPL_FLOW != PPL_FLOW_SiSL +TEST_CASE("ABY Gauss Newton pose estimation is computed", "[aby_gn]") { + float f = 715; + float cx = 354; + float cy = 245; + float _cM[] = {f, 0, cx, 0, f, cy, 0, 0, 1}; + cv::Mat cameraMatrix = cv::Mat(3, 3, cv::DataType::type, _cM); + cv::Mat distCoeffs = cv::Mat::zeros(4, 1, cv::DataType::type); + cv::Mat rvec = cv::Mat::zeros(3, 1, cv::DataType::type); + cv::Mat tvec = cv::Mat::zeros(3, 1, cv::DataType::type); + tvec.at(2) = 1; // cleartext has bug where it fails if z=0 + vector imagePoints; + vector objectPoints; + Hoffs2DPoints(imagePoints); + Hoffs3DPoints(objectPoints); + + std::thread bob([&]() { + for (auto ctype : ctypes) { + aby_localize(CLIENT, ctype, rvec, tvec, cameraMatrix, distCoeffs, + objectPoints, imagePoints, gaussNewton, + BuildAndRunGaussNewton); + } + }); + + for (auto ctype : ctypes) { + aby_localize(SERVER, ctype, rvec, tvec, cameraMatrix, distCoeffs, + objectPoints, imagePoints, gaussNewton, + BuildAndRunGaussNewton); + } + bob.join(); +} +//#endif + +// Note this test can take around an hour +TEST_CASE("ABY Levenburg Marquardt pose estimation is computed", "[aby_lm]") { + float f = 715; + float cx = 354; + float cy = 245; + float _cM[] = {f, 0, cx, 0, f, cy, 0, 0, 1}; + cv::Mat cameraMatrix = cv::Mat(3, 3, cv::DataType::type, _cM); + cv::Mat distCoeffs = cv::Mat::zeros(4, 1, cv::DataType::type); + cv::Mat rvec = cv::Mat::zeros(3, 1, cv::DataType::type); + cv::Mat tvec = cv::Mat::zeros(3, 1, cv::DataType::type); + tvec.at(2) = 1; // cleartext has bug where it fails if z=0 + vector imagePoints; + vector objectPoints; + Hoffs2DPoints(imagePoints); + Hoffs3DPoints(objectPoints); + + std::thread bob([&]() { + for (auto ctype : ctypes) { + aby_localize(CLIENT, ctype, rvec, tvec, cameraMatrix, distCoeffs, + objectPoints, imagePoints, lm, BuildAndRunLM); + } + }); + + for (auto ctype : ctypes) { + aby_localize(SERVER, ctype, rvec, tvec, cameraMatrix, distCoeffs, + objectPoints, imagePoints, lm, BuildAndRunLM); + } + bob.join(); +} \ No newline at end of file diff --git a/test/cleartext-fixed/CMakeLists.txt b/test/cleartext-fixed/CMakeLists.txt new file mode 100644 index 0000000..7891f3b --- /dev/null +++ b/test/cleartext-fixed/CMakeLists.txt @@ -0,0 +1,42 @@ +cmake_minimum_required(VERSION 3.12) +set(common_test_dir "${CMAKE_CURRENT_SOURCE_DIR}/../common-test/") +file(GLOB_RECURSE common_test CONFIGURE_DEPENDS "${common_test_dir}/*.cpp" "${common_test_dir}/*.hpp") +list(FILTER common_test EXCLUDE REGEX ".*eth3d_tester\\.cpp$") +list(FILTER common_test EXCLUDE REGEX ".*kitti_tester\\.cpp$") + +# required for emp +set(CMAKE_C_FLAGS "-pthread -Wall -march=native -O3 -maes -mrdseed") +set(CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} -std=c++17") +set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -ggdb -fno-omit-frame-pointer") +set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}") + + +find_package(Boost 1.71.0 REQUIRED) +include_directories(${Boost_INCLUDE_DIR}) + +add_executable(gn_fixed_server_tester "${CMAKE_CURRENT_SOURCE_DIR}/test.cpp" ${common_test}) +target_include_directories(gn_fixed_server_tester PUBLIC ${common_test_dir}) +target_compile_options(gn_fixed_server_tester PRIVATE "-Wall" "-Wextra" "-DPPL_GN") +target_link_libraries(gn_fixed_server_tester ${OpenCV_LIBS} EmpFloatLocalization ${Boost_LIBRARIES}) + +add_executable(lm_fixed_server_tester "${CMAKE_CURRENT_SOURCE_DIR}/test.cpp" ${common_test}) +target_include_directories(lm_fixed_server_tester PUBLIC ${common_test_dir}) +target_compile_options(lm_fixed_server_tester PRIVATE "-Wall" "-Wextra" "-DPPL_LM") +target_link_libraries(lm_fixed_server_tester ${OpenCV_LIBS} EmpFloatLocalization ${Boost_LIBRARIES}) + + +add_executable(fixed_server_benchmark "${CMAKE_CURRENT_SOURCE_DIR}/benchmark.cpp" ${common_test}) +target_include_directories(fixed_server_benchmark PUBLIC ${common_test_dir}) +target_compile_options(fixed_server_benchmark PRIVATE "-Wall" "-Wextra" "-DPPL_GN") +target_link_libraries(fixed_server_benchmark ${OpenCV_LIBS} EmpFloatLocalization ${Boost_LIBRARIES}) + + +add_executable(emp_float_vs_fixed_benchmark_add "${CMAKE_CURRENT_SOURCE_DIR}/float_vs_fixed_speed.cpp" ${common_test}) +target_include_directories(emp_float_vs_fixed_benchmark_add PUBLIC ${common_test_dir}) +target_compile_options(emp_float_vs_fixed_benchmark_add PRIVATE "-Wall" "-Wextra" "-DTEST_ADD") +target_link_libraries(emp_float_vs_fixed_benchmark_add ${OpenCV_LIBS} EmpFloatLocalization ${Boost_LIBRARIES}) + +add_executable(emp_float_vs_fixed_benchmark_mul "${CMAKE_CURRENT_SOURCE_DIR}/float_vs_fixed_speed.cpp" ${common_test}) +target_include_directories(emp_float_vs_fixed_benchmark_mul PUBLIC ${common_test_dir}) +target_compile_options(emp_float_vs_fixed_benchmark_mul PRIVATE "-Wall" "-Wextra" "-DTEST_MUL") +target_link_libraries(emp_float_vs_fixed_benchmark_mul ${OpenCV_LIBS} EmpFloatLocalization ${Boost_LIBRARIES}) diff --git a/test/cleartext-fixed/benchmark.cpp b/test/cleartext-fixed/benchmark.cpp new file mode 100644 index 0000000..3668ec5 --- /dev/null +++ b/test/cleartext-fixed/benchmark.cpp @@ -0,0 +1,568 @@ +//#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include // dirname +#include // PATH_MAX +#include // readlink + +//#include +//#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +using std::cout; +using std::vector; + +constexpr const int port = 8080; +constexpr const int seed = 0x666; +const static constexpr float localization_tol_abs = 0.05f; +const static constexpr float localization_tol_rel = 0.05f; + +bool withinRel(float v, float t) { + return fabs(v - t) <= std::max(localization_tol_rel, + fabs(localization_tol_rel * std::max(v, t))); +} + +template +bool convert_and_localize(vector> _objectPoints, + vector> _imagePoints, float _f, + float _cx, float _cy, T* _x, + auto cleartext_localize_func, vector& gtpose, + string logprint = "") { + + T f = _f; + T cx = _cx; + T cy = _cy; + vector> objectPoints; + vector> imagePoints; + for (auto float_point : _objectPoints) { + objectPoints.push_back({float_point.x, float_point.y, float_point.z}); + } + for (auto float_point : _imagePoints) { + imagePoints.push_back({float_point.x, float_point.y}); + } + + cleartext_localize_func(objectPoints, imagePoints, f, cx, cy, _x, logprint); + + bool good = true; + for (int i = 0; i < 6; ++i) { + float fx = _x[i]; // convert to float + good &= withinRel(fx, gtpose[i]); + } + return good; +} + +int eth3d_localize(NetIO* io, int party, int num_frames, int num_trials, + int max_num_pts, bool silent, std::string log_str = "") { + setup_semi_honest(io, party); + uint32_t cv_successes = 0; + uint32_t gn_double_successes = 0; + uint32_t lm_double_successes = 0; + uint32_t gn_float_successes = 0; + uint32_t lm_float_successes = 0; + uint32_t gn_fixed32_successes = 0; + uint32_t lm_fixed32_successes = 0; + uint32_t gn_fixed64_successes = 0; + uint32_t lm_fixed64_successes = 0; + uint32_t total_runs = 0; + + float f = 3408.57; + float cx = 3114.7; + float cy = 2070.92; + float _cM[] = {f, 0, cx, 0, f, cy, 0, 0, 1}; + cv::Mat cameraMatrix = cv::Mat(3, 3, cv::DataType::type, _cM); + cv::Mat distCoeffs = cv::Mat::zeros(4, 1, cv::DataType::type); + cv::Mat rvec = cv::Mat::zeros(3, 1, cv::DataType::type); + cv::Mat tvec = cv::Mat::zeros(3, 1, cv::DataType::type); + vector imagePoints; + vector objectPoints; + + char result[PATH_MAX]; + ssize_t count = readlink("/proc/self/exe", result, PATH_MAX); + const char* path; + if (count != -1) { + path = dirname(result); + } else { + error("cant find path\n"); + return 1; + } + std::string base_path = string(path) + "/../../data-eth3d/"; + + for (uint32_t l = 0; l < eth3d_locations.size(); l++) { + // for (int l=0; l<1; l++) { + auto feats = ETH3DFeatures(base_path, eth3d_locations[l]); + + int test_num_frames = MIN(feats.numberOfFrames(), num_frames); + + for (int frame = 0; frame < test_num_frames; frame++) { + imagePoints.clear(); + objectPoints.clear(); + feats.imageFeatures(imagePoints); + feats.worldFeatures(objectPoints); + auto gtpose = feats.getGroundTruthPose(); + vector initialGuess = {0, 0, 0, 0, 0, 0}; + if (!silent) { + printVector("Ground Truth Pose [rotation; translation]: ", >pose[0], + 6); + } + + int max_image_points = imagePoints.size(); + int test_num_pts = MIN(max_image_points, max_num_pts); + + // std::random_device dev; + // std::mt19937 rng(dev()); + std::mt19937 rng(seed); + std::uniform_int_distribution dist( + 0, max_image_points); + + for (int num_pts = 6; num_pts <= test_num_pts; + num_pts += MAX((test_num_pts - 6) / 5, 1)) { // at most 5 intervals + for (int t = 0; t < num_trials; t++) { + if (!silent) { + cout << "Trial " << t << " with " << num_pts + << " randomly selected points on frame " << frame + << " from location " << l << "\n"; + } + + // randomly sample i feature pairs from total + vector imagePointsSubset; + vector objectPointsSubset; + for (int p = 0; p < num_pts; p++) { + int r = dist(rng); + imagePointsSubset.push_back(imagePoints[r]); + objectPointsSubset.push_back(objectPoints[r]); + } + + { + if (!silent) { + cout << "\ntesting opencv\n"; + } + distCoeffs = cv::Mat::zeros(4, 1, cv::DataType::type); + rvec = cv::Mat::zeros(3, 1, cv::DataType::type); + tvec = cv::Mat::zeros(3, 1, cv::DataType::type); + // OpenCV PnP method + cv::solvePnP(objectPointsSubset, imagePointsSubset, cameraMatrix, + distCoeffs, rvec, tvec, false, cv::SOLVEPNP_ITERATIVE); + if (!silent) { + cout << "opencv result:" << endl; + cout << rvec << endl; + cout << tvec << endl << endl; + } + bool fail = false; + for (int i = 0; i < 3; ++i) { + fail |= withinRel(rvec.at(i), gtpose[i]); + fail |= withinRel(tvec.at(i), gtpose[i + 3]); + } + if (!fail) { + ++cv_successes; + } else { + continue; // if opencv does not converge, ignore this run + } + } + + { + if (!silent) { + cout << "testing cleartext - gn - double\n"; + } + vector initialGuessCopy; + for (auto v : initialGuess) { + initialGuessCopy.push_back(v); + } + bool good = convert_and_localize(objectPoints, imagePoints, f, cx, + cy, &initialGuessCopy[0], + gaussNewton, gtpose); + if (!silent) { + printVector("result:\n", &initialGuessCopy[0], 6); + } + if (good) { + ++gn_double_successes; + } + } + + { + if (!silent) { + cout << "testing cleartext - lm - double\n"; + } + vector initialGuessCopy; + for (auto v : initialGuess) { + initialGuessCopy.push_back(v); + } + bool good = + convert_and_localize(objectPoints, imagePoints, f, cx, cy, + &initialGuessCopy[0], lm, gtpose); + if (!silent) { + printVector("cleartext result:\n", &initialGuessCopy[0], 6); + } + bool fail = false; + for (int i = 0; i < 6; ++i) { + fail |= withinRel(initialGuessCopy[i], gtpose[i]); + } + if (!fail) { + ++lm_double_successes; + } + } + + { + if (!silent) { + cout << "testing cleartext - gn - float\n"; + } + vector initialGuessCopy; + for (auto v : initialGuess) { + initialGuessCopy.push_back(v); + } + bool good = convert_and_localize(objectPoints, imagePoints, f, cx, + cy, &initialGuessCopy[0], + gaussNewton, gtpose); + if (!silent) { + printVector("result:\n", &initialGuessCopy[0], 6); + } + if (good) { + ++gn_float_successes; + } + } + + { + if (!silent) { + cout << "testing cleartext - lm - float\n"; + } + vector initialGuessCopy; + for (auto v : initialGuess) { + initialGuessCopy.push_back(v); + } + bool good = + convert_and_localize(objectPoints, imagePoints, f, cx, cy, + &initialGuessCopy[0], lm, gtpose); + if (!silent) { + printVector("cleartext result:\n", &initialGuessCopy[0], 6); + } + bool fail = false; + for (int i = 0; i < 6; ++i) { + fail |= withinRel(initialGuessCopy[i], gtpose[i]); + } + if (!fail) { + ++lm_float_successes; + } + } + + { + if (!silent) { + cout << "testing cleartext - gn - fixed64\n"; + } + vector> initialGuessCopy; + for (auto v : initialGuess) { + initialGuessCopy.push_back(v); + } + bool good = convert_and_localize( + objectPoints, imagePoints, f, cx, cy, &initialGuessCopy[0], + gaussNewton>, gtpose); + if (!silent) { + printVector("result:\n", &initialGuessCopy[0], 6); + } + if (good) { + ++gn_fixed64_successes; + } + } + { + if (!silent) { + cout << "testing cleartext - lm - fixed64\n"; + } + vector> initialGuessCopy; + for (auto v : initialGuess) { + initialGuessCopy.push_back(v); + } + bool good = convert_and_localize( + objectPoints, imagePoints, f, cx, cy, &initialGuessCopy[0], + lm>, gtpose); + if (!silent) { + printVector("result:\n", &initialGuessCopy[0], 6); + } + if (good) { + ++lm_fixed64_successes; + } + } + + { + if (!silent) { + cout << "testing cleartext - gn - fixed32\n"; + } + vector> initialGuessCopy; + for (auto v : initialGuess) { + initialGuessCopy.push_back(v); + } + bool good = convert_and_localize( + objectPoints, imagePoints, f, cx, cy, &initialGuessCopy[0], + gaussNewton>, gtpose); + if (!silent) { + printVector("result:\n", &initialGuessCopy[0], 6); + } + if (good) { + ++gn_fixed32_successes; + } + } + { + if (!silent) { + cout << "testing cleartext - lm - fixed64\n"; + } + vector> initialGuessCopy; + for (auto v : initialGuess) { + initialGuessCopy.push_back(v); + } + bool good = convert_and_localize( + objectPoints, imagePoints, f, cx, cy, &initialGuessCopy[0], + lm>, gtpose); + if (!silent) { + printVector("result:\n", &initialGuessCopy[0], 6); + } + if (good) { + ++lm_fixed32_successes; + } + } + + ++total_runs; + } + } + feats.nextFrame(); + } + } + if (!silent) { + MSG("SeNtInAl,xy,%s,%s,%d,%u\n", __FUNCTION__, "cv_successes", 0, + cv_successes); + MSG("SeNtInAl,xy,%s,%s,%d,%u\n", __FUNCTION__, "gn_double_successes", 0, + gn_double_successes); + MSG("SeNtInAl,xy,%s,%s,%d,%u\n", __FUNCTION__, "lm_double_successes", 0, + lm_double_successes); + MSG("SeNtInAl,xy,%s,%s,%d,%u\n", __FUNCTION__, "gn_float_successes", 0, + gn_float_successes); + MSG("SeNtInAl,xy,%s,%s,%d,%u\n", __FUNCTION__, "lm_float_successes", 0, + lm_float_successes); + MSG("SeNtInAl,xy,%s,%s,%d,%u\n", __FUNCTION__, "gn_fixed32_successes", 0, + gn_fixed32_successes); + MSG("SeNtInAl,xy,%s,%s,%d,%u\n", __FUNCTION__, "lm_fixed32_successes", 0, + lm_fixed32_successes); + MSG("SeNtInAl,xy,%s,%s,%d,%u\n", __FUNCTION__, "gn_fixed64_successes", 0, + gn_fixed64_successes); + MSG("SeNtInAl,xy,%s,%s,%d,%u\n", __FUNCTION__, "lm_fixed64_successes", 0, + lm_fixed64_successes); + MSG("SeNtInAl,xy,%s,%s,%d,%u\n", __FUNCTION__, "total_runs", 0, total_runs); + } + finalize_semi_honest(); + return 0; +} + +int main(int argc, char** argv) { + if (argc != 4) { + MSG("Usage: %s \n", argv[0]); + return 1; + } + int num_frames = atoi(argv[1]); + int num_trials = atoi(argv[2]); + int max_num_pts = atoi(argv[3]); + + std::thread bob([&]() { + NetIO* io = new NetIO("127.0.0.1", port); + eth3d_localize(io, BOB, num_frames, num_trials, max_num_pts, true); + delete io; + }); + + NetIO* io = new NetIO(nullptr, port); + eth3d_localize(io, ALICE, num_frames, num_trials, max_num_pts, false); + bob.join(); + delete io; +} + +// using namespace std; +// +// typedef fixed_point baset; +// +// int main(int argc, char** argv) { +// vector> fimagePoints; +// vector> fobjectPoints; +// vector> bimagePoints; +// vector> bobjectPoints; +// +// // ETH3D setup +// baset ff=3408.57; +// baset fcx=3114.7; +// baset fcy=2070.92; +// baset bf=3408.57; +// baset bcx=3114.7; +// baset bcy=2070.92; +// +// char datadir[PATH_MAX]; +// strncpy(datadir, argv[0], sizeof(datadir)); +// dirname(datadir); +// std::string bindir(datadir); +// std::string base_path = bindir + "/../../data-eth3d/"; +// +// auto ffeats = ETH3DFeatures(base_path, eth3d_locations[0]); +// ffeats.imageFeatures(fimagePoints); +// ffeats.worldFeatures(fobjectPoints); +// auto fgtpose = ffeats.getGroundTruthPose(); +// vector finitialGuess = {0, 0, 0, 0, 0, 0}; +// +// auto bfeats = ETH3DFeatures(base_path, eth3d_locations[0]); +// bfeats.imageFeatures(bimagePoints); +// bfeats.worldFeatures(bobjectPoints); +// auto bgtpose = bfeats.getGroundTruthPose(); +// vector binitialGuess = {0, 0, 0, 0, 0, 0}; +// //vector initialGuess = {res.first[0], res.first[1], res.first[2], +// // res.second[0], res.second[1], res.second[2]}; +// //uint32_t numPts = imagePoints.size(); +// //MSG("Found %d points\n", numPts); +// //for (int i=0; i<5; i++) { +// //cout << "Point "<> fimagePointsSubset(fimagePoints.begin(), +// fimagePoints.begin() + i); vector> +// fobjectPointsSubset(fobjectPoints.begin(), fobjectPoints.begin() + +// i); vector> +// bimagePointsSubset(bimagePoints.begin(), bimagePoints.begin() + i); +// vector> bobjectPointsSubset(bobjectPoints.begin(), +// bobjectPoints.begin() + i); cout << "Testing with " << +// bimagePointsSubset.size() << " points\n"; +// +// { // requires float, does not work with fixed point +// cout << "\ntesting opencv\n"; +// float _cM[] = {ff, 0, fcx, +// 0, ff, fcy, +// 0, 0, 1}; +// cv::Mat cameraMatrix = cv::Mat(3, 3, cv::DataType::type, +// _cM); cv::Mat distCoeffs = +// cv::Mat::zeros(4,1,cv::DataType::type); cv::Mat rvec = +// cv::Mat::zeros(3,1,cv::DataType::type); cv::Mat tvec = +// cv::Mat::zeros(3,1,cv::DataType::type); CLOCK(opencv); +// TIC(opencv); +// // OpenCV PnP method +// cv::solvePnP(fobjectPointsSubset, fimagePointsSubset, +// cameraMatrix, distCoeffs, rvec, tvec, false, +// cv::SOLVEPNP_ITERATIVE); TOC(opencv); cout << "opencv result:" << +// endl; cout << rvec << endl; cout << tvec << endl << endl; +// +// float error[6]; +// error[0] = fgtpose[0] - rvec.at(0); +// error[1] = fgtpose[1] - rvec.at(1); +// error[2] = fgtpose[2] - rvec.at(2); +// error[3] = fgtpose[3] - tvec.at(0); +// error[4] = fgtpose[4] - tvec.at(1); +// error[5] = fgtpose[5] - tvec.at(2); +// printf("SeNtInAl,xy,%s,opencv_error_vs_numpts,%d,%f\n", +// __FUNCTION__, i, twonormsq(error, 6)); +// } +// +//#ifdef PPL_GN +// { +// cout << "testing gn - cleartext - float\n"; +// vector finitialGuessCopy = finitialGuess; +// CLOCK(gn_float_cleartext); +// TIC(gn_float_cleartext); +// gaussNewton(fobjectPointsSubset, fimagePointsSubset, ff, +// fcx, fcy, &finitialGuessCopy[0], "float"); +// TOC(gn_float_cleartext); printVector("[rotation; translation]", +// &finitialGuessCopy[0], 6); +// +// float error[6]; +// error[0] = fgtpose[0] - finitialGuessCopy[0]; +// error[1] = fgtpose[1] - finitialGuessCopy[1]; +// error[2] = fgtpose[2] - finitialGuessCopy[2]; +// error[3] = fgtpose[3] - finitialGuessCopy[3]; +// error[4] = fgtpose[4] - finitialGuessCopy[4]; +// error[5] = fgtpose[5] - finitialGuessCopy[5]; +// printf("SeNtInAl,xy,%s,gn_float_error_vs_numpts,%d,%f\n", +// __FUNCTION__, i, twonormsq(error, 6)); +// } +// { +// cout << "testing gn - cleartext - baset\n"; +// vector binitialGuessCopy = binitialGuess; +// CLOCK(gn_baset_cleartext); +// TIC(gn_baset_cleartext); +// gaussNewton(bobjectPointsSubset, bimagePointsSubset, bf, +// bcx, bcy, &binitialGuessCopy[0], "baset"); +// TOC(gn_baset_cleartext); printVector("[rotation; translation]", +// &binitialGuessCopy[0], 6); +// +// baset error[6]; +// error[0] = bgtpose[0] - binitialGuessCopy[0]; +// error[1] = bgtpose[1] - binitialGuessCopy[1]; +// error[2] = bgtpose[2] - binitialGuessCopy[2]; +// error[3] = bgtpose[3] - binitialGuessCopy[3]; +// error[4] = bgtpose[4] - binitialGuessCopy[4]; +// error[5] = bgtpose[5] - binitialGuessCopy[5]; +// printf("SeNtInAl,xy,%s,gn_baset_error_vs_numpts,%d,%f\n", +// __FUNCTION__, i, (double)twonormsq(error, 6)); +// } +//#endif +//#ifdef PPL_LM +// { +// cout << "testing lm - cleartext - float\n"; +// vector finitialGuessCopy = finitialGuess; +// CLOCK(lm_float_cleartext); +// TIC(lm_float_cleartext); +// lm(fobjectPointsSubset, fimagePointsSubset, ff, fcx, fcy, +// &finitialGuessCopy[0]); TOC(lm_float_cleartext); +// printVector("[rotation; translation]", &finitialGuessCopy[0], 6); +// +// float error[6]; +// error[0] = fgtpose[0] - finitialGuessCopy[0]; +// error[1] = fgtpose[1] - finitialGuessCopy[1]; +// error[2] = fgtpose[2] - finitialGuessCopy[2]; +// error[3] = fgtpose[3] - finitialGuessCopy[3]; +// error[4] = fgtpose[4] - finitialGuessCopy[4]; +// error[5] = fgtpose[5] - finitialGuessCopy[5]; +// printf("SeNtInAl,xy,%s,lm_float_error_vs_numpts,%d,%f\n", +// __FUNCTION__, i, twonormsq(error, 6)); +// } +// { +// cout << "testing lm - cleartext - baset\n"; +// vector binitialGuessCopy = binitialGuess; +// CLOCK(lm_baset_cleartext); +// TIC(lm_baset_cleartext); +// lm(bobjectPointsSubset, bimagePointsSubset, bf, bcx, bcy, +// &binitialGuessCopy[0]); TOC(lm_baset_cleartext); +// printVector("[rotation; translation]", &binitialGuessCopy[0], 6); +// +// baset error[6]; +// error[0] = bgtpose[0] - binitialGuessCopy[0]; +// error[1] = bgtpose[1] - binitialGuessCopy[1]; +// error[2] = bgtpose[2] - binitialGuessCopy[2]; +// error[3] = bgtpose[3] - binitialGuessCopy[3]; +// error[4] = bgtpose[4] - binitialGuessCopy[4]; +// error[5] = bgtpose[5] - binitialGuessCopy[5]; +// printf("SeNtInAl,xy,%s,lm_baset_error_vs_numpts,%d,%f\n", +// __FUNCTION__, i, (double)twonormsq(error, 6)); +// } +//#endif +// cout << "\n\n"; +// } +// +// return 0; +// } diff --git a/test/cleartext-fixed/float_vs_fixed_speed.cpp b/test/cleartext-fixed/float_vs_fixed_speed.cpp new file mode 100644 index 0000000..251d44d --- /dev/null +++ b/test/cleartext-fixed/float_vs_fixed_speed.cpp @@ -0,0 +1,223 @@ +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include "emp-sh2pc/emp-sh2pc.h" +#include "emp-tool/emp-tool.h" + +using namespace std; + +#define TEST_SZ 1000 * 10 + +#ifdef TEST_ADD +#define TEST_OP + +#define START() \ + WALL_CLOCK(EMP_ADD); \ + WALL_TIC(EMP_ADD); +#define STOP(LOGSTR_) WALL_TOC_CSV_GROUPED_BAR(EMP_ADD, LOGSTR_); +#endif + +#ifdef TEST_MUL +#define TEST_OP * +#define START() \ + WALL_CLOCK(EMP_MUL); \ + WALL_TIC(EMP_MUL); +#define STOP(LOGSTR_) WALL_TOC_CSV_GROUPED_BAR(EMP_MUL, LOGSTR_); +#endif + +int main(int argc, char** argv) { + if (argc != 2) { + MSG("Usage: %s \n", argv[0]); + return 1; + } + int party = atoi(argv[1]) + 1; + int port = 8080; + cout << "party: " << party << " port: " << port << endl; + NetIO* io = new NetIO(party == ALICE ? nullptr : "127.0.0.1", port); + + { + MSG("\nBenchmarking EMP float\n"); + + setup_semi_honest(io, party); + + Float* s_testa = + static_cast(operator new[](TEST_SZ * sizeof(Float))); + Float* s_testb = + static_cast(operator new[](TEST_SZ * sizeof(Float))); + Float* s_testc = + static_cast(operator new[](TEST_SZ * sizeof(Float))); + for (int i = 0; i < TEST_SZ; i++) { + s_testa[i] = Float(1.0f, ALICE); + s_testb[i] = Float(2.0f, ALICE); + } + + START(); + auto start = clock_start(); + for (int i = 0; i < TEST_SZ; i++) { + s_testc[i] = s_testa[i] TEST_OP s_testb[i]; + } + double interval = time_from(start); + STOP("32 bit float"); + + uint64_t numand = CircuitExecution::circ_exec->num_and(); + cout << "number of and gates: " << numand << endl; + cout << "garbling speed : " << numand / interval + << " million gate per second\n"; + + // for(int i=0; i() << " "; + // } + // cout << endl; + + delete[] s_testa; + delete[] s_testb; + delete[] s_testc; + cout << "deleted\n"; + finalize_semi_honest(); + } + + { + MSG("\nBenchmarking EMP Integer (32 bit)\n"); + + setup_semi_honest(io, party); + + Integer s_testa[TEST_SZ]; + Integer s_testb[TEST_SZ]; + Integer s_testc[TEST_SZ]; + for (int i = 0; i < TEST_SZ; i++) { + s_testa[i] = Integer(32, 1, ALICE); + s_testb[i] = Integer(32, 2, ALICE); + } + + START(); + auto start = clock_start(); + for (int i = 0; i < TEST_SZ; i++) { + s_testc[i] = s_testa[i] TEST_OP s_testb[i]; + } + double interval = time_from(start); + STOP("32 bit int"); + + uint64_t numand = CircuitExecution::circ_exec->num_and(); + cout << "number of and gates: " << numand << endl; + cout << "garbling speed : " << numand / interval + << " million gate per second\n"; + + // for(int i=0; i() << " "; + // } + // cout << endl; + + finalize_semi_honest(); + } + + { + MSG("\nBenchmarking EMP Integer (64 bit)\n"); + + setup_semi_honest(io, party); + + Integer s_testa[TEST_SZ]; + Integer s_testb[TEST_SZ]; + Integer s_testc[TEST_SZ]; + for (int i = 0; i < TEST_SZ; i++) { + s_testa[i] = Integer(64, 1, ALICE); + s_testb[i] = Integer(64, 2, ALICE); + } + + START(); + auto start = clock_start(); + for (int i = 0; i < TEST_SZ; i++) { + s_testc[i] = s_testa[i] TEST_OP s_testb[i]; + } + double interval = time_from(start); + STOP("64 bit int"); + + uint64_t numand = CircuitExecution::circ_exec->num_and(); + cout << "number of and gates: " << numand << endl; + cout << "garbling speed : " << numand / interval + << " million gate per second\n"; + + // for(int i=0; i() << " "; + // } + // cout << endl; + + finalize_semi_honest(); + } + + { + MSG("\nBenchmarking EMP fixed point (32 bits)\n"); + + setup_semi_honest(io, party); + + fixed_point_emp<16, 16> s_testa[TEST_SZ]; + fixed_point_emp<16, 16> s_testb[TEST_SZ]; + fixed_point_emp<16, 16> s_testc[TEST_SZ]; + for (int i = 0; i < TEST_SZ; i++) { + s_testa[i] = 1; + s_testb[i] = 2; + } + + START() + auto start = clock_start(); + for (int i = 0; i < TEST_SZ; i++) { + s_testc[i] = s_testa[i] TEST_OP s_testb[i]; + } + double interval = time_from(start); + STOP("32 bit fixed"); + + uint64_t numand = CircuitExecution::circ_exec->num_and(); + cout << "number of and gates: " << numand << endl; + cout << "garbling speed : " << numand / interval + << " million gate per second\n"; + + // for(int i=0; i() << " "; + // } + // cout << endl; + + finalize_semi_honest(); + } + + { + MSG("\nBenchmarking EMP fixed point (64 bits)\n"); + + setup_semi_honest(io, party); + + fixed_point_emp<32, 32> s_testa[TEST_SZ]; + fixed_point_emp<32, 32> s_testb[TEST_SZ]; + fixed_point_emp<32, 32> s_testc[TEST_SZ]; + for (int i = 0; i < TEST_SZ; i++) { + s_testa[i] = 1; + s_testb[i] = 2; + } + + START(); + auto start = clock_start(); + for (int i = 0; i < TEST_SZ; i++) { + s_testc[i] = s_testa[i] TEST_OP s_testb[i]; + } + double interval = time_from(start); + STOP("64 bit fixed"); + + uint64_t numand = CircuitExecution::circ_exec->num_and(); + cout << "number of and gates: " << numand << endl; + cout << "garbling speed : " << numand / interval + << " million gate per second\n"; + + // for(int i=0; i() << " "; + // } + // cout << endl; + + finalize_semi_honest(); + } + delete io; + return 0; +} diff --git a/test/cleartext-fixed/test.cpp b/test/cleartext-fixed/test.cpp new file mode 100644 index 0000000..e1e20ba --- /dev/null +++ b/test/cleartext-fixed/test.cpp @@ -0,0 +1,387 @@ +//#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +using namespace emp; +using namespace std; + +typedef fixed_point baset; + +int main(int argc, char** argv) { + + float f_float = 715; + float cx_float = 354; + float cy_float = 245; + float _cM_float[] = {f_float, 0, cx_float, 0, f_float, cy_float, 0, 0, 1}; + baset f = 715; + baset cx = 354; + baset cy = 245; + baset _cM[] = {f, 0, cx, 0, f, cy, 0, 0, 1}; + + vector> imagePoints; + vector> objectPoints; + Hoffs2DPoints(imagePoints); + Hoffs3DPoints(objectPoints); + + MSG("\n\ntesting trigfuncs - cleartext\n"); + { + float lulz = .2; + float lulzsin = sin(lulz); + float lulzcos = cos(lulz); + + cout << "libc sin: " << lulzsin << " cos: " << lulzcos << endl; + cout << "custom sin: " << mysin(lulz) << " cos: " << mycos(lulz) << endl; + } + { + baset lulz = .2; + cout << "baset sin: " << mysin(lulz) << " cos: " << mycos(lulz) << endl; + } + + MSG("\n\ntesting sqrt - cleartext\n"); + { + baset i = .2; + baset o = sqrt(i); + cout << "baset sqrt" << o << endl; + } + + MSG("\n\ntesting matmult - cleartext\n"); + { + float A[] = {1, 1, 1, 1, + 1, // 3x5 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; + float B[] = {2, 2, 2, + 2, // 5x4 + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2}; + float C[3 * 4]; // 3x4 + matmult(A, 3, 5, B, 5, 4, C); + printVector("float matmult", C, 3 * 4); + } + { + baset A[] = {1, 1, 1, 1, + 1, // 3x5 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; + baset B[] = {2, 2, 2, + 2, // 5x4 + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2}; + baset C[3 * 4]; // 3x4 + matmult(A, 3, 5, B, 5, 4, C); + printVector("baset matmult", C, 3 * 4); + } + + MSG("\n\ntesting matmult2DwTranspose - cleartext\n"); + { + float** A = new float*[3]; // 3x5 + for (int p = 0; p < 3; p++) { + A[p] = new float[5]; + for (int pp = 0; pp < 5; pp++) { + A[p][pp] = 1; + } + } + float** B = new float*[3]; // 3x5 + for (int p = 0; p < 3; p++) { + B[p] = new float[5]; + for (int pp = 0; pp < 5; pp++) { + B[p][pp] = 1; + } + } + float** C = new float*[3]; // 3x3 + for (int p = 0; p < 3; p++) + C[p] = new float[3]; + matmult2DwTranspose(A, 3, 5, false, B, 3, 5, true, C); + printMatrix("float matmult", C, 3, 3); + } + { + baset** A = new baset*[3]; // 3x5 + for (int p = 0; p < 3; p++) { + A[p] = new baset[5]; + for (int pp = 0; pp < 5; pp++) { + A[p][pp] = 1; + } + } + baset** B = new baset*[3]; // 3x5 + for (int p = 0; p < 3; p++) { + B[p] = new baset[5]; + for (int pp = 0; pp < 5; pp++) { + B[p][pp] = 1; + } + } + baset** C = new baset*[3]; // 3x3 + for (int p = 0; p < 3; p++) + C[p] = new baset[3]; + matmult2DwTranspose(A, 3, 5, false, B, 3, 5, true, C); + printMatrix("baset matmult", C, 3, 3); + } + + MSG("\n\ntesting rodrigues - cleartext\n"); + { + float r[3] = {.2, .4, .2}; // 3x1 + float R[3 * 4] = {}; + rodrigues(r, R); + printVector("float rodrigues", R, 3 * 4); + } + { + baset r[3] = {.2, .4, .2}; // 3x1 + baset R[3 * 4] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + rodrigues(r, R); + printVector("baset rodrigues", R, 3 * 4); + } + + MSG("\n\ntesting projectpoints - cleartext\n"); + { + float P[] = {0, 0, 2, 0, 0, 10, 2, 0, 10, 2, 6, 6, 6, 2, 2, 1, 1, 1, 1, 1}; + float x[] = {1.4888731, -0.58786786, 0.71628469, + 0.93423641, 2.983731, 18.328608}; + float projected[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + projectPoints(P, x, _cM_float, projected, 5); + printVector("float projection", projected, 2 * 5); + } + { + baset P[] = {0, 0, 2, 0, 0, 10, 2, 0, 10, 2, 6, 6, 6, 2, 2, 1, 1, 1, 1, 1}; + baset x[] = {1.4888731, -0.58786786, 0.71628469, + 0.93423641, 2.983731, 18.328608}; + baset projected[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + projectPoints(P, x, _cM, projected, 5); + printVector("baset projection", projected, 2 * 5); + } + + MSG("testing svd - sign subfunc - cleartext\n"); + { + float a = 2.40, b = 0; + cout << "float sign: " << mysign(a, b) << endl; + } + { + baset a = 2.40, b = 0; + cout << "baset sign: " << mysign(a, b) << endl; + } + + MSG("testing svd - pythag subfunc - cleartext\n"); + { + float a = -2.40, b = 96.0; + cout << "float pythag: " << mypythag(a, b) << endl; + } + { + baset a = -2.40, b = 96.0; + cout << "baset pythag: " << mypythag(a, b) << endl; + } + + MSG("\n\ntesting svd - cleartext\n"); + { + int m = 5, n = 4; + // svd overwrites with u matrix + float** in = new float*[m]; + for (int i = 0; i < m; i++) { + in[i] = new float[n](); + for (int j = 0; j < n; j++) { + in[i][j] = i + j; + } + } + in[0][0] = 0; + in[1][0] = 0; + in[2][0] = 0; + in[3][0] = 0; + in[4][0] = 0; + + float* w = new float[n](); // aka sigma, only diag + float** v = new float*[n]; // nxn + for (int i = 0; i < n; i++) + v[i] = new float[n](); + + svdcmp(in, m, n, w, v); + + printVector("w", w, n); + printMatrix("v", v, n, n); + printMatrix("a", in, m, n); + for (int i = 0; i < m; i++) { + delete[] in[i]; + } + delete[] in; + delete[] w; + for (int i = 0; i < n; i++) { + delete[] v[i]; + } + delete[] v; + } + { + int m = 5, n = 4; + // svd overwrites with u matrix + baset** in = new baset*[m]; + for (int i = 0; i < m; i++) { + in[i] = new baset[n](); + for (int j = 0; j < n; j++) { + in[i][j] = i + j; + } + } + in[0][0] = 0; + in[1][0] = 0; + in[2][0] = 0; + in[3][0] = 0; + in[4][0] = 0; + + baset* w = new baset[n](); // aka sigma, only diag + baset** v = new baset*[n]; // nxn + for (int i = 0; i < n; i++) + v[i] = new baset[n](); + + svdcmp(in, m, n, w, v); + + printVector("w", w, n); + printMatrix("v", v, n, n); + printMatrix("a", in, m, n); + for (int i = 0; i < m; i++) { + delete[] in[i]; + } + delete[] in; + delete[] w; + for (int i = 0; i < n; i++) { + delete[] v[i]; + } + delete[] v; + } + + // MSG("\ntesting myinvert\n"); + // //int m=7,n=3; + // //float one_d_M[] = {3, 0, 1, // toy example + // // 0, 0, 0, + // // 0, 4, 0, + // // 0, 0, 0, + // // 2, 0, 0, + // // 0, 0, 3, + // // 0, 0, 0}; + // int m=12,n=6; + // float one_d_M[] = {-9.1552734, 149.53613, -59.509277, 18.310547, + // 0, 3.0517578, + // -164.79492, -64.086914, -47.302246, + // 0, 19.836426, 1.5258789, + // 27.46582, 77.819824, 50.354004, 22.888184, + // 0, 1.5258789, -56.45752, -42.724609, -53.405762, + // 0, 22.888184, 3.0517578, + // 45.776367, 79.345703, 79.345703, 21.362305, 0, 0, + // -32.806396, -13.73291, -20.599365, + // 0, 24.414062, 4.5776367, -21.362305, 108.3374, + // -93.078613, 16.784668, 0, 3.0517578, -152.58789, + // -45.776367, -10.681152, 0, 19.836426, 3.0517578, + // 6.1035156, 39.672852, 3.0517578, 21.362305, 0, 0, + // -39.672852, -21.362305, -16.784668, + // 0, 22.888184, 1.5258789, + // 24.414062, 48.828125, 27.46582, 24.414062, 0, 0, + // -13.73291, 6.1035156, 12.207031, + // 0, 22.888184, 1.5258789}; + // + // cv::Mat cvM = cv::Mat(m, n, cv::DataType::type, one_d_M); + // cv::Mat ores; + // invert(cvM, ores, cv::DECOMP_SVD); + // cout << "opencv result:\n" << ores << endl; + // + // float** M = new float*[m]; + // for(int i=0; i::type); + // rvec = cv::Mat::zeros(3,1,cv::DataType::type); + // tvec = cv::Mat::zeros(3,1,cv::DataType::type); + // CLOCK(opencv); + // TIC(opencv); + // // OpenCV PnP method + // cv::solvePnP(objectPoints, imagePoints, cameraMatrix, distCoeffs, rvec, + // tvec, false, cv::SOLVEPNP_ITERATIVE); TOC(opencv); cout << "opencv + // result:" << endl; cout << rvec << endl; cout << tvec << endl << endl; + // + //#ifdef PPL_GN + // { + // cout << "testing gauss newton - cleartext\n"; + // float rt[6] = {0, 0, 0, 0, 0, 0}; // initial guess + // CLOCK(cleartext); + // TIC(cleartext); + // gaussNewton( objectPoints, imagePoints, f, cx, cy, rt); + // TOC(cleartext); + // printVector("[rotation; translation]", rt, 6); + // + // cout << "testing gauss newton\n"; + // float srt[6] = {0, 0, 0, 0, 0, 0}; // initial guess + // test_gaussnewton_circuit(party, io, + // objectPoints, imagePoints, f, cx, cy, srt); + // printVector("[rotation; translation]", srt, 6); + // } + //#endif + // + //#ifdef PPL_LM + // { + // cout << "testing lm - cleartext\n"; + // float rt[6] = {0, 0, 0, 0, 0, 0}; // initial guess + // CLOCK(cleartext); + // TIC(cleartext); + // lm( objectPoints, imagePoints, f, cx, cy, rt); + // TOC(cleartext); + // printVector("[rotation; translation]", rt, 6); + // + // cout << "testing lm\n"; + // float srt[6] = {0, 0, 0, 0, 0, 0}; // initial guess + // test_lm_circuit(party, io, + // objectPoints, imagePoints, f, cx, cy, srt); + // printVector("[rotation; translation]", srt, 6); + // } + //#endif + + return 0; +} diff --git a/test/common-test/CMakeLists.txt b/test/common-test/CMakeLists.txt new file mode 100644 index 0000000..1b63bcd --- /dev/null +++ b/test/common-test/CMakeLists.txt @@ -0,0 +1,18 @@ +cmake_minimum_required(VERSION 3.12) +set(common_test_dir "${CMAKE_CURRENT_SOURCE_DIR}/") +file(GLOB_RECURSE common_test CONFIGURE_DEPENDS "${common_test_dir}/*.cpp" "${common_test_dir}/*.h") +list(FILTER common_test EXCLUDE REGEX ".*eth3d_tester\\.cpp$") +list(FILTER common_test EXCLUDE REGEX ".*kitti_tester\\.cpp$") + +add_executable(eth3d_tester "${CMAKE_CURRENT_SOURCE_DIR}/eth3d_tester.cpp" ${common_test}) +target_include_directories(eth3d_tester PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) +target_compile_options(eth3d_tester PRIVATE "-Wall" "-Wextra") +target_link_libraries(eth3d_tester ${OpenCV_LIBS}) + + +# WARNING - NOT IMPLEMENTED +#add_executable(kitti_tester "${CMAKE_CURRENT_SOURCE_DIR}/kitti_tester.cpp") +#target_include_directories(kitti_tester PUBLIC) +#target_compile_options(kitti_tester PRIVATE "-Wall" "-Wextra") +#target_link_libraries(kitti_tester ${OpenCV_LIBS}) + diff --git a/test/common-test/april_snail_features.hpp b/test/common-test/april_snail_features.hpp new file mode 100644 index 0000000..d6839dd --- /dev/null +++ b/test/common-test/april_snail_features.hpp @@ -0,0 +1,24 @@ +#pragma once +#include +#include +#include +#include +#include + +constexpr const float ms = 0.65; + +template +void AprilSnail3DPoints(std::vector>& points) { + points.push_back(cv::Point3_(-ms, -ms, 0)); + points.push_back(cv::Point3_(-ms, ms, 0)); + points.push_back(cv::Point3_(ms, ms, 0)); + points.push_back(cv::Point3_(ms, -ms, 0)); +} + +template +void AprilSnail2DPoints(std::vector>& points) { + points.push_back(cv::Point_(207.71281433, 159.10252380)); + points.push_back(cv::Point_(230.89843750, 334.47018433)); + points.push_back(cv::Point_(403.77438354, 314.49819946)); + points.push_back(cv::Point_(384.17871094, 138.54327393)); +} diff --git a/test/common-test/cleartext-ref/gaussnewtonlocalization.hpp b/test/common-test/cleartext-ref/gaussnewtonlocalization.hpp new file mode 100644 index 0000000..fe37b62 --- /dev/null +++ b/test/common-test/cleartext-ref/gaussnewtonlocalization.hpp @@ -0,0 +1,273 @@ +#pragma once +//#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +using namespace std; + +// See William Hoff's lecure: https://www.youtube.com/watch?v=kq3c6QpcAGc +template +bool gaussNewton(vector> threeDPts, + vector> twoDPts, T _f, T _cx, T _cy, + T* _x, /* initial guess for { r1, r2, r3, t1, t2, t3 } */ + string logprint = "") { + + if (typeid(T) == typeid(int) || typeid(T) == typeid(int16_t) || + typeid(T) == typeid(uint16_t) || typeid(T) == typeid(int32_t) || + typeid(T) == typeid(uint32_t) || typeid(T) == typeid(int64_t) || + typeid(T) == typeid(uint64_t)) { + cout << "type not supported\n"; + assert(false); + } + + if (_verbosity & DBG_FLOW) + cout << "Gauss-Newton Gradient Decent" << endl; + int n = threeDPts.size(); + + T min_er; + T jacob_epsilon; + + // Initial Guess + T x[6]; + // Camera Params + T K[9] = {_f, 0, _cx, 0, _f, _cy, 0, 0, 1}; + // 3D World Points + // _ _ + // | x1 x2 x3 | + // | y1 y2 y3 ... | + // | z1 z2 z3 | + // |_ 1 1 1 _| + T P[4 * n]; + // 2D Image Points + // _ _ + // | u1 u2 u3 | + // | v1 v2 v3 ... | + // |_ 1 1 1 _| + T y0Homog[3 * n]; + + min_er = MIN_ER; + jacob_epsilon = JACOB_EPSILON; + for (int i = 0; i < 6; i++) { + x[i] = _x[i]; + } + for (int i = 0; i < n; i++) { + P[i] = threeDPts[i].x; + P[n + i] = threeDPts[i].y; + P[(2 * n) + i] = threeDPts[i].z; + P[(3 * n) + i] = 1; + + y0Homog[i] = twoDPts[i].x; + y0Homog[n + i] = twoDPts[i].y; + y0Homog[(2 * n) + i] = 1; + } + + // throw away last "row" of y0Homog (constant ones), + // interleave, and transpose into 2nx1 vector + // e.g. [x1; y1; x2; y2 ...] + T y0[2 * n]; + for (int p = 0; p < n; ++p) { + y0[p * 2] = y0Homog[p]; + y0[(p * 2) + 1] = y0Homog[p + n]; + } + + if (_verbosity & DBG_ARGS) { + printVector("P", P, 4 * n); + printVector("y0Homog", y0Homog, 3 * n); + printVector("y0", y0, 2 * n); + } + + // GN Iteration + int i; + for (i = 0; i < GN_MAX_ITR; i++) { + if (_verbosity & DBG_FLOW) { + cout << RED << "\n\nGN Iteration " << i << RESET << endl; + printVector("x", x, 6); + } + + // project points using x guess + T yHomog[3 * n]; + projectPoints(P, x, K, yHomog, n); + // throw away last "row" of result (constant ones), + // interleave, and transpose y into 2nx1 vector + // e.g. [x1; y1; x2; y2 ...] + T y[2 * n]; + for (int p = 0; p < n; ++p) { + y[p * 2] = yHomog[p]; + y[(p * 2) + 1] = yHomog[p + n]; + } + if (_verbosity & DBG_PROJECT) { + printVector("y", y, 2 * n); + } + + // calculate jacobian + // - - + // | df/dr1 df/dr2 df/dr3 df/t1 df/t2 df/t3 | + // | . | + // | . | + // | . (2*n) | + // - - + T** jacob = new T*[n * 2]; + for (int p = 0; p < n * 2; p++) + jacob[p] = new T[6]; + for (int j = 0; j < 6; j++) { // each dof + // perturb x + T oldx = x[j]; + x[j] += jacob_epsilon; + + // project with epsilon + T ytempHomog[3 * n]; + projectPoints(P, x, K, ytempHomog, n); + // throw away last "row" of result, + // reshape, and transpose y into 2nx1 vector + // e.g. [x1; y1; x2; y2 ...] + T ytemp[2 * n]; + for (int p = 0; p < n; ++p) { + ytemp[p * 2] = ytempHomog[p]; + ytemp[(p * 2) + 1] = ytempHomog[p + n]; + } + if (_verbosity & DBG_PROJECT) { + printVector("ytemp", ytemp, 2 * n); + } + + // copy into jacob + for (int k = 0; k < 2 * n; k++) { + jacob[k][j] = (ytemp[k] - y[k]) / jacob_epsilon; + } + + // put x back + x[j] = oldx; + } + if (_verbosity & DBG_JACOB) { + printMatrix("jacob", jacob, 2 * n, 6); + } + + // error (nx2) + T dy[2 * n]; + for (int p = 0; p < 2 * n; p++) { + dy[p] = y0[p] - y[p]; + } + if (_verbosity & DBG_ER) + printVector("dy", dy, 2 * n); + + T** jacobI = new T*[6]; // 6x2n + for (int p = 0; p < 6; p++) + jacobI[p] = new T[2 * n]; + // pseudo inverse to solve dy = J dx -> dx = Jdag dy + // OpenCV's invert function + // cv::Mat jacobI; + // invert(jacob, jacobI, cv::DECOMP_SVD); + + //// Compute SVD directly with type T + // if (myinvert(jacob, 2*n, 6, jacobI)) { + // for (int p=0; p<2*n; p++) { + // delete[] jacob[p]; + // } + // delete[] jacob; + // for (int p=0; p<6; p++) { + // delete[] jacobI[p]; + // } + // delete[] jacobI; + // return false; + // } + + // Force compute SVD with floats + // Convert to floats + float** f_jacob = new float*[2 * n]; // 2nx6 + for (int p = 0; p < 2 * n; p++) { + f_jacob[p] = new float[6]; + for (int pp = 0; pp < 6; pp++) { + f_jacob[p][pp] = jacob[p][pp]; + } + } + float** f_jacobI = new float*[6]; // 6x2n + for (int p = 0; p < 6; p++) + f_jacobI[p] = new float[2 * n]; + if (myinvert(f_jacob, 2 * n, 6, f_jacobI)) { + for (int p = 0; p < 2 * n; p++) { + delete[] jacob[p]; + delete[] f_jacob[p]; + } + delete[] jacob; + delete[] f_jacob; + for (int p = 0; p < 6; p++) { + delete[] jacobI[p]; + delete[] f_jacobI[p]; + } + delete[] jacobI; + delete[] f_jacobI; + return false; + } + // Convert back to T template type + for (int p = 0; p < 6; p++) { + for (int pp = 0; pp < 2 * n; pp++) { + jacobI[p][pp] = f_jacobI[p][pp]; + } + } + for (int p = 0; p < 6; p++) + delete[] f_jacobI[p]; + delete[] f_jacobI; + for (int p = 0; p < 2 * n; p++) + delete[] f_jacob[p]; + delete[] f_jacob; + + // cout << "jacobI" << endl << jacobI << endl; + + for (int p = 0; p < 2 * n; p++) { + delete[] jacob[p]; + } + delete[] jacob; + + // dx = jacobI * dy; + // linearize jacobI for matmult + T jacobILinear[6 * 2 * n]; + for (int p = 0; p < 6; p++) { + for (int pp = 0; pp < 2 * n; pp++) { + jacobILinear[(p * 2 * n) + pp] = jacobI[p][pp]; + } + } + for (int p = 0; p < 6; p++) { + delete[] jacobI[p]; + } + delete[] jacobI; + T dx[6]; + matmult(jacobILinear, 6, 2 * n, dy, 2 * n, 1, dx); + if (_verbosity & DBG_POSE_UPDATE) + printVector("dx", dx, 6); + + // break if error under threshold + // if (twonormsq(dy, 2 * n) < min_er) + // break; + if (twonormsq(dx, 6) < min_er) + break; + + // update pose + for (int p = 0; p < 6; p++) + x[p] = x[p] + dx[p]; + } + if (_verbosity & DBG_FLOW) + cout << "Did " << i << " GN iterations\n"; + printf("SeNtInAl,xy,%s,gn_%s_iterations_vs_numpts,%d,%d\n", __FUNCTION__, + logprint.c_str(), n, i); + + // copy data back to res + for (int i = 0; i < 6; i++) { + _x[i] = x[i]; + } + + return i < GN_MAX_ITR; +} diff --git a/test/common-test/cleartext-ref/invert.hpp b/test/common-test/cleartext-ref/invert.hpp new file mode 100644 index 0000000..4f1cf73 --- /dev/null +++ b/test/common-test/cleartext-ref/invert.hpp @@ -0,0 +1,194 @@ +#pragma once +//#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include + +using namespace std; + +// The matrix version +template +int myinvert(cv::Mat M, cv::Mat result) { + int m = M.rows; + int n = M.cols; + assert(m >= n); + assert(n == result.rows); + assert(m == result.cols); + + // need to copy M because svd overwrites with u matrix + T** in = new T*[m]; + for (int i = 0; i < m; i++) { + in[i] = new T[n]; + for (int j = 0; j < n; j++) { + in[i][j] = M.at(i, j); + } + } + + T** out = new T*[n]; + for (int i = 0; i < n; i++) { + out[i] = new T[m]; + } + + T* w = new T[m]; // aka sigma, only diag + T** v = new T*[n]; // nxn + for (int i = 0; i < n; i++) + v[i] = new T[n]; + + // Mat u(m, nm, type, buf); + // Mat w(nm, 1, type, u.ptr() + m*nm*esz); + // Mat vt(nm, n, type, w.ptr() + nm*esz); + // SVD::compute(src, w, u, vt); + // SVD::backSubst(w, u, vt, Mat(), _dst); + + // compute_svd(in, m, n, w, v);// in overwritten with u + // svd_backsubstitute(in, w, v, m, n, NULL, out[0]); // why is out is 1D? - + // because of b :( + + if (svdcmp(in, m, n, w, v)) { + return -1; + } + T** u = in; + + // cout << "w\n"; + // printVector(w, m); + + // cout << "v\n"; + // printMatrix(v, n, n); + // cout << endl; + + // cout << "u\n"; + // printMatrix(u, m, n); + // cout << endl; + + // result = inv(in) = v*inv(w)*uT + // inv(w)*uT + for (int j = 0; j < n; j++) { + if (w[j]) { + for (int i = 0; i < m; i++) { + u[i][j] /= w[j]; // note u indices do transpose + } + } else { + for (int i = 0; i < m; i++) + u[i][j] = 0; + } + } + + // cout << "u\n"; + // printMatrix(u, m, n); + // cout << endl; + + // v*(w*uT) (don't use matmult so we can do transpose ourselves) + for (int j = 0; j < n; j++) { + for (int jj = 0; jj < m; jj++) { + out[j][jj] = 0.0; + for (int k = 0; k < n; k++) { + out[j][jj] += v[j][k] * u[jj][k]; // note u indices do transpose + } + } + } + + // cout << "result\n"; + // printMatrix(out, n, m); + // cout << endl; + + // copy into result + for (int i = 0; i < n; i++) + for (int j = 0; j < m; j++) + result.at(i, j) = out[i][j]; + + for (int i = 0; i < m; i++) + delete[] in[i]; + delete[] in; + for (int i = 0; i < n; i++) + delete[] out[i]; + delete[] out; + delete[] w; + for (int i = 0; i < n; i++) + delete[] v[i]; + delete[] v; + return 0; +} + +template +int myinvert(T** M, int m, int n, T** res) { + assert(m >= n); + + // need to copy M because svd overwrites with u matrix + T** in = new T*[m]; + for (int i = 0; i < m; i++) { + in[i] = new T[n]; + for (int j = 0; j < n; j++) { + // in[i][j]=M.at(i,j); + in[i][j] = M[i][j]; + } + } + + T* w = new T[m]; // aka sigma, only diag + T** v = new T*[n]; // nxn + for (int i = 0; i < n; i++) + v[i] = new T[n]; + + if (svdcmp(in, m, n, w, v)) { + for (int i = 0; i < m; i++) + delete[] in[i]; + delete[] in; + delete[] w; + for (int i = 0; i < n; i++) + delete[] v[i]; + delete[] v; + return -1; + } + T** u = in; + + // printVector("w", w, m); + // printMatrix("v", v, n, n); + // printMatrix("u", u, m, n); + + // res = inv(in) = v*inv(w)*uT + // inv(w)*uT + for (int j = 0; j < n; j++) { + if (w[j]) { + for (int i = 0; i < m; i++) { + u[i][j] /= w[j]; // note u indices do transpose + } + } else { + for (int i = 0; i < m; i++) + u[i][j] = 0; + } + } + + // cout << "u\n"; + // printMatrix(u, m, n); + // cout << endl; + + // v*(w*uT) (don't use matmult so we can do transpose ourselves) + for (int j = 0; j < n; j++) { + for (int jj = 0; jj < m; jj++) { + res[j][jj] = 0.0; + for (int k = 0; k < n; k++) { + res[j][jj] += v[j][k] * u[jj][k]; // note u indices do transpose + } + } + } + + // cout << "result\n"; + // printMatrix(res, n, m); + // cout << endl; + + for (int i = 0; i < m; i++) + delete[] in[i]; + delete[] in; + delete[] w; + for (int i = 0; i < n; i++) + delete[] v[i]; + delete[] v; + return 0; +} diff --git a/test/common-test/cleartext-ref/lmlocalization.hpp b/test/common-test/cleartext-ref/lmlocalization.hpp new file mode 100644 index 0000000..d3cd320 --- /dev/null +++ b/test/common-test/cleartext-ref/lmlocalization.hpp @@ -0,0 +1,348 @@ +#pragma once +//#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +using namespace std; + +template +bool lm(vector> threeDPts, vector> twoDPts, T _f, + T _cx, T _cy, T* _x, /* initial guess for { r1, r2, r3, t1, t2, t3 } */ + string logprint = "") { + + if (typeid(T) == typeid(int) || typeid(T) == typeid(int16_t) || + typeid(T) == typeid(uint16_t) || typeid(T) == typeid(int32_t) || + typeid(T) == typeid(uint32_t) || typeid(T) == typeid(int64_t) || + typeid(T) == typeid(uint64_t)) { + cout << "type not supported\n"; + assert(false); + } + + if (_verbosity & DBG_FLOW) + cout << "Levenberg–Marquardt\n"; + int n = threeDPts.size(); + + T min_er; + T jacob_epsilon; + T lambda_init; + T lambda_max; + T lambda_min; + + T x[6]; + // Camera Params + T K[9] = {_f, 0, _cx, 0, _f, _cy, 0, 0, 1}; + // 3D World Points + // _ _ + // | x1 x2 x3 | + // | y1 y2 y3 ... | + // | z1 z2 z3 | + // |_ 1 1 1 _| + T P[4 * n]; + // 2D Image Points + // _ _ + // | u1 u2 u3 | + // | v1 v2 v3 ... | + // |_ 1 1 1 _| + T y0Homog[3 * n]; + min_er = MIN_ER; + jacob_epsilon = JACOB_EPSILON; + lambda_init = LM_LAMBDA_INIT; + lambda_max = LM_LAMBDA_MAX; + lambda_min = LM_LAMBDA_MIN; + for (int i = 0; i < 6; i++) { + x[i] = _x[i]; + } + for (int i = 0; i < n; i++) { + P[i] = threeDPts[i].x; + P[n + i] = threeDPts[i].y; + P[(2 * n) + i] = threeDPts[i].z; + P[(3 * n) + i] = 1; + + y0Homog[i] = twoDPts[i].x; + y0Homog[n + i] = twoDPts[i].y; + y0Homog[(2 * n) + i] = 1; + } + + // throw away last "row" of y0Homog (constant ones), + // interleave, and transpose into 2nx1 vector + // e.g. [x1; y1; x2; y2 ...] + T y0[2 * n]; + for (int p = 0; p < n; ++p) { + y0[p * 2] = y0Homog[p]; + y0[(p * 2) + 1] = y0Homog[p + n]; + } + + if (_verbosity & DBG_ARGS) { + printVector("P", P, 4 * n); + printVector("y0Homog", y0Homog, 3 * n); + printVector("y0", y0, 2 * n); + } + + T lambda = lambda_init; + T prevErrNorm = std::numeric_limits::max(); + + // LM Iteration + int i; + for (i = 0; i < LM_MAX_ITR; i++) { + if (_verbosity & DBG_FLOW) { + cout << RED << "\n\nLM Iteration " << i << RESET << endl; + printVector("x", x, 6); + } + + // project points using x guess + T yHomog[3 * n]; + projectPoints(P, x, K, yHomog, n); + // throw away last "row" of result (constant ones), + // interleave, and transpose y into 2nx1 vector + // e.g. [x1; y1; x2; y2 ...] + T y[2 * n]; + for (int p = 0; p < n; ++p) { + y[p * 2] = yHomog[p]; + y[(p * 2) + 1] = yHomog[p + n]; + } + if (_verbosity & DBG_PROJECT) { + printVector("reshapen y", y, 2 * n); + } + + // calculate jacobian + // - - + // | df/dr1 df/dr2 df/dr3 df/t1 df/t2 df/t3 | + // | . | + // | . | + // | . (2*n) | + // - - + T** jacob = new T*[n * 2]; + for (int p = 0; p < n * 2; p++) + jacob[p] = new T[6]; + for (int j = 0; j < 6; j++) { // each dof + // perturb x + T oldx = x[j]; + x[j] += jacob_epsilon; + + // project with epsilon + T ytempHomog[3 * n]; + projectPoints(P, x, K, ytempHomog, n); + // throw away last "row" of result, + // reshape, and transpose y into 2nx1 vector + // e.g. [x1; y1; x2; y2 ...] + T ytemp[2 * n]; + for (int p = 0; p < n; ++p) { + ytemp[p * 2] = ytempHomog[p]; + ytemp[(p * 2) + 1] = ytempHomog[p + n]; + } + if (_verbosity & DBG_PROJECT) { + printVector("ytemp", ytemp, 2 * n); + } + + // copy into jacob + for (int k = 0; k < 2 * n; k++) { // calculate ^ by hand + jacob[k][j] = (ytemp[k] - y[k]) / jacob_epsilon; + } + + // put x back + x[j] = oldx; + } + if (_verbosity & DBG_JACOB) + printMatrix("jacob", jacob, 2 * n, 6); + + // error (nx2) + T dy[2 * n]; + for (int p = 0; p < 2 * n; p++) { + dy[p] = y0[p] - y[p]; + } + if (_verbosity & DBG_ER) + printVector("dy", dy, 2 * n); + + // LM with fletcher improvement + // inv(Jt*J + lambda*diag(Jt*J)) * Jt * dy + T** JtJ = new T*[6]; // 6x6 + for (int p = 0; p < 6; p++) + JtJ[p] = new T[6]; + matmult2DwTranspose(jacob, n * 2, 6, true, jacob, n * 2, 6, false, JtJ); + + if (_verbosity & DBG_JACOB) + printMatrix("JtJ", JtJ, 6, 6); + + // add lambda * diag(Jt*J) in-place + for (int li = 0; li < 6; li++) { + // add fletcher column-wise to JtJ + T fletcher = lambda * JtJ[li][li]; + + for (int lj = 0; lj < 6; lj++) { + JtJ[lj][li] += fletcher; + } + } + + T** JtJ_i = new T*[6]; // 6x6 + for (int p = 0; p < 6; p++) + JtJ_i[p] = new T[6]; + + // OpenCV's invert + // cv::Mat JtJ_i = cv::Mat(6, 6, cv::DataType::type); + // invert(JtJ, JtJ_i); + + // Compute SVD with type T + // if (myinvert(JtJ, 6, 6, JtJ_i)) { + // for (int p=0; p<2*n; p++) + // delete[] jacob[p]; + // delete[] jacob; + // for (int p=0; p<6; p++) { + // delete[] JtJ[p]; + // } + // delete[] JtJ; + // for (int p=0; p<6; p++) + // delete[] JtJ_i[p]; + // delete[] JtJ_i; + // return -1; + //} + + // Compute SVD with floats + // Convert to floats + float** f_JtJ = new float*[6]; // 6x6 + for (int p = 0; p < 6; p++) { + f_JtJ[p] = new float[6]; + for (int pp = 0; pp < 6; pp++) { + f_JtJ[p][pp] = JtJ[p][pp]; + } + } + float** f_JtJ_i = new float*[6]; // 6x6 + for (int p = 0; p < 6; p++) + f_JtJ_i[p] = new float[6]; + if (myinvert(f_JtJ, 6, 6, f_JtJ_i)) { + for (int p = 0; p < 2 * n; p++) { + delete[] jacob[p]; + } + delete[] jacob; + for (int p = 0; p < 6; p++) { + delete[] JtJ[p]; + delete[] f_JtJ[p]; + delete[] JtJ_i[p]; + delete[] f_JtJ_i[p]; + } + delete[] JtJ; + delete[] f_JtJ; + delete[] JtJ_i; + delete[] f_JtJ_i; + return -1; + } + // Convert back to T template type + for (int p = 0; p < 6; p++) { + for (int pp = 0; pp < 6; pp++) { + JtJ_i[p][pp] = f_JtJ_i[p][pp]; + } + } + for (int p = 0; p < 6; p++) { + delete[] f_JtJ[p]; + delete[] f_JtJ_i[p]; + } + delete[] f_JtJ; + delete[] f_JtJ_i; + + for (int p = 0; p < 6; p++) { + delete[] JtJ[p]; + } + delete[] JtJ; + + if (_verbosity & DBG_JACOB) + printMatrix("JtJ_i", JtJ_i, 6, 6); + + T** JtJ_i_Jt = new T*[6]; // 6x2n + for (int p = 0; p < 6; p++) + JtJ_i_Jt[p] = new T[2 * n]; + matmult2DwTranspose(JtJ_i, 6, 6, false, jacob, n * 2, 6, true, JtJ_i_Jt); + for (int p = 0; p < 2 * n; p++) + delete[] jacob[p]; + delete[] jacob; + for (int p = 0; p < 6; p++) + delete[] JtJ_i[p]; + delete[] JtJ_i; + + if (_verbosity & DBG_JACOB) + printMatrix("JtJ_i_Jt", JtJ_i_Jt, 6, 2 * n); + + // linearize for matmult + T JtJ_i_Jt_linear[6 * 2 * n]; + for (int p = 0; p < 6; p++) { + for (int pp = 0; pp < n * 2; pp++) { + JtJ_i_Jt_linear[(p * n * 2) + pp] = JtJ_i_Jt[p][pp]; + } + } + for (int p = 0; p < 6; p++) + delete[] JtJ_i_Jt[p]; + delete[] JtJ_i_Jt; + + T dx[6]; + matmult(JtJ_i_Jt_linear, 6, 2 * n, dy, 2 * n, + 1, // column vector + dx); + + if (_verbosity & DBG_POSE_UPDATE) + printVector("dx", dx, 6); + + // T errNorm = twonormsq(dy, 2*n); + T errNorm = twonormsq(dx, 6); + if (_verbosity & DBG_ER) { + if (printints) { + cout << "errNorm " << *(int*)&errNorm << endl; + } else { + cout << "errNorm " << errNorm << endl; + } + } + + if (errNorm > prevErrNorm) + lambda *= 10; + else + lambda /= 10; + lambda = MIN(lambda, lambda_max); + lambda = MAX(lambda, lambda_min); + if (_verbosity & DBG_LAMBDA) { + if (printints) { + cout << "lambda " << *(int*)&lambda << endl; + } else { + cout << "lambda " << lambda << endl; + } + } + + prevErrNorm = errNorm; + + // update pose + for (int p = 0; p < 6; p++) + x[p] = x[p] + dx[p]; + + // break if error under threshold + if (errNorm < min_er) { + if (_verbosity & DBG_ER) { + cout << "breaking because errNorm " << errNorm << " is smaller than " + << min_er << endl; + } + break; + } + } + + if (_verbosity & DBG_FLOW) + cout << "Did " << i + 1 << " GN iterations\n"; + printf("SeNtInAl,xy,%s,lm_%s_iterations_vs_numpts,%d,%d\n", __FUNCTION__, + logprint.c_str(), n, i); + + // copy data back to x + for (int i = 0; i < 6; i++) { + _x[i] = x[i]; + } + + return i < LM_MAX_ITR; +} diff --git a/test/common-test/cleartext-ref/matmult.hpp b/test/common-test/cleartext-ref/matmult.hpp new file mode 100644 index 0000000..298fd75 --- /dev/null +++ b/test/common-test/cleartext-ref/matmult.hpp @@ -0,0 +1,48 @@ +#pragma once +#include +#include + +// M mxn +// N mmxnn +// res mxnn +template +void matmult(T* M, int m, int n, T* N, int mm, int nn, T* res) { + (void)mm; + assert(n == mm); + for (int i = 0; i < m; i++) { // row + for (int j = 0; j < nn; j++) { // col + res[i * nn + j] = 0; + for (int k = 0; k < n; k++) { + res[i * nn + j] += M[i * n + k] * N[k * nn + j]; + } + } + } +} + +// M mxn +// N mmxnn +// res mxnn +template +void matmult2DwTranspose(T** M, int m, int n, const bool transposeM, T** N, + int mm, int nn, const bool transposeN, T** res) { + if (transposeM) + std::swap(m, n); + if (transposeN) + std::swap(mm, nn); + assert(n == mm); + for (int i = 0; i < m; i++) { // row + for (int j = 0; j < nn; j++) { // col + res[i][j] = 0; + for (int k = 0; k < n; k++) { + if (!transposeM && !transposeN) + res[i][j] += M[i][k] * N[k][j]; + else if (!transposeM && transposeN) + res[i][j] += M[i][k] * N[j][k]; + else if (transposeM && !transposeN) + res[i][j] += M[k][i] * N[k][j]; + else if (transposeM && transposeN) + res[i][j] += M[k][i] * N[j][k]; + } + } + } +} diff --git a/test/common-test/cleartext-ref/projectpoints.hpp b/test/common-test/cleartext-ref/projectpoints.hpp new file mode 100644 index 0000000..863229b --- /dev/null +++ b/test/common-test/cleartext-ref/projectpoints.hpp @@ -0,0 +1,48 @@ +#pragma once +#include +#include +#include +#include + +#include +#include + +using namespace std; + +// Note: All 2D matrix inputs passed as single dimension array +// P=4xnumPoints [x1, x2, ...; y1, y2, ...; z1, z2, ...; 1, 1, ... ] +// x=6x1 [ rotation angles; translation] (column) +// K=3x3 camera intrinsic +// result=3xnumPoints preallocated, only first two rows are x,y. +// last row needed for intermediate calculation (homogeneous) +// [ u1, u2, ...; v1, v2, ...; 1, 1, ... ] +template +void projectPoints(T* _P, T* _x, T* _K, T* _res, int numPoints) { + // rodreigues operates on a 3x4 matrix + T _R[12]; // 3x4 + rodrigues(_x, _R); + + // copy translation into last column of R + _R[3] = _x[3]; + _R[7] = _x[4]; + _R[11] = _x[5]; + + // printVector("R", _R, 12); + // printVector("K", _K, 9); + + // res = K*R*P; + T* scratch = new T[3 * 4]; + matmult(_K, 3, 3, _R, 3, 4, scratch); + // printVector("scratch", scratch, 3*4); + matmult(scratch, 3, 4, _P, 4, numPoints, _res); + // printVector("_res", _res, 3*numPoints); + delete[] scratch; + + // homogeneous coords to x,y + for (int i = 0; i < numPoints; i++) { + _res[i] = _res[i] / _res[2 * numPoints + i]; + _res[numPoints + i] = _res[numPoints + i] / _res[2 * numPoints + i]; + _res[2 * numPoints + i] = 1; + } + // printVector("_resHomog", _res, 3*numPoints); +} diff --git a/test/common-test/cleartext-ref/rodrigues.hpp b/test/common-test/cleartext-ref/rodrigues.hpp new file mode 100644 index 0000000..a2362da --- /dev/null +++ b/test/common-test/cleartext-ref/rodrigues.hpp @@ -0,0 +1,33 @@ +#pragma once +#include +#include + +// r=3x1, R=3x4 (only 3x3 is used here) +template +void rodrigues(T* r, T* R) { + T theta = sqrt(r[0] * r[0] + r[1] * r[1] + r[2] * r[2]); + + T c = mycos(theta); + T s = mysin(theta); + T one = 1; + T zero = 0; + T c1 = one - c; + T itheta = theta ? one / theta : zero; + + T x = r[0] * itheta; + T y = r[1] * itheta; + T z = r[2] * itheta; + + // R = cos(theta)*I + (1 - cos(theta))*r*rT + sin(theta)*[r_x] + R[0] = c + c1 * x * x; + R[1] = c1 * x * y - s * z; + R[2] = c1 * x * z + s * y; + + R[4] = c1 * x * y + s * z; + R[5] = c + c1 * y * y; + R[6] = c1 * y * z - s * x; + + R[8] = c1 * x * z - s * y; + R[9] = c1 * y * z + s * x; + R[10] = c + c1 * z * z; +} diff --git a/test/common-test/cleartext-ref/svd.hpp b/test/common-test/cleartext-ref/svd.hpp new file mode 100644 index 0000000..3a29c5e --- /dev/null +++ b/test/common-test/cleartext-ref/svd.hpp @@ -0,0 +1,363 @@ +#pragma once +/* + An implementation of SVD from Numerical Recipes in C and Mike Erhdmann's + lectures + */ + +#include +#include +#include +#include + +#include +#include +#include +#include + +using namespace std; + +template +T myfmax(T a, T b) { + return a > b ? a : b; +} + +static int iminarg1, iminarg2; +#define IMIN(a, b) \ + (iminarg1 = (a), iminarg2 = (b), \ + (iminarg1 < (iminarg2) ? (iminarg1) : iminarg2)) + +template +T mysqr(T a) { + T zero = 0; + return a == zero ? zero : a * a; +} + +template +T mysign(T a, T b) { + //#define SIGN(a,b) ((b) > 0.0 ? fabs(a) : - fabs(a)) + T zero = 0; + return (b) > zero ? fabs(a) : -fabs(a); +} + +// calculates sqrt( a^2 + b^2 ) with decent precision +template +T mypythag(T a, T b) { + // return sqrt( (a*a) + (b*b) ); // more likely to overflow? + + T zero = 0; + T one = 1; + T absa = fabs(a); + T absb = fabs(b); + + if (absa < absb) + std::swap(absa, absb); + + if (absa == zero) + return zero; + + return (absa * sqrt(one + mysqr(absb / absa))); +} + +template +bool checkeq(T a, T b) { + if (typeid(T) == typeid(float) || typeid(T) == typeid(double)) { + return (a + b) == b; + } else if (typeid(T) == typeid(fixed_point) || + typeid(T) == typeid(fixed_point)) { + T epsilon = 1e-5; + T margin = b * epsilon; + return (a > b - margin) && (a < b + margin); + } else { + assert(true); // type not supported + } + return false; +} + +/* + Modified from Numerical Recipes in C + Given a matrix a[nRows][nCols], svdcmp() computes its singular value + decomposition, A = U * W * Vt. A is replaced by U when svdcmp + returns. The diagonal matrix W is output as a vector w[nCols]. + V (not V transpose) is output as the matrix V[nCols][nCols]. + */ +template +int svdcmp(T** a, int nRows, int nCols, T* w, T** v) { + int flag = 0, i = 0, its = 0, j = 0, jj = 0, k = 0, l = 0, nm = 0; + T anorm, c, f, g, h, s, scale, x, y, z, *rv1; + + T zero = 0; + T one = 1; + T two = 2; + T epsilon = 1e-5; + + // rv1 = (T*) malloc(sizeof(T) * nCols); // does not call default constructor + rv1 = new T[nCols]; + if (rv1 == NULL) { + printf("svdcmp(): Unable to allocate vector\n"); + return (-1); + } + + // Householder reduction to bidiagonal form + g = scale = anorm = zero; + for (i = 0; i < nCols; i++) { + l = i + 1; + rv1[i] = scale * g; + g = s = scale = zero; + if (i < nRows) { + for (k = i; k < nRows; k++) + scale += fabs(a[k][i]); + // if (scale) { // not fixed point friendly + if (!checkeq(scale, zero)) { + for (k = i; k < nRows; k++) { + a[k][i] /= scale; + s += a[k][i] * a[k][i]; + } + f = a[i][i]; + g = -mysign(sqrt(s), f); + h = f * g - s; + a[i][i] = f - g; + for (j = l; j < nCols; j++) { + for (s = zero, k = i; k < nRows; k++) { + s += a[k][i] * a[k][j]; + } + f = s / h; + for (k = i; k < nRows; k++) { + a[k][j] += f * a[k][i]; + } + } + for (k = i; k < nRows; k++) + a[k][i] *= scale; + } + } + w[i] = scale * g; + g = s = scale = zero; + if (i < nRows && i != nCols - 1) { + for (k = l; k < nCols; k++) + scale += fabs(a[i][k]); + // if (scale) { // not fixed point friendly + if (!checkeq(scale, zero)) { + for (k = l; k < nCols; k++) { + a[i][k] /= scale; + s += a[i][k] * a[i][k]; + } + f = a[i][l]; + g = -mysign(sqrt(s), f); + h = f * g - s; + a[i][l] = f - g; + for (k = l; k < nCols; k++) + rv1[k] = a[i][k] / h; + for (j = l; j < nRows; j++) { + for (s = zero, k = l; k < nCols; k++) + s += a[j][k] * a[i][k]; + for (k = l; k < nCols; k++) + a[j][k] += s * rv1[k]; + } + for (k = l; k < nCols; k++) + a[i][k] *= scale; + } + } + anorm = myfmax(anorm, (fabs(w[i]) + fabs(rv1[i]))); + + printf("."); + fflush(stdout); + } + + // accumulation of right hand transformations + for (i = nCols - 1; i >= 0; i--) { + if (i < nCols - 1) { + // if (g) { // not fixed point friendly + if (!checkeq(g, zero)) { + for (j = l; j < nCols; j++) { + v[j][i] = (a[i][j] / a[i][l]) / g; + } + for (j = l; j < nCols; j++) { + for (s = zero, k = l; k < nCols; k++) + s += a[i][k] * v[k][j]; + for (k = l; k < nCols; k++) + v[k][j] += s * v[k][i]; + } + } + for (j = l; j < nCols; j++) + v[i][j] = v[j][i] = zero; + } + v[i][i] = one; + g = rv1[i]; + l = i; + printf(":"); + fflush(stdout); + } + + // accumulation of left hand transformations + for (i = IMIN(nRows, nCols) - 1; i >= 0; i--) { + l = i + 1; + g = w[i]; + for (j = l; j < nCols; j++) + a[i][j] = zero; + // if (g) { // not fixed point friendly + if (!checkeq(g, zero)) { + g = one / g; + for (j = l; j < nCols; j++) { + for (s = zero, k = l; k < nRows; k++) + s += a[k][i] * a[k][j]; + f = (s / a[i][i]) * g; + for (k = i; k < nRows; k++) { + a[k][j] += f * a[k][i]; + } + } + for (j = i; j < nRows; j++) { + a[j][i] *= g; + } + } else { + for (j = i; j < nRows; j++) + a[j][i] = zero; + } + ++a[i][i]; + printf("|"); + fflush(stdout); + } + + // Check if diagonal entries = superdiagonal + for (k = 0; k > nCols; k++) { + if (rv1[k] > w[k] - epsilon && rv1[k] < w[k] + epsilon) { + cout << "WARNING diagonal equals superdiagonal\n"; + throw "security vulnerability"; + } + } + + // Diagonalization of the bidiagonal form: loop over singular + // values and over allowed iterations + for (k = nCols - 1; k >= 0; k--) { + +#if PPL_FLOW == PPL_FLOW_SiSL + for (its = 0; its < 2; its++) { + cout << k; + l = 0; + nm = -1; + flag = 0; +#else + for (its = 0; its < 30; its++) { + cout << k; + flag = 1; + for (l = k; l >= 0; l--) { // test for splitting + nm = l - 1; // note rv1[0] is always zero + // if ((fabs(rv1[l]) + anorm) == anorm) { // not friendly to fixed point + if (checkeq(rv1[l], anorm)) { + flag = 0; + break; + } + + assert(nm >= 0); // sanity check, should never happen since rv1[0] = 0 + // if ((fabs(w[nm]) + anorm) == anorm) { // not friendly to fixed point + if (checkeq(w[nm], anorm)) { + break; + } + } + +#endif + // flag check needs to be data obl + if (flag) { // cancellation of rv1(l), if l >= 1 + c = zero; + s = one; + for (i = l; i <= k; i++) { + f = s * rv1[i]; + rv1[i] = c * rv1[i]; + // if ((fabs(f) + anorm) == anorm) { // not friendly to fixed point + if (checkeq(f, anorm)) { + break; + } + g = w[i]; + h = mypythag(f, g); + w[i] = h; + h = one / h; + c = g * h; + s = -f * h; + for (j = 0; j < nRows; j++) { + y = a[j][nm]; + z = a[j][i]; + a[j][nm] = y * c + z * s; + a[j][i] = z * c - y * s; + } + } + } + z = w[k]; + if (l == k) { // convergence + if (z < zero) { // singular value is made nonnegative + w[k] = -z; + for (j = 0; j < nCols; j++) + v[j][k] = -v[j][k]; + } + break; + } + if (its == 29) { + printf("no convergence in 30 svdcmp iterations\n"); + delete[] rv1; + return -1; + } + +#if PPL_FLOW == PPL_FLOW_SiSL + if (k == 0) + break; +#endif + + // shift from bottom 2-by-2 minor + x = w[l]; + nm = k - 1; + y = w[nm]; + g = rv1[nm]; + h = rv1[k]; + f = ((y - z) * (y + z) + (g - h) * (g + h)) / (two * h * y); + // T denom = two * h * y; + // f = ((y - z) * (y + z))/denom + ((g - h) * (g + h))/denom; + g = mypythag(f, one); + // f = ((x - z) * (x + z) + h * ((y / (f + mysign(g,f)))- h)) / x; // + // overflows on x-z * x+z + f = ((x - z) / x * (x + z)) + (h * ((y / (f + mysign(g, f))) - h)) / x; + // Next QR transformation + c = s = one; + for (j = l; j <= nm; j++) { + i = j + 1; + g = rv1[i]; + y = w[i]; + h = s * g; + g = c * g; + z = mypythag(f, h); + rv1[j] = z; + c = f / z; + s = h / z; + f = x * c + g * s; + g = g * c - x * s; + h = y * s; + y *= c; + for (jj = 0; jj < nCols; jj++) { + x = v[jj][j]; + z = v[jj][i]; + v[jj][j] = x * c + z * s; + v[jj][i] = z * c - x * s; + } + z = mypythag(f, h); + w[j] = z; // rotation can be artibitrary if z=0 + if (z) { + z = one / z; + c = f * z; + s = h * z; + } + f = c * g + s * y; + x = c * y - s * g; + for (jj = 0; jj < nRows; jj++) { + y = a[jj][j]; + z = a[jj][i]; + a[jj][j] = y * c + z * s; + a[jj][i] = z * c - y * s; + } + } + rv1[l] = zero; + rv1[k] = f; + w[k] = x; + } + printf("/"); + fflush(stdout); + } + printf("\n"); + delete[] rv1; + return 0; +} diff --git a/test/common-test/cleartext-ref/trigfuncs.hpp b/test/common-test/cleartext-ref/trigfuncs.hpp new file mode 100644 index 0000000..00c2cd4 --- /dev/null +++ b/test/common-test/cleartext-ref/trigfuncs.hpp @@ -0,0 +1,17 @@ +#pragma once +#include +#include +#include + +template +T mysin(T x) { + // return x - (pow(x,3)/(3*2)) + (pow(x,5)/(5*4*3*2));// - + // (pow(x,7)/(7*6*5*4*3*2)); + return sin(x); +} + +template +T mycos(T x) { + // return 1 - (pow(x,2)/2) + (pow(x,4)/(4*3*2));// - (pow(x,6)/(6*5*4*3*2)); + return cos(x); +} diff --git a/test/common-test/cleartext-ref/twonormsq.hpp b/test/common-test/cleartext-ref/twonormsq.hpp new file mode 100644 index 0000000..8de7ac0 --- /dev/null +++ b/test/common-test/cleartext-ref/twonormsq.hpp @@ -0,0 +1,12 @@ +#pragma once +// M mxn +// N mmxnn +// res mxnn +template +T twonormsq(T* vect, int sz) { + T sum = 0.0; + for (int i = 0; i < sz; i++) { + sum += vect[i] * vect[i]; + } + return sum; +} diff --git a/test/common-test/eth3d_features.hpp b/test/common-test/eth3d_features.hpp new file mode 100644 index 0000000..076180e --- /dev/null +++ b/test/common-test/eth3d_features.hpp @@ -0,0 +1,280 @@ +//#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using cv::DualQuatd; +using cv::Mat; +using cv::Point2f; +using cv::Point3_; +using cv::Point3f; +using cv::Point_; +using cv::Quatd; +using cv::Vec3f; + +// vector eth3d_indoor_locations = { +////"courtyard/", +//"delivery_area/", +////"electro/", +////"facade/", +//"kicker/", +////"meadow/", +//"office/", +//"pipes/", +////"playground/", +//"relief/", +//"relief_2/", +////"terrace/", +//"terrains/", +//}; +// +// vector eth3d_outdoor_locations = { +//"courtyard/", +////"delivery_area/", +//"electro/", +//"facade/", +////"kicker/", +//"meadow/", +////"office/", +////"pipes/", +//"playground/", +////"relief/", +////"relief_2/", +//"terrace/", +////"terrains/", +//}; + +std::vector eth3d_locations = { + "courtyard", "delivery_area", "electro", "facade", "kicker", + "meadow", "office", "pipes", "playground", "relief", + "relief_2", "terrace", "terrains", +}; + +// typedef Eigen::Transform SE3f; + +struct ColmapFeatureObservation { + // EIGEN_MAKE_ALIGNED_OPERATOR_NEW + + // Sub-pixel coordinates of the observation in its image, given in pixels. + // Eigen::Vector2f xy; + Point2f xy; + + // Id of the corresponding 3D point or -1 if no 3D point is associated to this + // observation. + int point3d_id; +}; + +// Holds data of a COLMAP image. +struct ColmapImage { + // Unique image id. + int image_id; + + // Id of the camera model for this image. + int camera_id; + + // Path to the image file, may be a relative path. + std::string file_path; + + // Global-to-image transformation. + // SE3f image_T_global; + DualQuatd image_T_global; + + // Image-to-global transformation. + // SE3f global_T_image; + DualQuatd global_T_image; + + // 2D feature observations in this image. + // std::vector> observations; + std::vector observations; +}; + +typedef std::shared_ptr ColmapImagePtr; +typedef std::shared_ptr ColmapImageConstPtr; + +typedef std::vector ColmapImagePtrVector; +typedef std::unordered_map ColmapImagePtrMap; + +template +class ETH3DFeatures { + private: + std::unordered_map> all_world_features; + ColmapImagePtrMap images; + bool error = false; + std::vector frame_to_id; + int frame_number = 0; + + public: + ETH3DFeatures(const std::string& base_path, const std::string& location) { + const std::string calib_path = "/dslr_calibration_undistorted/"; + std::string images_txt_path = + base_path + location + calib_path + "images.txt"; + std::string points3d_txt_path = + base_path + location + calib_path + "points3D.txt"; + + // open images.txt file + std::ifstream images_file_stream(images_txt_path, std::ios::in); + if (!images_file_stream) { + ERROR_MSG("Could not open images.txt file\n"); + error = true; + return; + } + + // read all 2d points + while (!images_file_stream.eof() && !images_file_stream.bad()) { + std::string line; + std::getline(images_file_stream, line); + if (line.size() == 0 || line[0] == '#') + continue; + + // Read image info line. + ColmapImage* new_image = new ColmapImage(); + // Eigen::Quaternionf image_R_global; + float qw, qx, qy, qz, tx, ty, tz; + std::istringstream image_stream(line); + image_stream >> new_image->image_id >> qw >> qx >> qy >> qz >> tx >> ty >> + tz >> new_image->camera_id >> new_image->file_path; + + Quatd iTg_rot(qw, qx, qy, qz); + // new_image->image_T_global.linear() = image_R_global.toRotationMatrix(); + new_image->image_T_global = DualQuatd::createFromAngleAxisTrans( + iTg_rot + .norm(), // angle is the norm of rotation vector or quaternion? + iTg_rot.toRotVec(), // rvec describes rotation axis + {tx, ty, tz} // transation vector is easy + ); + // new_image->global_T_image = new_image->image_T_global.inverse(); + new_image->global_T_image = new_image->image_T_global.inv(); + + // Read feature observations line. + std::getline(images_file_stream, line); + std::istringstream observations_stream(line); + while (!observations_stream.eof() && !observations_stream.bad()) { + new_image->observations.emplace_back(); + ColmapFeatureObservation* new_observation = + &new_image->observations.back(); + // observations_stream >> new_observation->xy.x() + // >> new_observation->xy.y() + // >> new_observation->point3d_id; + observations_stream >> new_observation->xy.x >> new_observation->xy.y >> + new_observation->point3d_id; + } + + images.insert( + std::make_pair(new_image->image_id, ColmapImagePtr(new_image))); + frame_to_id.push_back(new_image->image_id); + } + + // open points3d.txt file + std::ifstream points3d_file_stream(points3d_txt_path, std::ios::in); + if (!points3d_file_stream) { + ERROR_MSG("Could not open points3d.txt file\n"); + error = true; + return; + } + + while (!points3d_file_stream.eof() && !points3d_file_stream.bad()) { + std::string line; + std::getline(points3d_file_stream, line); + if (line.size() == 0 || line[0] == '#') + continue; + + // Read 3d point line. + int id; + Point3_ p; + std::istringstream image_stream(line); + image_stream >> id >> p.x >> p.y >> p.z; + //>> red + //>> green + //>> blue + //>> reprojection error + //>> in these 2d images + all_world_features[id] = p; + } + } + + void imageFeatures(std::vector>& points) { + if (error) { + ERROR_MSG("ETH3DFeatures not properly configured\n"); + return; + } + + points.resize(0); + int id = frame_to_id[frame_number]; + + for (auto& obs : images[id]->observations) { + if (obs.point3d_id == -1) + continue; + points.push_back({obs.xy.x, obs.xy.y}); + } + } + + void worldFeatures(std::vector>& points) { + if (error) { + ERROR_MSG("ETH3DFeatures not properly configured\n"); + return; + } + + int id = frame_to_id[frame_number]; + + // for each 2d point find 3d correspondance + for (auto& obs : images[id]->observations) { + if (obs.point3d_id == -1) + continue; // no correspondance found by dataset owners + + const auto& f = all_world_features.find(obs.point3d_id); + if (f == all_world_features.end()) { + ERROR_MSG("2d-3d point correspondence could not be made\n"); + MSG("offending 2d point ID: %d\n", obs.point3d_id); + + } else { + points.push_back({f->second.x, f->second.y, f->second.z}); + } + } + } + + std::vector getGroundTruthPose() { + if (error) { + ERROR_MSG("ETH3DFeatures not properly configured\n"); + return {}; + } + + int id = frame_to_id[frame_number]; + auto m = images[id]->image_T_global; // not global_T_image, right? + auto r = m.getRotation().toRotVec(); + auto t = m.getTranslation(); + std::vector pose = {static_cast(r[0]), static_cast(r[1]), + static_cast(r[2]), static_cast(t[0]), + static_cast(t[1]), static_cast(t[2])}; + return pose; + } + + std::string getRelImageFilePath() { + int id = frame_to_id[frame_number]; + return images[id]->file_path; + } + + void nextFrame() { + if (error) { + ERROR_MSG("ETH3DFeatures not properly configured\n"); + return; + } + frame_number++; + } + int numberOfFrames() { + if (error) { + ERROR_MSG("ETH3DFeatures not properly configured\n"); + return 0; + } + return frame_to_id.size(); + } +}; diff --git a/test/common-test/eth3d_tester.cpp b/test/common-test/eth3d_tester.cpp new file mode 100644 index 0000000..38d02f2 --- /dev/null +++ b/test/common-test/eth3d_tester.cpp @@ -0,0 +1,50 @@ +#include +#include +#include +#include + +#include +#include + +#include + +#include + +using cv::Point2f; + +int main(int argc, char** argv) { + char datadir[PATH_MAX]; + strncpy(datadir, argv[0], sizeof(datadir)); + dirname(datadir); + std::string bindir(datadir); + + std::string base_path = bindir + "/../../data-eth3d/"; + auto feats = ETH3DFeatures(base_path, eth3d_locations[0]); + + std::string rel_image_path = feats.getRelImageFilePath(); + std::string image_path = base_path + "images/" + rel_image_path; + + std::cout << "First image directory:" << std::endl; + std::cout << image_path << std::endl; + + std::cout << "Getting points..." << std::flush; + std::vector image_points; + std::vector world_points; + feats.imageFeatures(image_points); + feats.worldFeatures(world_points); + auto res = feats.getGroundTruthPose(); + std::cout << " done\n"; + + std::cout << "first point\n"; + std::cout << "2d " << image_points[0].x << " " << image_points[0].y + << std::endl; + std::cout << "3d " << world_points[0].x << " " << world_points[0].y << " " + << world_points[0].z << std::endl; + + std::cout << "pose: "; + for (int i = 0; i < 6; i++) + std::cout << res[i] << " "; + std::cout << std::endl; + + return 0; +} diff --git a/test/common-test/hoff_features.hpp b/test/common-test/hoff_features.hpp new file mode 100644 index 0000000..75887d8 --- /dev/null +++ b/test/common-test/hoff_features.hpp @@ -0,0 +1,26 @@ +#pragma once +#include +#include +#include +#include +#include + +template +void Hoffs3DPoints(std::vector>& points) { + points.push_back(cv::Point3_(0, 10, 6)); + points.push_back(cv::Point3_(0, 2, 6)); + points.push_back(cv::Point3_(2, 0, 6)); + points.push_back(cv::Point3_(0, 10, 2)); + points.push_back(cv::Point3_(0, 2, 2)); + points.push_back(cv::Point3_(2, 0, 2)); +} + +template +void Hoffs2DPoints(std::vector>& points) { + points.push_back(cv::Point_(183, 147)); + points.push_back(cv::Point_(350, 133)); + points.push_back(cv::Point_(454, 144)); + points.push_back(cv::Point_(176, 258)); + points.push_back(cv::Point_(339, 275)); + points.push_back(cv::Point_(444, 286)); +} diff --git a/test/common-test/jlog.h b/test/common-test/jlog.h new file mode 100644 index 0000000..a6e1c3d --- /dev/null +++ b/test/common-test/jlog.h @@ -0,0 +1,279 @@ +#ifndef __JLOG_H +#define __JLOG_H + +//////////////////////// +// Logging Intrinsics // +//////////////////////// +#include + +#define RESET "\033[0m" +#define BLACK "\033[30m" /* Black */ +#define RED "\033[31m" /* Red */ +#define GREEN "\033[32m" /* Green */ +#define YELLOW "\033[33m" /* Yellow */ +#define BLUE "\033[34m" /* Blue */ +#define MAGENTA "\033[35m" /* Magenta */ +#define CYAN "\033[36m" /* Cyan */ +#define WHITE "\033[37m" /* White */ +#define BOLDBLACK "\033[1m\033[30m" /* Bold Black */ +#define BOLDRED "\033[1m\033[31m" /* Bold Red */ +#define BOLDGREEN "\033[1m\033[32m" /* Bold Green */ +#define BOLDYELLOW "\033[1m\033[33m" /* Bold Yellow */ +#define BOLDBLUE "\033[1m\033[34m" /* Bold Blue */ +#define BOLDMAGENTA "\033[1m\033[35m" /* Bold Magenta */ +#define BOLDCYAN "\033[1m\033[36m" /* Bold Cyan */ +#define BOLDWHITE "\033[1m\033[37m" /* Bold White */ + +#define MSG(...) \ + do { \ + printf(__VA_ARGS__); \ + } while (0) + +#define MSGF(enable, ...) \ + do { \ + if (_verbosity & enable) { \ + printf(MAGENTA "[" BOLDMAGENTA "%s" MAGENTA "] " RESET, #enable); \ + printf(__VA_ARGS__); \ + } \ + } while (0) + +#define MSGO(...) \ + do { \ + static uint8_t printed = 0; \ + if (!printed) { \ + printf(__VA_ARGS__); \ + printed = 1; \ + } \ + } while (0) + +#define DEBUG_MSG(...) \ + do { \ + printf(BLUE "(%s:%d - %s)" RESET "\n", __FILE__, __LINE__, __FUNCTION__); \ + printf(__VA_ARGS__); \ + } while (0) + +#define WARNING_MSG(...) \ + do { \ + printf(YELLOW "WARNING! (%s:%d - %s)" RESET "\n", __FILE__, __LINE__, \ + __FUNCTION__); \ + printf(__VA_ARGS__); \ + } while (0) + +#define ERROR_MSG(...) \ + do { \ + fprintf(stderr, RED "ERROR! (%s:%d - %s)" RESET "\n", __FILE__, __LINE__, \ + __FUNCTION__); \ + fprintf(stderr, __VA_ARGS__); \ + } while (0) + +///////////////////////// +// CSV Plotting Tools /// +///////////////////////// + +#include + +// TIC TOC only counts process time, it will not count sleeps or io waits +#define CLOCK(ID_) static clock_t tic##ID_; + +#define TIC(ID_) tic##ID_ = clock(); + +#define TOC(ID_) \ + do { \ + clock_t toc##ID_ = clock(); \ + if (tic##ID_ > toc##ID_) { \ + MSG(BLUE "%s" RESET " - " RED "Timer %s Overflowed" RESET "\n", \ + __FUNCTION__, #ID_); \ + } else { \ + MSG(BLUE "%s" RESET " - " GREEN "Timer %s: %g s" RESET "\n", \ + __FUNCTION__, #ID_, (toc##ID_ - tic##ID_) / (double)CLOCKS_PER_SEC); \ + } \ + } while (0) + +#define TOC_CSV_XY(ID_, X_) \ + do { \ + clock_t toc##ID_ = clock(); \ + if (tic##ID_ > toc##ID_) { \ + ERROR_MSG("SeNtInAl,error,%s,%s,%d,overflow\n", __FUNCTION__, #ID_, X_); \ + } else { \ + MSG("SeNtInAl,xy,%s,%s,%d,%g\n", __FUNCTION__, #ID_, X_, \ + (toc##ID_ - tic##ID_) / (double)CLOCKS_PER_SEC); \ + } \ + } while (0) + +// will stack ID's with same XLABEL_ on top +#define TOC_CSV_BAR(ID_, XLABEL_) \ + do { \ + clock_t toc##ID_ = clock(); \ + if (tic##ID_ > toc##ID_) { \ + ERROR_MSG("SeNtInAl,error,%s,%s,%s,overflow\n", __FUNCTION__, #ID_, \ + XLABEL_); \ + } else { \ + MSG("SeNtInAl,bar,%s,%s,%s,%g\n", __FUNCTION__, #ID_, XLABEL_, \ + (toc##ID_ - tic##ID_) / (double)CLOCKS_PER_SEC); \ + } \ + } while (0) + +#define TOC_CSV_GROUPED_BAR(ID_, XGROUP_) \ + do { \ + clock_t toc##ID_ = clock(); \ + if (tic##ID_ > toc##ID_) { \ + ERROR_MSG("SeNtInAl,error,%s,%s,%s,overflow\n", __FUNCTION__, #ID_, \ + XGROUP_); \ + } else { \ + MSG("SeNtInAl,bar,%s,%s,%s,%g\n", __FUNCTION__, #ID_, XGROUP_, \ + (toc##ID_ - tic##ID_) / (double)CLOCKS_PER_SEC); \ + } \ + } while (0) + +#define TOC_CSV_BOX(ID_) \ + do { \ + clock_t toc##ID_ = clock(); \ + if (tic##ID_ > toc##ID_) { \ + ERROR_MSG("SeNtInAl,error,%s,%s,no-x,overflow\n", __FUNCTION__, #ID_); \ + } else { \ + MSG("SeNtInAl,box,%s,%s,no-x,%g\n", __FUNCTION__, #ID_, \ + (toc##ID_ - tic##ID_) / (double)CLOCKS_PER_SEC); \ + } \ + } while (0) + +// Wall clock measures absolute time + +#include +#include +#include +using namespace std::chrono; + +#define WALL_CLOCK(ID_) static time_point tic##ID_; + +#define WALL_TIC(ID_) tic##ID_ = high_resolution_clock::now(); + +#define WALL_TOC(ID_) \ + do { \ + time_point toc##ID_ = high_resolution_clock::now(); \ + if (tic##ID_ > toc##ID_) { \ + MSG(BLUE "%s" RESET " - " RED "Timer %s Overflowed" RESET "\n", \ + __FUNCTION__, #ID_); \ + } else { \ + MSG(BLUE "%s" RESET " - " GREEN "Timer %s: %g s" RESET "\n", \ + __FUNCTION__, #ID_, \ + std::chrono::duration_cast(toc##ID_ - \ + tic##ID_) \ + .count() / \ + 1000000.0); \ + } \ + } while (0) + +#define WALL_TOC_CSV_XY(ID_, X_) \ + do { \ + time_point toc##ID_ = high_resolution_clock::now(); \ + if (tic##ID_ > toc##ID_) { \ + ERROR_MSG("SeNtInAl,error,%s,%s,%d,overflow\n", __FUNCTION__, #ID_, X_); \ + } else { \ + MSG("SeNtInAl,xy,%s,%s,%d,%g\n", __FUNCTION__, #ID_, X_, \ + std::chrono::duration_cast(toc##ID_ - \ + tic##ID_) \ + .count() / \ + 1000000.0); \ + } \ + } while (0) + +#define WALL_TOC_CSV_XY_NORMALIZED(ID_, X_, DIV_) \ + do { \ + time_point toc##ID_ = high_resolution_clock::now(); \ + if (tic##ID_ > toc##ID_) { \ + ERROR_MSG("SeNtInAl,error,%s,%s,%d,overflow\n", __FUNCTION__, #ID_, X_); \ + } else { \ + MSG("SeNtInAl,xy,%s,%s_normalized,%d,%g\n", __FUNCTION__, #ID_, X_, \ + std::chrono::duration_cast(toc##ID_ - \ + tic##ID_) \ + .count() / \ + (1000000.0 * DIV_)); \ + } \ + } while (0) + +// will stack ID's with same XLABEL on top +#define WALL_TOC_CSV_BAR(ID_, XLABEL_) \ + do { \ + time_point toc##ID_ = high_resolution_clock::now(); \ + if (tic##ID_ > toc##ID_) { \ + ERROR_MSG("SeNtInAl,error,%s,%s,%s,overflow\n", __FUNCTION__, #ID_, \ + XLABEL_); \ + } else { \ + MSG("SeNtInAl,bar,%s,%s,%s,%g\n", __FUNCTION__, #ID_, XLABEL_, \ + std::chrono::duration_cast(toc##ID_ - \ + tic##ID_) \ + .count() / \ + 1000000.0); \ + } \ + } while (0) + +#define WALL_TOC_CSV_GROUPED_BAR(ID_, XGROUP_) \ + do { \ + time_point toc##ID_ = high_resolution_clock::now(); \ + if (tic##ID_ > toc##ID_) { \ + ERROR_MSG("SeNtInAl,error,%s,%s,%s,overflow\n", __FUNCTION__, #ID_, \ + XGROUP_); \ + } else { \ + MSG("SeNtInAl,grouped_bar,%s,%s,%s,%g\n", __FUNCTION__, #ID_, XGROUP_, \ + std::chrono::duration_cast(toc##ID_ - \ + tic##ID_) \ + .count() / \ + 1000000.0); \ + } \ + } while (0) + +#define WALL_TOC_CSV_BOX(ID_) \ + do { \ + static time_point toc##ID_ = \ + high_resolution_clock::now(); \ + if (tic##ID_ > toc##ID_) { \ + ERROR_MSG("SeNtInAl,error,%s,%s,no-x,overflow\n", __FUNCTION__, #ID_); \ + } else { \ + MSG("SeNtInAl,box,%s,%s,no-x,%g\n", __FUNCTION__, #ID_, \ + std::chrono::duration_cast(toc##ID_ - \ + tic##ID_) \ + .count() / \ + 1000000.0); \ + } \ + } while (0) + +// Counter +#define COUNTER(ID_) static uint32_t count##ID_ = 0; + +#define COUNTER_RESET(ID_, VAL_) \ + do { \ + count##ID_ = VAL_; \ + } while (0) + +#define COUNTER_INC(ID_, INC_BY_) \ + do { \ + static uint32_t tmp = count##ID_ + INC_BY_; \ + if (count##ID_ > tmp) { \ + printf("SeNtInAl,error,%s,%s,no-x,overflow\n", __FUNCTION__, #ID_); \ + } \ + count##ID_ = tmp; \ + } while (0) + +#define COUNTER_PRINT(ID_, X_) \ + do { \ + MSG(BLUE "%s" RESET " - " GREEN "Counter %s: \t%d,\t%d" RESET "\n", \ + __FUNCTION__, #ID_, X_, count##ID_); \ + } while (0) + +#define COUNTER_CSV_XY(ID_, X_) \ + do { \ + printf("SeNtInAl,xy,%s,%s,%d,%d\n", __FUNCTION__, #ID_, X_, count##ID_); \ + } while (0) + +#define COUNTER_CSV_BOX(ID_) \ + do { \ + printf("SeNtInAl,box,%s,%s,no-x,%d\n", __FUNCTION__, #ID_, count##ID_); \ + } while (0) + +// Histogram +#define HISTOCSV(ID_, VAL_) \ + do { \ + printf("SeNtInAl,histogram,%s,%s,no-x,%d\n", __FUNCTION__, #ID_, VAL_); \ + } while (0) + +#endif diff --git a/test/common-test/kitti_tester.cpp b/test/common-test/kitti_tester.cpp new file mode 100644 index 0000000..ce9d787 --- /dev/null +++ b/test/common-test/kitti_tester.cpp @@ -0,0 +1,318 @@ +// Adapted from +// https://github.com/mez/monocular-visual-odometry/blob/master/main.cpp +// https://github.com/avisingh599/mono-vo/blob/master/src/visodo.cpp +// explained here https://avisingh599.github.io/vision/monocular-vo/ +// One has MIT license, one does not... Not sure who copied whom. +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/** + ***************** Algorithm Outline + 1. Capture images: It, It+1, + 2. Undistort the above images. (using kitti so this step is done for you!) + 3. Use FAST algorithm to detect features in It, and track those features to + It+1. A new detection is triggered if the number of features drop below a + certain threshold. + 4. Use Nister’s 5-point alogirthm with RANSAC to compute the essential + matrix. + 5. Estimate R,t from the essential matrix that was computed in the previous + step. + 6. Take scale information from some external source (like a speedometer), + and concatenate the translation vectors, and rotation matrices. + *************************/ + +using namespace std; +using cv::KeyPoint; +using cv::Mat; +using cv::Point2f; +using cv::Size; +using cv::TermCriteria; + +#define MAX_FRAME 2000 +#define MIN_NUM_FEAT 2000 + +// const static char* dataset_images_location = +// "/home/mez/dataset/kitti/dataset/sequences/00/image_1"; const static char* +// dataset_poses_location = +// "/home/mez/dataset/kitti/odomerty_poses/poses/00.txt"; +const static char* dataset_images_location = + "../../data-kitti/dataset/sequences/00/image_1"; +const static char* dataset_poses_location = + "../../data-kitti/dataset/poses/poses/00.txt"; + +vector getGreyCamGroundPoses() { + string line; + int i = 0; + ifstream myfile(dataset_poses_location); + double value = 0; + vector poses; + if (myfile.is_open()) { + while (getline(myfile, line)) { + Point2f pose; + std::istringstream in(line); + for (int j = 0; j < 12; j++) { + in >> value; + if (j == 11) + pose.y = value; + if (j == 3) + pose.x = value; + } + + poses.push_back(pose); + i++; + } + myfile.close(); + } + + return poses; +} + +vector getAbsoluteScales() { + + vector scales; + string line; + int i = 0; + ifstream myfile(dataset_poses_location); + double x = 0, y = 0, z = 0; + double x_prev, y_prev, z_prev; + if (myfile.is_open()) { + while (getline(myfile, line)) { + z_prev = z; + x_prev = x; + y_prev = y; + std::istringstream in(line); + // cout << line << '\n'; + for (int j = 0; j < 12; j++) { + in >> z; + if (j == 7) + y = z; + if (j == 3) + x = z; + } + + scales.push_back(sqrt((x - x_prev) * (x - x_prev) + + (y - y_prev) * (y - y_prev) + + (z - z_prev) * (z - z_prev))); + } + myfile.close(); + } + + return scales; +} + +void featureTracking(Mat img_1, Mat img_2, vector& points1, + vector& points2, vector& status) { + // this function automatically gets rid of points for which tracking fails + + vector err; + Size winSize(21, 21); + TermCriteria termcrit(TermCriteria::COUNT + TermCriteria::EPS, 30, 0.01); + + calcOpticalFlowPyrLK(img_1, img_2, points1, points2, status, err, winSize, 3, + termcrit, 0, 0.001); + + // getting rid of points for which the KLT tracking failed or those who have + // gone outside the frame + int indexCorrection = 0; + for (int i = 0; i < status.size(); i++) { + Point2f pt = points2.at(i - indexCorrection); + if ((status.at(i) == 0) || (pt.x < 0) || (pt.y < 0)) { + if ((pt.x < 0) || (pt.y < 0)) { + status.at(i) = 0; + } + + points1.erase(points1.begin() + i - indexCorrection); + points2.erase(points2.begin() + i - indexCorrection); + indexCorrection++; + } + } +} + +void featureDetection(Mat img, vector& points) { + vector keypoints; + int fast_threshold = 20; + bool nonmaxSupression = true; + + cv::FAST(img, keypoints, fast_threshold, nonmaxSupression); + KeyPoint::convert(keypoints, points, vector()); +} + +int main(int argc, char** argv) { + + bool renderFeatures = false; + if (argc > 1) { + renderFeatures = true; + } + + Mat img_1, img_2; + Mat R_f, t_f; + + double scale = 1.00; + char filename1[200]; + char filename2[200]; + sprintf(filename1, "%s/%06d.png", dataset_images_location, 0); + sprintf(filename2, "%s/%06d.png", dataset_images_location, 1); + + char text[100]; + int fontFace = cv::FONT_HERSHEY_PLAIN; + double fontScale = 1; + int thickness = 1; + cv::Point textOrg(10, 50); + + // read the first two frames from the dataset + Mat img_1_c = cv::imread(filename1); + Mat img_2_c = cv::imread(filename2); + + if (!img_1_c.data || !img_2_c.data) { + std::cout << " --(!) Error reading images " << std::endl; + return -1; + } + + // we work with grayscale images + cvtColor(img_1_c, img_1, cv::COLOR_BGR2GRAY); + cvtColor(img_2_c, img_2, cv::COLOR_BGR2GRAY); + + // feature detection, tracking + vector points1, + points2; // vectors to store the coordinates of the feature points + featureDetection(img_1, points1); // detect features in img_1 + vector status; + featureTracking(img_1, img_2, points1, points2, + status); // track those features to img_2 + + // TODO: add a fucntion to load these values directly from KITTI's calib files + // WARNING: different sequences in the KITTI VO dataset have different + // intrinsic/extrinsic parameters + /* + * Projection matrix example after rectification: Right Grey scale Camera + * (3x4) + * + * 7.188560000000e+02 0.000000000000e+00 6.071928000000e+02 + * -3.861448000000e+02 + * 0.000000000000e+00 7.188560000000e+02 1.852157000000e+02 + * 0.000000000000e+00 0.000000000000e+00 0.000000000000e+00 1.000000000000e+00 + * 0.000000000000e+00 + */ + double focal = 718.8560; + cv::Point2d pp(607.1928, 185.2157); + + // recovering the pose and the essential matrix + Mat E, R, t, mask; + E = findEssentialMat(points2, points1, focal, pp, cv::RANSAC, 0.999, 1.0, + mask); + recoverPose(E, points2, points1, R, t, focal, pp, mask); + + Mat prevImage = img_2; + Mat currImage; + vector prevFeatures = points2; + vector currFeatures; + + char filename[100]; + + R_f = R.clone(); + t_f = t.clone(); + + clock_t begin = clock(); + + cv::namedWindow("Road facing camera | Top Down Trajectory", + cv::WINDOW_AUTOSIZE); // Create a window for display. + + Mat traj = Mat::zeros(600, 1241, CV_8UC3); + + auto groundPoses = getGreyCamGroundPoses(); + auto groundScales = getAbsoluteScales(); + + for (int numFrame = 2; numFrame < MAX_FRAME; numFrame++) { + sprintf(filename, "%s/%06d.png", dataset_images_location, numFrame); + + Mat currImage_c = cv::imread(filename); + cvtColor(currImage_c, currImage, cv::COLOR_BGR2GRAY); + vector status; + featureTracking(prevImage, currImage, prevFeatures, currFeatures, status); + + E = findEssentialMat(currFeatures, prevFeatures, focal, pp, cv::RANSAC, + 0.999, 1.0, mask); + recoverPose(E, currFeatures, prevFeatures, R, t, focal, pp, mask); + + Mat prevPts(2, prevFeatures.size(), CV_64F), + currPts(2, currFeatures.size(), CV_64F); + + for (int i = 0; i < prevFeatures.size(); + i++) { // this (x,y) combination makes sense as observed from the + // source code of triangulatePoints on GitHub + prevPts.at(0, i) = prevFeatures.at(i).x; + prevPts.at(1, i) = prevFeatures.at(i).y; + + currPts.at(0, i) = currFeatures.at(i).x; + currPts.at(1, i) = currFeatures.at(i).y; + } + + // This is cheating because ideally you'd want to figure out a way to get + // scale, but without this cheat there is a lot of drift. + scale = groundScales[numFrame]; + + // only update the current R and t if it makes sense. + if ((scale > 0.1) && (t.at(2) > t.at(0)) && + (t.at(2) > t.at(1))) { + + t_f = t_f + scale * (R_f * t); + R_f = R * R_f; + } + + // Make sure we have enough features to track + if (prevFeatures.size() < MIN_NUM_FEAT) { + featureDetection(prevImage, prevFeatures); + featureTracking(prevImage, currImage, prevFeatures, currFeatures, status); + } + + prevImage = currImage.clone(); + prevFeatures = currFeatures; + + int x = int(t_f.at(0)) + 600; + int y = int(t_f.at(2)) + 100; + + circle(traj, cv::Point(x, y), 1, CV_RGB(255, 0, 0), 2); + circle( + traj, + cv::Point(groundPoses[numFrame].x + 600, groundPoses[numFrame].y + 100), + 1, CV_RGB(0, 255, 0), 2); + + rectangle(traj, cv::Point(10, 30), cv::Point(550, 50), CV_RGB(0, 0, 0), + cv::FILLED); + sprintf(text, "Coordinates: x = %02fm y = %02fm z = %02fm", + t_f.at(0), t_f.at(1), t_f.at(2)); + putText(traj, text, textOrg, fontFace, fontScale, cv::Scalar::all(255), + thickness, 8); + + if (renderFeatures) { + // Draw features as markers for fun + for (auto point : currFeatures) + cv::drawMarker(currImage_c, cv::Point(point.x, point.y), + CV_RGB(0, 255, 0), cv::MARKER_TILTED_CROSS, 2, 1, + cv::LINE_AA); + } + + Mat concated; + cv::vconcat(currImage_c, traj, concated); + + imshow("Road facing camera | Top Down Trajectory", concated); + + cv::waitKey(1); + } + + clock_t end = clock(); + double elapsed_secs = double(end - begin) / CLOCKS_PER_SEC; + cout << "Total time taken: " << elapsed_secs << "s" << endl; + + // let the user press a key to exit. + cv::waitKey(0); + + return 0; +} diff --git a/test/common-test/test_harness.hpp b/test/common-test/test_harness.hpp new file mode 100644 index 0000000..262ce41 --- /dev/null +++ b/test/common-test/test_harness.hpp @@ -0,0 +1,247 @@ +#include + +#include // dirname +#include // PATH_MAX +#include +#include +#include +#include // readlink + +#include +#include +#include + +#include +#include +#include +#include +#include + +using std::cout; +using std::vector; + +constexpr const int seed = 0x666; + +bool withinRel(float v, float t) { + bool res = true; + res &= fabs(v - t) <= std::max(localization_tol_rel, + fabs(localization_tol_rel * std::max(v, t))); + res &= v != std::numeric_limits::max(); + res &= v != std::numeric_limits::min(); + res &= v != std::numeric_limits::infinity(); + res &= v != std::numeric_limits::quiet_NaN(); + return res; +} + +typedef std::function objectPointsSubset, + vector imagePointsSubset, vector& res, + const vector& groundTruthRes)> + localize_test_cb; + +int eth3d_test_harness(int num_frames, int num_trials, int max_num_pts, + localize_test_cb cleartext_localize_func, + localize_test_cb secure_localize_func, bool silent) { + uint32_t cv_successes = 0; + uint32_t cleartext_successes = 0; + uint32_t secure_successes = 0; + uint32_t total_runs = 0; + + float f = 3408.57; + float cx = 3114.7; + float cy = 2070.92; + float _cM[] = {f, 0, cx, 0, f, cy, 0, 0, 1}; + cv::Mat cameraMatrix = cv::Mat(3, 3, cv::DataType::type, _cM); + cv::Mat distCoeffs = cv::Mat::zeros(4, 1, cv::DataType::type); + cv::Mat rvec = cv::Mat::zeros(3, 1, cv::DataType::type); + cv::Mat tvec = cv::Mat::zeros(3, 1, cv::DataType::type); + vector imagePoints; + vector objectPoints; + + char result[PATH_MAX]; + ssize_t count = readlink("/proc/self/exe", result, PATH_MAX); + const char* path; + if (count != -1) { + path = dirname(result); + } else { + std::cerr << "cant find path\n"; + return 1; + } + std::string base_path = string(path) + "/../../data-eth3d/"; + + // Multiple locations affects the convergence properties of the various gd + // algorithms Single location is better for comparison. + // for (uint32_t l = 0; l < eth3d_locations.size(); l++) { + for (int l = 0; l < 1; l++) { + auto feats = ETH3DFeatures(base_path, eth3d_locations[l]); + + int test_num_frames = MIN(feats.numberOfFrames(), num_frames); + + for (int frame = 0; frame < test_num_frames; frame++) { + imagePoints.clear(); + objectPoints.clear(); + feats.imageFeatures(imagePoints); + feats.worldFeatures(objectPoints); + auto gtpose = feats.getGroundTruthPose(); + vector initialGuess = {0, 0, 0, 0, 0, 1}; + // vector initialGuess = {res.first[0], res.first[1], res.first[2], + // res.second[0], res.second[1], res.second[2]}; + // uint32_t numPts = imagePoints.size(); + // MSG("Found %d points\n", numPts); + // for (int i=0; i dist( + 0, max_image_points); + + for (int num_pts = 6; num_pts <= test_num_pts; + num_pts += MAX((test_num_pts - 6) / 5, 1)) { // at most 5 intervals + + for (int t = 0; t < num_trials; t++) { + if (!silent) { + cout << "Trial " << t << " with " << num_pts + << " randomly selected points on frame " << frame + << " from location " << l << "\n"; + } + ++total_runs; + + // randomly sample i feature pairs from total + vector imagePointsSubset; + vector objectPointsSubset; + for (int p = 0; p < num_pts; p++) { + int r = dist(rng); + imagePointsSubset.push_back(imagePoints[r]); + objectPointsSubset.push_back(objectPoints[r]); + } + + vector opencvRes(6); + { + if (!silent) { + cout << "\ntesting opencv\n"; + } + distCoeffs = cv::Mat::zeros(4, 1, cv::DataType::type); + rvec = cv::Mat::zeros(3, 1, cv::DataType::type); + tvec = cv::Mat::zeros(3, 1, cv::DataType::type); + tvec.at(2) = 1; // cleartext has bug where it fails if z=0 + CLOCK(opencv); + TIC(opencv); + // OpenCV PnP method + cv::solvePnP(objectPointsSubset, imagePointsSubset, cameraMatrix, + distCoeffs, rvec, tvec, false, cv::SOLVEPNP_ITERATIVE); + TOC(opencv); + if (!silent) { + cout << "opencv result:" << endl; + cout << rvec << endl; + cout << tvec << endl << endl; + } + bool good = true; + for (int i = 0; i < 3; ++i) { + good &= withinRel(rvec.at(i), gtpose[i]); + good &= withinRel(tvec.at(i), gtpose[i + 3]); + opencvRes[i] = rvec.at(i); + opencvRes[i + 3] = tvec.at(i); + } + if (good) { + cout << "opencv converged!\n"; + ++cv_successes; + } else { + cout << "opencv did not converge.\n"; + } + + // If opencv does not converge to ground truth but does converge to + // some reasonable value, we can proceed. Inf is not a reasonable + // value and we should retry. + if (std::any_of(opencvRes.begin(), opencvRes.end(), [](float x) { + return x == std::numeric_limits::max(); + })) { + continue; + } + } + + { + if (!silent) { + cout << "testing cleartext\n"; + } + vector res(6, 0); + rvec = cv::Mat::zeros(3, 1, cv::DataType::type); + tvec = cv::Mat::zeros(3, 1, cv::DataType::type); + tvec.at(2) = 1; // cleartext has bug where it fails if z=0 + CLOCK(cleartext); + TIC(cleartext); + cleartext_localize_func(rvec, tvec, cameraMatrix, distCoeffs, + objectPointsSubset, imagePointsSubset, res, + opencvRes); + TOC(cleartext); + if (!silent) { + printVector("cleartext result:\n", &res[0], 6); + } + bool good = true; + for (int i = 0; i < 6; ++i) { + good &= withinRel(res[i], opencvRes[i]); + } + if (good) { + cout << "cleartext converged!\n"; + ++cleartext_successes; + } else { + MSG("cleartext did not converge to same value as opencv.\n"); + continue; + } + } + + { + if (!silent) { + cout << "testing secure\n"; + } + vector res(6, 0); + rvec = cv::Mat::zeros(3, 1, cv::DataType::type); + tvec = cv::Mat::zeros(3, 1, cv::DataType::type); + tvec.at(2) = 1; // cleartext has bug where it fails if z=0 + if (secure_localize_func(rvec, tvec, cameraMatrix, distCoeffs, + objectPointsSubset, imagePointsSubset, res, + opencvRes)) { + ++secure_successes; + } + } + + if (!silent) { + cout << "\n\n"; + } + } + } + feats.nextFrame(); + } + } + if (!silent) { + MSG("SeNtInAl,xy,%s,%s,%d,%u\n", __FUNCTION__, "cv_successes", 0, + cv_successes); + MSG("SeNtInAl,xy,%s,%s,%d,%u\n", __FUNCTION__, "cleartext_successes", 0, + cleartext_successes); + MSG("SeNtInAl,xy,%s,%s,%d,%u\n", __FUNCTION__, "secure_successes", 0, + secure_successes); + MSG("SeNtInAl,xy,%s,%s,%d,%u\n", __FUNCTION__, "total_runs", 0, total_runs); + MSG("OpenCV converged to the ground truth pose %d / %d (%f\%)\n", + cv_successes, total_runs, + static_cast(cv_successes) / total_runs); + MSG("cleartext converged to the opencv pose %d / %d (%f\%)\n", + cleartext_successes, total_runs, + static_cast(cleartext_successes) / total_runs); + MSG("secure localization converged to the opencv pose %d / %d (%f\%)\n", + secure_successes, total_runs, + static_cast(secure_successes) / total_runs); + } + return 0; +} \ No newline at end of file diff --git a/test/common-test/test_params.h b/test/common-test/test_params.h new file mode 100644 index 0000000..3f1311e --- /dev/null +++ b/test/common-test/test_params.h @@ -0,0 +1,8 @@ +#pragma once + +constexpr const float float_test_epsilon = 1e-5; +// reprojection error can be more than epsilon +const constexpr float reprojection_tol = 5.f; +const constexpr float svd_tol = 0.01f; +const constexpr float localization_tol_rel = 0.05f; +const constexpr float localization_tol_abs = 0.1f; diff --git a/test/emp-fixed/CMakeLists.txt b/test/emp-fixed/CMakeLists.txt new file mode 100644 index 0000000..dfa1e7c --- /dev/null +++ b/test/emp-fixed/CMakeLists.txt @@ -0,0 +1,21 @@ +cmake_minimum_required(VERSION 3.12) +set(common_test_dir "${CMAKE_CURRENT_SOURCE_DIR}/../common-test/") +file(GLOB_RECURSE common_test CONFIGURE_DEPENDS "${common_test_dir}/*.cpp" "${common_test_dir}/*.hpp") +list(FILTER common_test EXCLUDE REGEX ".*eth3d_tester\\.cpp$") +list(FILTER common_test EXCLUDE REGEX ".*kitti_tester\\.cpp$") + +# required for emp +set(CMAKE_C_FLAGS "-pthread -Wall -march=native -O3 -maes -mrdseed") +set(CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} -std=c++17") +set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -ggdb -fno-omit-frame-pointer") +set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}") + + +find_package(Boost 1.71.0 REQUIRED) +include_directories(${Boost_INCLUDE_DIR}) + + +add_executable(emp_fixed_server_tester "${CMAKE_CURRENT_SOURCE_DIR}/test.cpp" ${common_test}) +target_include_directories(emp_fixed_server_tester PUBLIC ${common_test_dir}) +target_compile_options(emp_fixed_server_tester PRIVATE "-Wall" "-Wextra") +target_link_libraries(emp_fixed_server_tester ${EMP-SH2PC_LIBRARIES} ${OpenCV_LIBS} EmpFloatLocalization) diff --git a/test/emp-fixed/README.md b/test/emp-fixed/README.md new file mode 100644 index 0000000..1eb0bf2 --- /dev/null +++ b/test/emp-fixed/README.md @@ -0,0 +1,3 @@ +Not implemented! + +Fixed point is projected to be slower than floating for this application diff --git a/test/emp-fixed/test.cpp b/test/emp-fixed/test.cpp new file mode 100644 index 0000000..2709114 --- /dev/null +++ b/test/emp-fixed/test.cpp @@ -0,0 +1,105 @@ +//#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "emp-sh2pc/emp-sh2pc.h" + +#include + +using namespace emp; +using namespace std; + +typedef fixed_point_emp<16, 16> baset; + +int main(int argc, char** argv) { + if (argc != 2) { + MSG("Usage: %s \n", argv[0]); + return 1; + } + + int party = atoi(argv[1]) + 1; + int port = 8080; + cout << "party: " << party << " port: " << port << endl; + NetIO* io = new NetIO(party == ALICE ? nullptr : "127.0.0.1", port); + + { // sanity test + setup_semi_honest(io, party); + float fa = .1; + float fb = -.2; + baset a(fa); + baset b(fb); + baset c = a + b; + baset d = a - b; + baset e = a * b; + baset f = a / b; + cout << fa << "+" << fb << "=" << c << endl; + cout << fa << "-" << fb << "=" << d << endl; + cout << fa << "*" << fb << "=" << e << " " << e.value_.reveal() + << endl; + cout << fa << "/" << fb << "=" << f << endl; + finalize_semi_honest(); + } + + { // fabs + setup_semi_honest(io, party); + baset a(-.1); + baset c = fabs(a); + cout << "fabs(" << a << ")=" << c << endl; + finalize_semi_honest(); + } + + { // trig + setup_semi_honest(io, party); + float fa = .1; + baset a(fa); + baset c = cos(a); + baset s = sin(a); + cout << "cos(" << a << ")=" << c << endl; + cout << "real cos=" << cos(fa) << endl; + cout << "sin(" << a << ")=" << s << endl; + cout << "real sin=" << sin(fa) << endl; + finalize_semi_honest(); + } + + { // sqrt + setup_semi_honest(io, party); + float fa = 16; + baset a(fa); + baset s = sqrt(a); + cout << "sqrt(" << a << ")=" << s << endl; + cout << "real sqrt=" << sqrt(fa) << endl; + finalize_semi_honest(); + } + return 0; +} diff --git a/test/emp-float/CMakeLists.txt b/test/emp-float/CMakeLists.txt new file mode 100644 index 0000000..401457a --- /dev/null +++ b/test/emp-float/CMakeLists.txt @@ -0,0 +1,37 @@ +cmake_minimum_required(VERSION 3.12) +set(common_test_dir "${CMAKE_CURRENT_SOURCE_DIR}/../common-test/") +file(GLOB_RECURSE common_test CONFIGURE_DEPENDS "${common_test_dir}/*.cpp" "${common_test_dir}/*.hpp") +list(FILTER common_test EXCLUDE REGEX ".*eth3d_tester\\.cpp$") +list(FILTER common_test EXCLUDE REGEX ".*kitti_tester\\.cpp$") + +# required for emp +set(CMAKE_C_FLAGS "-pthread -Wall -march=native -O3 -maes -mrdseed") +set(CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} -std=c++17") +set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -ggdb -fno-omit-frame-pointer") +set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}") + +add_executable(emp_float_server_tester + "${CMAKE_CURRENT_SOURCE_DIR}/test.cpp" + ${common_test}) +target_include_directories(emp_float_server_tester PUBLIC + "${CMAKE_CURRENT_SOURCE_DIR}" + ${common_test_dir}) +target_compile_options(emp_float_server_tester PRIVATE "-Wall" "-Wextra" "-DPPL_GN" "-fsanitize=address") +target_link_options(emp_float_server_tester PRIVATE -fsanitize=address) +target_link_libraries(emp_float_server_tester ${EMP-SH2PC_LIBRARIES} ${OpenCV_LIBS} EmpFloatLocalization Catch2::Catch2WithMain) +# data obliv needs higher timeout +catch_discover_tests(emp_float_server_tester PROPERTIES TIMEOUT 2000) + +add_executable(emp_float_eth3d_bench + "${CMAKE_CURRENT_SOURCE_DIR}/eth3d_bench.cpp" + ${common_test}) +target_include_directories(emp_float_eth3d_bench PUBLIC + "${CMAKE_CURRENT_SOURCE_DIR}" + ${common_test_dir}) +target_compile_options(emp_float_eth3d_bench PRIVATE "-Wall" "-Wextra") +target_link_libraries(emp_float_eth3d_bench ${EMP-SH2PC_LIBRARIES} ${OpenCV_LIBS} EmpFloatLocalization) + +add_executable(emp_float_client "${CMAKE_CURRENT_SOURCE_DIR}/client.cpp" ${common_test}) +target_include_directories(emp_float_client PUBLIC ${common_test_dir}) +target_compile_options(emp_float_client PRIVATE "-Wall" "-Wextra") +target_link_libraries(emp_float_client ${EMP-SH2PC_LIBRARIES} ${OpenCV_LIBS} EmpFloatLocalization Catch2::Catch2WithMain) diff --git a/test/emp-float/client.cpp b/test/emp-float/client.cpp new file mode 100644 index 0000000..7e688d6 --- /dev/null +++ b/test/emp-float/client.cpp @@ -0,0 +1,172 @@ +//#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include "emp-sh2pc/emp-sh2pc.h" + +// to get path name +#include +#include +#include +#include + +using namespace emp; +using namespace std; + +PRG prg; +block delta; + +void p128_hex_u32(__m128i in) { + alignas(16) uint32_t v[4]; + _mm_store_si128((__m128i*)v, in); + printf("v4_u32: %x %x %x %x\n", v[0], v[1], v[2], v[3]); +} + +void sendFloatBlock(NetIO* io, float f) { + int* in = (int*)&f; + bool* b = new bool[32]; + int_to_bool(b, *in, 32); + + block label[32]; + prg.random_block(label, 32); + uint32_t mask = 1; + for (int i = 0; i < 32; ++i) { + if (b[i]) { + label[i] = label[i] ^ delta; + } + } + io->send_block(label, 32); + // p128_hex_u32(label[0]); + io->flush(); +} + +void recvFloatBlock(NetIO* aliceio, NetIO* bobio, float* f) { + bool alicelsb = 0; + bool boblsb = 0; + uint32_t res = 0; + for (int i = 0; i < 32; ++i) { + aliceio->recv_data(&alicelsb, 1); + bobio->recv_data(&boblsb, 1); + uint32_t tmp = alicelsb ^ boblsb; + res |= (tmp << i); + } + float* fp = (float*)(&res); + *f = *fp; +} + +int main(int argc, char** argv) { + if (argc != 1) { + MSG("Usage: %s\n", argv[0]); + return 1; + } + MSG("Running\n"); + + int baseport = 8080; + NetIO* aliceio = new NetIO("127.0.0.1", baseport + ALICE * 17); + NetIO* bobio = new NetIO("127.0.0.1", baseport + BOB * 17); + MSG("Connected to alice port: %d, bob port: %d\n", baseport + ALICE * 17, + baseport + BOB * 17); + + vector imagePoints; + vector objectPoints; + + // float f=715; + // float cx=354; + // float cy=245; + // Hoffs2DPoints(imagePoints); + // Hoffs3DPoints(objectPoints); + // vector initialGuess = {1.5, -1.0, 0.0, 0, 0, 30}; + // uint32_t numPts = imagePoints.size(); + + float f = 3408.57; + float cx = 3114.7; + float cy = 2070.92; + char datadir[PATH_MAX]; + strncpy(datadir, argv[0], sizeof(datadir)); + dirname(datadir); + std::string bindir(datadir); + std::string base_path = bindir + "/../../data-eth3d/"; + auto feats = ETH3DFeatures(base_path, eth3d_locations[0]); + feats.imageFeatures(imagePoints); + feats.worldFeatures(objectPoints); + imagePoints.resize(6); + objectPoints.resize(6); + auto gtpose = feats.getGroundTruthPose(); + vector initialGuess = {0, 0, 0, 0, 0, 0}; + // vector initialGuess = {1, 1, -1, 0, 0, 4}; + // vector initialGuess = {res.first[0], res.first[1], res.first[2], + // res.second[0], res.second[1], res.second[2]}; + uint32_t numPts = imagePoints.size(); + MSG("Found %d points\n", numPts); + for (int i = 0; i < imagePoints.size(); i++) { + MSG("Point %d 2d %f, %f\n", i, imagePoints[i].x, imagePoints[i].y); + MSG("Point %d 3d %f, %f, %f\n", i, objectPoints[i].x, objectPoints[i].y, + objectPoints[i].z); + } + printVector("Ground Truth Pose [rotation; translation]: ", >pose[0], 6); + + // Setup + aliceio->send_data(&numPts, sizeof(uint32_t)); + bobio->send_data(&numPts, sizeof(uint32_t)); + aliceio->flush(); + bobio->flush(); + MSG("sent alice and bob numpoints\n"); + + block seed; + prg.random_block(&seed, 1); // used for alice's inputs only + prg.reseed(&seed); + + aliceio->recv_block(&delta, 1); + MSG("got delta from alice\n"); + // p128_hex_u32(delta); + + // Distribute secret input data via 3-way OT + // send prg seed directly to bob so he can generate his own labels + bobio->send_block(&seed, 1); + bobio->flush(); + MSG("sent bob seed\n"); + // p128_hex_u32(seed); + + // must send all labels ^ gc->delta to alice, she cannot learn seed + for (int i = 0; i < numPts; i++) { + sendFloatBlock(aliceio, objectPoints[i].x); + sendFloatBlock(aliceio, objectPoints[i].y); + sendFloatBlock(aliceio, objectPoints[i].z); + + sendFloatBlock(aliceio, imagePoints[i].x); + sendFloatBlock(aliceio, imagePoints[i].y); + } + sendFloatBlock(aliceio, f); + sendFloatBlock(aliceio, cx); + sendFloatBlock(aliceio, cy); + for (auto g : initialGuess) { + sendFloatBlock(aliceio, g); + } + aliceio->flush(); + MSG("sent alice labels\n"); + + // Offload servers run computation... + MSG("Waiting for computation to finish...\n"); + + // Collect result + for (int i = 0; i < 6; i++) { + recvFloatBlock(aliceio, bobio, &initialGuess[i]); + cout << initialGuess[i] << " "; + } + cout << endl; + + return 0; +} diff --git a/test/emp-float/eth3d_bench.cpp b/test/emp-float/eth3d_bench.cpp new file mode 100644 index 0000000..0ce3cdb --- /dev/null +++ b/test/emp-float/eth3d_bench.cpp @@ -0,0 +1,213 @@ +//#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include // dirname +#include // PATH_MAX +#include // readlink + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using std::cout; +using std::vector; + +constexpr const int port = 8080; + +static_assert(std::is_same_v, + "EMP GN localizer function sig does not match what " + "localize_wrapper expects. Tests will fail."); + +static_assert(std::is_same_v, + "EMP LM localizer function sig does not match what " + "localize_wrapper expects. Tests will fail."); + +// These tests require cleartext functions to have this signature +typedef bool (*cleartext_localizer)( + vector objectPointsSubset, + vector imagePointsSubset, float f, float cx, float xy, + float* x, /* initial guess for { r1, r2, r3, t1, t2, t3 } */ + string logprint); + +static_assert( + std::is_same_v), cleartext_localizer>, + "cleartext GN localizer function sig does not match what " + "this test expects."); + +static_assert(std::is_same_v), cleartext_localizer>, + "cleartext LM localizer function sig does not match what " + "this test expects."); + +int main(int argc, char** argv) { + if (argc != 5) { + MSG("Usage: %s \n", + argv[0]); + return 1; + } + int num_frames = atoi(argv[2]); + int num_trials = atoi(argv[3]); + int max_num_pts = atoi(argv[4]); + + std::string lm_str("lm"); + auto clear_func = lm_str == argv[1] ? lm : gaussNewton; + auto secure_func = lm_str == argv[1] ? BuildLM : BuildGaussNewton; + std::string log_str = lm_str == argv[1] ? "lm" : "gn"; + bool silent = false; + + eth3d_test_harness( + num_frames, num_trials, max_num_pts, + [&](cv::Mat rvec, cv::Mat tvec, cv::Mat cameraMatrix, cv::Mat distCoeffs, + vector objectPointsSubset, + vector imagePointsSubset, vector& res, + const vector& groundTruthRes) { + vector initialGuess(6); + for (int i = 0; i < 3; ++i) { + initialGuess[i] = rvec.at(i); + initialGuess[i + 3] = tvec.at(i); + } + float f = cameraMatrix.at(0, 0); + float cx = cameraMatrix.at(0, 2); + float cy = cameraMatrix.at(1, 2); + + bool out = clear_func(objectPointsSubset, imagePointsSubset, f, cx, cy, + &initialGuess[0], "clearlm"); + for (int i = 0; i < 6; ++i) { + res[i] = initialGuess[i]; + } + return out; + }, + [&](cv::Mat rvec, cv::Mat tvec, cv::Mat cameraMatrix, cv::Mat distCoeffs, + vector objectPointsSubset, + vector imagePointsSubset, vector& res, + const vector& groundTruthRes) { + int num_pts = objectPointsSubset.size(); + + if (!silent) { + std::cout << "testing secure\n"; + } + + std::thread bob([&]() { + NetIO* io = new NetIO("127.0.0.1", port); + setup_semi_honest(io, ALICE); + + rvec = cv::Mat::zeros(3, 1, cv::DataType::type); + tvec = cv::Mat::zeros(3, 1, cv::DataType::type); + tvec.at(2) = 1; // cleartext has bug where it fails if z=0 + auto [converged, num_loc_iterations] = emp_localize_wrapper( + ALICE, rvec, tvec, cameraMatrix, distCoeffs, objectPointsSubset, + imagePointsSubset, res, secure_func); + if (!silent) { + MSG("EMP reported %s after %d iterations.\n", + converged ? "convergence" : "failure to converge", + num_loc_iterations); + } + delete io; + }); + + NetIO* io = new NetIO(NULL, port); + setup_semi_honest(io, BOB); + rvec = cv::Mat::zeros(3, 1, cv::DataType::type); + tvec = cv::Mat::zeros(3, 1, cv::DataType::type); + tvec.at(2) = 1; // cleartext has bug where it fails if z=0 + + time_point tic = high_resolution_clock::now(); + + auto [converged, num_loc_iterations] = emp_localize_wrapper( + BOB, rvec, tvec, cameraMatrix, distCoeffs, objectPointsSubset, + imagePointsSubset, res, secure_func); + + bob.join(); + + time_point toc = high_resolution_clock::now(); + + if (!silent) { + cout << "secure result "; + for (auto const& f : res) + cout << f << ' '; + } + + bool result_matches = true; + for (int i = 0; i < 6; ++i) { + result_matches &= withinRel(res[i], groundTruthRes[i]); + } + + if (result_matches) { + if (!silent) { + cout << "emp converged!\n"; + } + + std::string timer_name = "emp_float_" + log_str + "_time_vs_points"; +#if PPL_FLOW == PPL_FLOW_DO + timer_name += "_dataobl"; +#endif + MSG("SeNtInAl,xy,%s,%s,%d,%g\n", __FUNCTION__, timer_name.c_str(), + num_pts, + std::chrono::duration_cast(toc - tic) + .count() / + 1000000.0); + MSG("SeNtInAl,xy,%s,%s,%d,%g\n", __FUNCTION__, + (timer_name + "_per_loc_itr").c_str(), num_pts, + std::chrono::duration_cast(toc - tic) + .count() / + (1000000.0 * num_loc_iterations)); + + MSG("SeNtInAl,grouped_bar,%s,%s%s,%d,%d\n", __FUNCTION__, + log_str.c_str(), "_additions", num_pts, num_additions); + MSG("SeNtInAl,grouped_bar,%s,%s%s,%d,%d\n", __FUNCTION__, + log_str.c_str(), "_subtractions", num_pts, num_subtractions); + MSG("SeNtInAl,grouped_bar,%s,%s%s,%d,%d\n", __FUNCTION__, + log_str.c_str(), "_multiplications", num_pts, + num_multiplications); + MSG("SeNtInAl,grouped_bar,%s,%s%s,%d,%d\n", __FUNCTION__, + log_str.c_str(), "_divisions", num_pts, num_divisions); + + MSG("SeNtInAl,xy,%s,%s%s,%d,%lu\n", __FUNCTION__, log_str.c_str(), + "_bytes_tx", num_pts, io->size_tx); + MSG("SeNtInAl,xy,%s,%s%s,%d,%lu\n", __FUNCTION__, log_str.c_str(), + "_bytes_tx_per_itr", num_pts, io->size_tx / num_loc_iterations); + MSG("SeNtInAl,xy,%s,%s%s,%d,%lu\n", __FUNCTION__, log_str.c_str(), + "_bytes_tx_per_itr_per_feat", num_pts, + io->size_tx / num_loc_iterations / num_pts); + + MSG("SeNtInAl,xy,%s,%s%s,%d,%lu\n", __FUNCTION__, log_str.c_str(), + "_bytes_rx", num_pts, io->size_rx); + MSG("SeNtInAl,xy,%s,%s%s,%d,%lu\n", __FUNCTION__, log_str.c_str(), + "_bytes_rx_per_itr", num_pts, io->size_rx / num_loc_iterations); + MSG("SeNtInAl,xy,%s,%s%s,%d,%lu\n", __FUNCTION__, log_str.c_str(), + "_bytes_rx_per_itr_per_feat", num_pts, + io->size_rx / num_loc_iterations / num_pts); + + } else { + if (!silent) { + MSG("Not printing timing - not converged.\n"); + } + } + + // reset stats + num_additions = 0; + num_subtractions = 0; + num_multiplications = 0; + num_divisions = 0; + delete io; + + return result_matches; + }, + silent); +} diff --git a/test/emp-float/localize_wrapper.hpp b/test/emp-float/localize_wrapper.hpp new file mode 100644 index 0000000..8eb9dc1 --- /dev/null +++ b/test/emp-float/localize_wrapper.hpp @@ -0,0 +1,66 @@ +#include +#include +#include +#include +#include + +#include "emp-sh2pc/emp-sh2pc.h" + +using std::vector; + +typedef std::pair (*emp_localizer)(Float threeDPts[], Float y0[], + int numPts, Float f, Float cx, + Float cy, Float x[]); + +std::pair emp_localize_wrapper(int party, cv::Mat rvec, cv::Mat tvec, + cv::Mat cameraMatrix, + cv::Mat distCoeffs, + vector objectPoints, + vector imagePoints, + vector& res, + emp_localizer secure_localize_func) { + + float f = cameraMatrix.at(0, 0); + float cx = cameraMatrix.at(0, 2); + float cy = cameraMatrix.at(1, 2); + + int numPts = objectPoints.size(); + Float* sobjectPoints = + static_cast(operator new[](4 * numPts * sizeof(Float))); + Float* simagePoints = + static_cast(operator new[](3 * numPts * sizeof(Float))); + for (int i = 0; i < numPts; i++) { + // [x1, x2... ; y1, y2... ; z1, z2...] + sobjectPoints[i] = Float(objectPoints[i].x, ALICE); + sobjectPoints[numPts + i] = Float(objectPoints[i].y, ALICE); + sobjectPoints[2 * numPts + i] = Float(objectPoints[i].z, ALICE); + // sobjectPoints[3*numPts + i] = Float(1.0, PUBLIC); + + // [x1, y1; x2, y2; ...] + simagePoints[2 * i] = Float(imagePoints[i].x, ALICE); + simagePoints[2 * i + 1] = Float(imagePoints[i].y, ALICE); + // simagePoints[2*numPts + i] = Float(1.0, PUBLIC); + } + + Float sf = Float(f, ALICE); + Float scx = Float(cx, ALICE); + Float scy = Float(cy, ALICE); + + Float* sx = static_cast(operator new[](6 * sizeof(Float))); + for (int i = 0; i < 3; i++) { + sx[i] = Float(rvec.at(i), ALICE); + sx[i + 3] = Float(tvec.at(i), ALICE); + } + + auto sres = secure_localize_func(sobjectPoints, simagePoints, numPts, sf, scx, + scy, sx); + + for (int i = 0; i < 6; ++i) { + res[i] = sx[i].reveal(); + } + + delete[] sx; + delete[] sobjectPoints; + delete[] simagePoints; + return sres; +} \ No newline at end of file diff --git a/test/emp-float/test.cpp b/test/emp-float/test.cpp new file mode 100644 index 0000000..72d9133 --- /dev/null +++ b/test/emp-float/test.cpp @@ -0,0 +1,359 @@ +//#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "emp-sh2pc/emp-sh2pc.h" + +#include + +using namespace emp; +using Catch::Matchers::WithinAbs; +using std::cout; +using std::vector; + +constexpr const int port = 8080; + +TEST_CASE("EMP trig functions are computed", "[emp_trig]") { + float angle = .2; + + std::thread bob([angle]() { + NetIO* io = new NetIO("127.0.0.1", port); + emp_trig(io, BOB, angle); + delete io; + }); + + NetIO* io = new NetIO(nullptr, port); + emp_trig(io, ALICE, angle); + bob.join(); + delete io; +} + +TEST_CASE("EMP matrix multiply is computed", "[emp_mat]") { + cv::Mat A = cv::Mat::ones(3, 5, cv::DataType::type); + cv::Mat B = cv::Mat::ones(5, 4, cv::DataType::type) * 2; + + std::thread bob([A, B]() { + NetIO* io = new NetIO("127.0.0.1", port); + emp_mat(io, BOB, A, B); + delete io; + }); + + NetIO* io = new NetIO(nullptr, port); + emp_mat(io, ALICE, A, B); + bob.join(); + delete io; +} + +TEST_CASE("EMP rodriguez transformation is computed", "[emp_rod]") { + float r[3] = {.2, .4, .2}; // 3x1 + + std::thread bob([&r]() { + NetIO* io = new NetIO("127.0.0.1", port); + emp_rod(io, BOB, r); + delete io; + }); + + NetIO* io = new NetIO(nullptr, port); + emp_rod(io, ALICE, r); + bob.join(); + delete io; +} + +TEST_CASE("EMP point projection is computed", "[emp_proj]") { + float f = 715; + float cx = 354; + float cy = 245; + float _cM[] = {f, 0, cx, 0, f, cy, 0, 0, 1}; + cv::Mat cameraMatrix = cv::Mat(3, 3, cv::DataType::type, _cM); + cv::Mat distCoeffs = cv::Mat::zeros(4, 1, cv::DataType::type); + cv::Mat rvec(3, 1, cv::DataType::type); + cv::Mat tvec(3, 1, cv::DataType::type); + vector imagePoints; + vector objectPoints; + Hoffs2DPoints(imagePoints); + Hoffs3DPoints(objectPoints); + + std::thread bob([&]() { + NetIO* io = new NetIO("127.0.0.1", port); + emp_proj(io, BOB, cameraMatrix, distCoeffs, objectPoints, imagePoints); + delete io; + }); + + NetIO* io = new NetIO(nullptr, port); + emp_proj(io, ALICE, cameraMatrix, distCoeffs, objectPoints, imagePoints); + bob.join(); + delete io; +} + +TEST_CASE("EMP SVD sign function is computed", "[emp_svd]") { + float a = 2.40, b = 0; + + std::thread bob([a, b]() { + NetIO* io = new NetIO("127.0.0.1", port); + emp_svd_sign(io, BOB, a, b); + delete io; + }); + + NetIO* io = new NetIO(nullptr, port); + emp_svd_sign(io, ALICE, a, b); + bob.join(); + delete io; +} + +TEST_CASE("EMP SVD pythag function is computed", "[emp_svd]") { + float a = -2.40, b = 96.0; + + std::thread bob([a, b]() { + NetIO* io = new NetIO("127.0.0.1", port); + emp_svd_pythag(io, BOB, a, b); + delete io; + }); + + NetIO* io = new NetIO(nullptr, port); + emp_svd_pythag(io, ALICE, a, b); + bob.join(); + delete io; +} + +// TEST_CASE("Small SVD function is computed", "[svd]") { +// int m=5,n=4; +// float** in = new float*[m]; +// for(int i=0; i::type, _cM); + cv::Mat distCoeffs = cv::Mat::zeros(4, 1, cv::DataType::type); + cv::Mat rvec = cv::Mat::zeros(3, 1, cv::DataType::type); + cv::Mat tvec = cv::Mat::zeros(3, 1, cv::DataType::type); + vector imagePoints; + vector objectPoints; + Hoffs2DPoints(imagePoints); + Hoffs3DPoints(objectPoints); + + std::thread bob([&]() { + NetIO* io = new NetIO("127.0.0.1", port); + emp_localize(io, BOB, rvec, tvec, cameraMatrix, distCoeffs, objectPoints, + imagePoints, gaussNewton, BuildGaussNewton); + delete io; + }); + + NetIO* io = new NetIO(nullptr, port); + emp_localize(io, ALICE, rvec, tvec, cameraMatrix, distCoeffs, objectPoints, + imagePoints, gaussNewton, BuildGaussNewton); + bob.join(); + delete io; +} +//#endif + +TEST_CASE("EMP Levenburg Marquardt pose estimation is computed on Hoff points", + "[emp_lm]") { + float f = 715; + float cx = 354; + float cy = 245; + float _cM[] = {f, 0, cx, 0, f, cy, 0, 0, 1}; + cv::Mat cameraMatrix = cv::Mat(3, 3, cv::DataType::type, _cM); + cv::Mat distCoeffs = cv::Mat::zeros(4, 1, cv::DataType::type); + cv::Mat rvec = cv::Mat::zeros(3, 1, cv::DataType::type); + cv::Mat tvec = cv::Mat::zeros(3, 1, cv::DataType::type); + vector imagePoints; + vector objectPoints; + Hoffs2DPoints(imagePoints); + Hoffs3DPoints(objectPoints); + + std::thread bob([&]() { + NetIO* io = new NetIO("127.0.0.1", port); + emp_localize(io, BOB, rvec, tvec, cameraMatrix, distCoeffs, objectPoints, + imagePoints, lm, BuildLM); + delete io; + }); + + NetIO* io = new NetIO(nullptr, port); + emp_localize(io, ALICE, rvec, tvec, cameraMatrix, distCoeffs, objectPoints, + imagePoints, lm, BuildLM); + bob.join(); + delete io; +} + +TEST_CASE( + "EMP Levenburg Marquardt pose estimation is computed on April tag points", + "[emp_lm_april]") { + float f = 715; + float cx = 354; + float cy = 245; + float _cM[] = {f, 0, cx, 0, f, cy, 0, 0, 1}; + cv::Mat cameraMatrix = cv::Mat(3, 3, cv::DataType::type, _cM); + cv::Mat distCoeffs = cv::Mat::zeros(4, 1, cv::DataType::type); + cv::Mat rvec = cv::Mat::zeros(3, 1, cv::DataType::type); + cv::Mat tvec = cv::Mat::zeros(3, 1, cv::DataType::type); + tvec.at(2) = 1.0f; // cleartext has bug where it fails if z=0 + vector imagePoints; + vector objectPoints; + AprilSnail3DPoints(objectPoints); + AprilSnail2DPoints(imagePoints); + + std::thread bob([&]() { + NetIO* io = new NetIO("127.0.0.1", port); + emp_localize(io, BOB, rvec, tvec, cameraMatrix, distCoeffs, objectPoints, + imagePoints, lm, BuildLM); + delete io; + }); + + NetIO* io = new NetIO(nullptr, port); + emp_localize(io, ALICE, rvec, tvec, cameraMatrix, distCoeffs, objectPoints, + imagePoints, lm, BuildLM); + bob.join(); + delete io; +} diff --git a/test/emp-float/test_utils.hpp b/test/emp-float/test_utils.hpp new file mode 100644 index 0000000..11b0da2 --- /dev/null +++ b/test/emp-float/test_utils.hpp @@ -0,0 +1,474 @@ +//#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "emp-sh2pc/emp-sh2pc.h" + +using namespace emp; +using Catch::Matchers::WithinAbs; +using Catch::Matchers::WithinRel; +using std::cout; +using std::vector; + +void emp_trig(NetIO* io, int party, float angle) { + setup_semi_honest(io, party); + Float v = Float(angle, ALICE); + Float sinres = BuildSinCircuit(v); + REQUIRE_THAT(sinres.reveal(), + WithinAbs(sin(angle), float_test_epsilon)); + + Float cosres = BuildCosCircuit(v); + REQUIRE_THAT(cosres.reveal(), + WithinAbs(cos(angle), float_test_epsilon)); + finalize_semi_honest(); +} + +void emp_mat(NetIO* io, int party, cv::Mat A, cv::Mat B) { + setup_semi_honest(io, party); + cv::Mat clear_res = cv::Mat::zeros(A.rows, B.cols, cv::DataType::type); + + matmult(A.ptr(), A.rows, A.cols, B.ptr(), B.rows, B.cols, + clear_res.ptr()); + + void* raw_memoryA = operator new[](A.rows* A.cols * sizeof(Float)); + void* raw_memoryB = operator new[](B.rows* B.cols * sizeof(Float)); + void* raw_memoryres = operator new[](A.rows* B.cols * sizeof(Float)); + Float* eA = static_cast(raw_memoryA); + Float* eB = static_cast(raw_memoryB); + Float* eres = static_cast(raw_memoryres); + + for (int i = 0; i < A.rows * A.cols; ++i) { + new (&eA[i]) Float(A.ptr()[i], ALICE); + } + + for (int i = 0; i < B.rows * B.cols; ++i) { + new (&eB[i]) Float(B.ptr()[i], ALICE); + } + + BuildMatmultCircuit(eA, A.rows, A.cols, eB, B.rows, B.cols, eres); + + for (int i = 0; i < A.rows * B.cols; ++i) { + REQUIRE_THAT(eres[i].reveal(), + WithinAbs(clear_res.ptr()[i], float_test_epsilon)); + } + + finalize_semi_honest(); + delete[] eA; + delete[] eB; + delete[] eres; +} + +void emp_rod(NetIO* io, int party, float* r) { + setup_semi_honest(io, party); + + cv::Mat R = cv::Mat::zeros(3, 4, cv::DataType::type); + rodrigues(r, R.ptr()); + + void* raw_memorysr = operator new[](3 * sizeof(Float)); + Float* sr = static_cast(raw_memorysr); + new (&sr[0]) Float(r[0], ALICE); + new (&sr[1]) Float(r[1], ALICE); + new (&sr[2]) Float(r[2], ALICE); + + void* raw_memoryR = operator new[](12 * sizeof(Float)); + Float* sR = static_cast(raw_memoryR); + + BuildRodriguesCircuit(sr, sR); + + cout << "rodrigues result:" << endl; + for (int i = 0; i < 12; ++i) { + if (i == 3 || i == 7 || i == 11) { + continue; + } + REQUIRE_THAT(sR[i].reveal(), + WithinAbs(R.ptr()[i], float_test_epsilon)); + } + delete[] sr; + delete[] sR; + + finalize_semi_honest(); +} + +// From points, first estimate the pose using opencv. +// Then reproject into 3d using the estimated pose +// and compare to the origional points. +void emp_proj(NetIO* io, int party, cv::Mat cameraMatrix, cv::Mat distCoeffs, + vector objectPoints, + vector imagePoints) { + cv::Mat rvec = cv::Mat::zeros(3, 1, cv::DataType::type); + cv::Mat tvec = cv::Mat::zeros(3, 1, cv::DataType::type); + + // Find rotation and translation + cv::solvePnP(objectPoints, imagePoints, cameraMatrix, distCoeffs, rvec, tvec, + false, cv::SOLVEPNP_ITERATIVE); + + int nOp = objectPoints.size(); + cv::Mat x = cv::Mat::zeros(6, 1, cv::DataType::type); + float* xptr = x.ptr(); + xptr[0] = rvec.at(0); + xptr[1] = rvec.at(1); + xptr[2] = rvec.at(2); + xptr[3] = tvec.at(0); + xptr[4] = tvec.at(1); + xptr[5] = tvec.at(2); + + cv::Mat matOP = cv::Mat(objectPoints, false); + matOP = matOP.reshape(1, matOP.total()); + matOP.convertTo(matOP, cv::DataType::type); + cv::Mat P = cv::Mat::ones(4, nOp, cv::DataType::type); + P(cv::Range(0, 3), cv::Range(0, nOp)) = + matOP.t(); // copy points into new matrix + + // cleartext projection + cv::Mat projected = cv::Mat(3, nOp, cv::DataType::type); + projectPoints(P.ptr(), x.ptr(), cameraMatrix.ptr(), + projected.ptr(), nOp); + for (int i = 0; i < nOp; ++i) { + REQUIRE_THAT(imagePoints[i].x, + WithinAbs(projected.at(0, i), reprojection_tol)); + REQUIRE_THAT(imagePoints[i].y, + WithinAbs(projected.at(1, i), reprojection_tol)); + } + + // mpc projection + setup_semi_honest(io, party); + void* raw_memoryP = operator new[](4 * nOp * sizeof(Float)); + void* raw_memoryx = operator new[](6 * sizeof(Float)); + void* raw_memoryK = operator new[](9 * sizeof(Float)); + void* raw_memoryres = operator new[](3 * nOp * sizeof(Float)); + Float* sP = static_cast(raw_memoryP); + Float* sx = static_cast(raw_memoryx); + Float* sK = static_cast(raw_memoryK); + Float* sres = static_cast(raw_memoryres); + + for (int i = 0; i < 3 * nOp; i++) { + new (&sP[i]) Float(P.ptr()[i], ALICE); + } + for (int i = 3 * nOp; i < 4 * nOp; i++) { // need contant 1s + new (&sP[i]) Float(1.0, PUBLIC); + } + for (int i = 0; i < 6; i++) { + new (&sx[i]) Float(x.ptr()[i], ALICE); + } + for (int i = 0; i < 9; i++) { + new (&sK[i]) Float(cameraMatrix.ptr()[i], ALICE); + } + + BuildProjectPointsCircuit(sP, sx, sK, sres, nOp, true); + + for (int i = 0; i < 2 * nOp; ++i) { + REQUIRE_THAT(projected.ptr()[i], + WithinAbs(sres[i].reveal(), reprojection_tol)); + } + delete[] sP; + delete[] sx; + delete[] sK; + delete[] sres; + finalize_semi_honest(); +} + +void emp_svd_sign(NetIO* io, int party, float a, float b) { + setup_semi_honest(io, party); + float cleartext_res = mysign(a, b); + Float sa = Float(a, ALICE); + Float sb = Float(b, ALICE); + BuildSignCircuit(&sa, &sb); + REQUIRE_THAT(sa.reveal(), + WithinAbs(cleartext_res, float_test_epsilon)); + finalize_semi_honest(); +} + +void emp_svd_pythag(NetIO* io, int party, float a, float b) { + setup_semi_honest(io, party); + float cleartext_res = mypythag(a, b); + Float sa = Float(a, ALICE); + Float sb = Float(b, ALICE); + Float out = BuildPythagCircuit(&sa, &sb); + REQUIRE_THAT(out.reveal(), + WithinAbs(cleartext_res, float_test_epsilon)); + finalize_semi_honest(); +} + +void emp_svd(NetIO* io, int party, float** in, int m, int n) { + // linearize for cv + float* cvin = new float[m * n]; + for (int i = 0; i < m; i++) { + for (int j = 0; j < n; j++) { + cvin[i * n + j] = in[i][j]; + } + } + cv::Mat cvinM = cv::Mat(m, n, cv::DataType::type, in); + cv::SVD cvsvd(cvinM, cv::SVD::FULL_UV); // constructor + if (party == ALICE) { + cout << "opencv result\n" + << cvsvd.u << '\n' + << cvsvd.w << '\n' + << cvsvd.vt << '\n'; + } + delete[] cvin; + + // svd overwrites with u matrix, so copy + float** a = new float*[m]; + for (int i = 0; i < m; i++) { + a[i] = new float[n]; + for (int j = 0; j < n; j++) { + a[i][j] = in[i][j]; + } + } + float* w = new float[n](); // aka sigma, only diag + float** v = new float*[n]; // nxn + for (int i = 0; i < n; i++) + v[i] = new float[n](); + + setup_semi_honest(io, party); + Float** sa = new Float*[m]; + for (int i = 0; i < m; i++) { + sa[i] = static_cast(operator new[](n * sizeof(Float))); + for (int j = 0; j < n; j++) { + new (&sa[i][j]) Float(in[i][j], ALICE); + } + } + Float* sw = static_cast(operator new[](n * sizeof(Float))); + for (int i = 0; i < n; i++) { + sw[i] = Float(0.0, PUBLIC); + } + Float** sv = new Float*[n]; + for (int i = 0; i < n; i++) { + sv[i] = static_cast(operator new[](n * sizeof(Float))); + for (int j = 0; j < n; j++) { + sv[i][j] = Float(0.0, PUBLIC); + } + } + + svdcmp(a, m, n, w, v); + if (party == ALICE) { + cout << "cleartext" << '\n'; + printMatrix("a", a, m, n); + printVector("w", w, n); + printMatrix("v", v, n, n); + } + + BuildSvdCircuit(sa, m, n, sw, sv); + // both parties must print if uncommented + cout << "secure" << '\n'; + printFloatMatrix(sa, m, n, party == BOB); + printFloatVector(sw, n, party == BOB); + printFloatMatrix(sv, n, n, party == BOB); + + for (int i = 0; i < m; i++) { + for (int j = 0; j < n; j++) { + // svd is not unique, do not compare with opencv + // REQUIRE_THAT(a[i][j], WithinAbs(cvsvd.u.at(i,j), svd_tol)); + REQUIRE_THAT(sa[i][j].reveal(), WithinAbs(a[i][j], svd_tol)); + } + delete[] a[i]; + delete[] sa[i]; + } + delete[] a; + delete[] sa; + + for (int i = 0; i < n; i++) { + // svd is not unique, do not compare with opencv + // REQUIRE_THAT(w[i], WithinAbs(cvsvd.w.at(i), svd_tol)); + REQUIRE_THAT(sw[i].reveal(), WithinAbs(w[i], svd_tol)); + } + delete[] w; + delete[] sw; + + for (int i = 0; i < n; i++) { + for (int j = 0; j < n; j++) { + // svd is not unique, do not compare with opencv + // REQUIRE_THAT(v[i][j], WithinAbs(cvsvd.vt.at(i,j), svd_tol)); + REQUIRE_THAT(sv[i][j].reveal(), WithinAbs(v[i][j], svd_tol)); + } + delete[] v[i]; + delete[] sv[i]; + } + delete[] v; + delete[] sv; + + finalize_semi_honest(); +} + +void emp_invert(NetIO* io, int party, float* in, int m, int n) { + setup_semi_honest(io, party); + + // opencv computation + cv::Mat cvM = cv::Mat(m, n, cv::DataType::type, in); + cv::Mat cvres; + invert(cvM, cvres, cv::DECOMP_SVD); + // cout << "opencv result:\n" << ores << endl; + + // convert to 2d matrix + float** M = new float*[m]; + for (int i = 0; i < m; i++) { + M[i] = new float[n]; + for (int j = 0; j < n; j++) { + M[i][j] = in[i * n + j]; + } + } + + // result is nxm (not mxn) + float** res = new float*[n]; + for (int i = 0; i < n; i++) { + res[i] = new float[m]; + } + + // share arrays + Float** a = new Float*[m]; + for (int i = 0; i < m; i++) { + a[i] = static_cast(operator new[](n * sizeof(Float))); + for (int j = 0; j < n; j++) { + new (&a[i][j]) Float(M[i][j], ALICE); + } + } + + Float** sres = new Float*[n]; + for (int i = 0; i < n; i++) { + sres[i] = static_cast(operator new[](m * sizeof(Float))); + // for(int j=0; j(i, j), + WithinAbs(res[i][j], float_test_epsilon)); + } + } + + BuildInvertCircuit(a, m, n, sres); + + // check privacy preserving vs cleartext + for (int i = 0; i < n; i++) { + for (int j = 0; j < m; j++) { + REQUIRE_THAT(sres[i][j].reveal(), + WithinAbs(res[i][j], float_test_epsilon)); + } + delete[] res[i]; + delete[] sres[i]; + } + delete[] res; + delete[] sres; + + for (int i = 0; i < m; i++) { + delete[] M[i]; + delete[] a[i]; + } + delete[] M; + delete[] a; + + finalize_semi_honest(); +} + +void emp_twonorm(NetIO* io, int party, float* vec, int sz) { + setup_semi_honest(io, party); + Float* svec = static_cast(operator new[](sz * sizeof(Float))); + for (int i = 0; i < sz; i++) { + svec[i] = Float(vec[i], PUBLIC); + } + + Float sres = BuildTwoNormSqCircuit(svec, sz); + + REQUIRE_THAT(sres.reveal(), + WithinAbs(twonormsq(vec, sz), float_test_epsilon)); + delete[] svec; + finalize_semi_honest(); +} + +void emp_localize(NetIO* io, int party, cv::Mat rvec, cv::Mat tvec, + cv::Mat cameraMatrix, cv::Mat distCoeffs, + vector objectPoints, + vector imagePoints, auto cleartext_localize_func, + auto secure_localize_func) { + setup_semi_honest(io, party); + + cv::Mat cvrvec = rvec.clone(); + cv::Mat cvtvec = tvec.clone(); + cv::solvePnP(objectPoints, imagePoints, cameraMatrix, distCoeffs, cvrvec, + cvtvec, false, cv::SOLVEPNP_ITERATIVE); + if (party == ALICE) { + cout << "opencv\n" << cvrvec.t() << cvtvec.t() << '\n'; + } + + float rt[6] = {rvec.at(0), rvec.at(1), rvec.at(2), + tvec.at(0), tvec.at(1), tvec.at(2)}; + float f = cameraMatrix.at(0, 0); + float cx = cameraMatrix.at(0, 2); + float cy = cameraMatrix.at(1, 2); + cleartext_localize_func(objectPoints, imagePoints, f, cx, cy, rt, ""); + if (party == ALICE) { + cout << "cleartext\n"; + printVector("rt", rt, 6); + } + // check cleartext vs opencv + for (int i = 0; i < 3; i++) { + REQUIRE_THAT(cvrvec.at(i), + WithinRel(rt[i], localization_tol_rel) || + WithinAbs(rt[i], localization_tol_abs)); + REQUIRE_THAT(cvtvec.at(i), + WithinRel(rt[i + 3], localization_tol_rel) || + WithinAbs(rt[i + 3], localization_tol_abs)); + } + + vector res(6); + emp_localize_wrapper(party, rvec, tvec, cameraMatrix, distCoeffs, + objectPoints, imagePoints, res, secure_localize_func); + + // check opencv vs secure + for (int i = 0; i < 6; i++) { + if (party == BOB) { + cout << "secure result: "; + for (auto const& f : res) + cout << f << ' '; + cout << '\n'; + } + REQUIRE_THAT(res[i], WithinRel(rt[i], localization_tol_rel) || + WithinAbs(rt[i], localization_tol_abs)); + } + + finalize_semi_honest(); +}