From cf598ab9d422d1488ffdb8d771f3ec730f1056fc Mon Sep 17 00:00:00 2001 From: Jamie Date: Mon, 30 Sep 2024 16:59:33 +0800 Subject: [PATCH] repo-sync-2024-09-30T16:00:47+0800 (#395) * repo-sync-2024-09-30T16:00:47+0800 * Update .licenserc.yaml * Update BUILD.bazel --- GETTING_STARTED.md | 20 ++--- yacl/base/BUILD.bazel | 2 + yacl/base/int128.h | 11 +++ yacl/base/int128_test.cc | 18 +++- yacl/crypto/ecc/lib25519/BUILD.bazel | 10 ++- yacl/crypto/rand/rand.cc | 7 ++ yacl/crypto/rand/rand.h | 41 +++++++++ yacl/crypto/rand/rand_test.cc | 36 ++++++++ yacl/crypto/tools/BUILD.bazel | 1 + yacl/crypto/tools/prg.h | 43 +++++++++ yacl/crypto/tools/prg_test.cc | 35 ++++++++ yacl/math/galois_field/benchmark/BUILD.bazel | 24 +++++ .../galois_field/benchmark/operation_bench.cc | 88 +++++++++++++++++++ yacl/math/galois_field/factory/BUILD.bazel | 3 + yacl/math/mpint/benchmark/BUILD.bazel | 9 ++ yacl/math/mpint/benchmark/mod_bench.cc | 80 +++++++++++++++++ 16 files changed, 416 insertions(+), 12 deletions(-) create mode 100644 yacl/math/galois_field/benchmark/BUILD.bazel create mode 100644 yacl/math/galois_field/benchmark/operation_bench.cc create mode 100644 yacl/math/mpint/benchmark/mod_bench.cc diff --git a/GETTING_STARTED.md b/GETTING_STARTED.md index 63c6275..8413c79 100644 --- a/GETTING_STARTED.md +++ b/GETTING_STARTED.md @@ -49,16 +49,16 @@ $ sudo mv bazel /usr/local/bin # you need sudo to do this To build Yacl, at yacl's root directory, run the following ```sh -$ bazel build //... -$ bazel build //... -c opt # build as optimized mode -$ bazel build //... -c dbg # build as debug mode -$ bazel build //... --config gm # build with gm mode +$ bazel build //yacl/... +$ bazel build //yacl/... -c opt # build as optimized mode +$ bazel build //yacl/... -c dbg # build as debug mode +$ bazel build //yacl/... --config gm # build with gm mode ``` To test Yacl ```sh -$ bazel test //... +$ bazel test //yacl/... ``` ### MacOS @@ -83,16 +83,16 @@ $ brew install bazelisk cmake ninja nasm automake libtool libomp To build Yacl, at yacl's root directory, run the following ```sh -$ bazel build //... -$ bazel build //... -c opt # build as optimized mode -$ bazel build //... -c dbg # build as debug mode -$ bazel build //... --config gm # build with gm mode +$ bazel build //yacl/... +$ bazel build //yacl/... -c opt # build as optimized mode +$ bazel build //yacl/... -c dbg # build as debug mode +$ bazel build //yacl/... --config gm # build with gm mode ``` To test Yacl ```sh -$ bazel test //... +$ bazel test //yacl/... ``` ## Setup compilation database for your lsp diff --git a/yacl/base/BUILD.bazel b/yacl/base/BUILD.bazel index 163ad11..a4afa04 100644 --- a/yacl/base/BUILD.bazel +++ b/yacl/base/BUILD.bazel @@ -41,6 +41,7 @@ yacl_cc_library( srcs = ["int128.cc"], hdrs = ["int128.h"], deps = [ + "@com_google_absl//absl/numeric:bits", "@com_google_absl//absl/numeric:int128", ], ) @@ -50,6 +51,7 @@ yacl_cc_test( srcs = ["int128_test.cc"], deps = [ ":int128", + "//yacl/crypto/rand", ], ) diff --git a/yacl/base/int128.h b/yacl/base/int128.h index 175fdc5..3e69d69 100644 --- a/yacl/base/int128.h +++ b/yacl/base/int128.h @@ -18,6 +18,8 @@ #include #include +#include "absl/numeric/bits.h" + // NOTE: // We add our own int128 due to: // - absl::int128 forget to support fully `constexpr`, i.e. `operator>>`. Giving @@ -75,6 +77,15 @@ std::pair DecomposeInt128(int128_t v); std::pair DecomposeUInt128(uint128_t v); +inline int CountLZ(uint128_t v) { + auto [hi, lo] = DecomposeUInt128(v); + return hi == 0 ? absl::countl_zero(lo) + 64 : absl::countl_zero(hi); +} + +inline int CountBitWidth(uint128_t v) { + return std::numeric_limits::digits - CountLZ(v); +} + } // namespace yacl #if !defined(__GNUC__) diff --git a/yacl/base/int128_test.cc b/yacl/base/int128_test.cc index 9bd403d..240ffbc 100644 --- a/yacl/base/int128_test.cc +++ b/yacl/base/int128_test.cc @@ -16,6 +16,8 @@ #include "gtest/gtest.h" +#include "yacl/crypto/rand/rand.h" + TEST(Int128Test, NumericLimitsTest) { #ifdef __clang__ #pragma clang diagnostic push @@ -76,4 +78,18 @@ TEST(Int128Test, Decompose) { } } -TEST(Int128Test, RandomTest) {} +TEST(Int128Test, CountLzTest) { + uint128_t x = 0; + EXPECT_EQ(yacl::CountLZ(x), 128); + + x = 1; + EXPECT_EQ(yacl::CountLZ(x), 127); + + x = yacl::crypto::FastRandU128(); + EXPECT_EQ(yacl::CountLZ(x), yacl::CountLZ(x)); + + x = std::numeric_limits::max(); + int offset = yacl::crypto::RandLtN(128); + x >>= offset; + EXPECT_EQ(yacl::CountLZ(x), offset); +} diff --git a/yacl/crypto/ecc/lib25519/BUILD.bazel b/yacl/crypto/ecc/lib25519/BUILD.bazel index 03d08e7..470c6c4 100644 --- a/yacl/crypto/ecc/lib25519/BUILD.bazel +++ b/yacl/crypto/ecc/lib25519/BUILD.bazel @@ -16,10 +16,18 @@ load("//bazel:yacl.bzl", "yacl_cc_library", "yacl_cc_test") package(default_visibility = ["//visibility:public"]) +config_setting( + name = "linux_x64", + constraint_values = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], +) + yacl_cc_library( name = "lib25519", deps = select({ - "@platforms//cpu:x86_64": [ + ":linux_x64": [ ":lib25519_factory", ], "//conditions:default": [], diff --git a/yacl/crypto/rand/rand.cc b/yacl/crypto/rand/rand.cc index 1f2bb22..ec93d1e 100644 --- a/yacl/crypto/rand/rand.cc +++ b/yacl/crypto/rand/rand.cc @@ -67,6 +67,13 @@ void FillRand(char *buf, size_t len, bool fast_mode) { // --------------------- // Other Implementations // --------------------- +// +uint32_t RandU32(bool fast_mode) { + uint32_t rand32 = 0; + FillRand(reinterpret_cast(&rand32), sizeof(uint32_t), fast_mode); + return rand32; +} + uint64_t RandU64(bool fast_mode) { uint64_t rand64 = 0; FillRand(reinterpret_cast(&rand64), sizeof(uint64_t), fast_mode); diff --git a/yacl/crypto/rand/rand.h b/yacl/crypto/rand/rand.h index fcfe1fb..f5948b5 100644 --- a/yacl/crypto/rand/rand.h +++ b/yacl/crypto/rand/rand.h @@ -52,6 +52,16 @@ void FillRand(char *buf, size_t len, bool fast_mode = false); // Random Support for Generic Types // -------------------------------- +// Generate uint32_t random value +uint32_t RandU32(bool fast_mode = false); + +// Generate uint32_t random value, in a faster but less secure way +inline uint32_t FastRandU32() { return RandU32(true); } + +// Generate uint32_t random value, in a slower but more secure way +// (randomness comes directly from an random entropy source) +inline uint32_t SecureRandU32() { return RandU32(false); } + // Generate uint64_t random value uint64_t RandU64(bool fast_mode = false); @@ -225,4 +235,35 @@ class YaclReplayUrbg { const CType ctype_; }; +template +void ReplayShuffle(RandomIt first, RandomIt last, uint128_t seed, + uint64_t *ctr) { + YACL_ENFORCE(ctr != nullptr); + + using diff_t = typename std::iterator_traits::difference_type; + diff_t n = last - first; + + // prepare n-1 random numbers + // ind[0] in [0, 1], ind[1] in [0, 2] ... ind[n-2] in [0, n-1] + std::vector ind(n - 1); + + *ctr = yacl::crypto::FillPRand( + yacl::crypto::SymmetricCrypto::CryptoType::AES128_CTR, seed, 0, *ctr, + (char *)ind.data(), (n - 1) * sizeof(uint128_t)); + + // Though this is not strictly uniform random. it will + // provide statistical security of no less than 40 bits. + // i.e. for some fixed k, the statistical distance between our random + // variables and the ground truth uniform distribution over [0, k-1] is no + // more that 1/2 * (k / 2^{128}) < 2^{-64} (if we assume k < 2^64). + for (int64_t idx = 0; idx < n - 1; ++idx) { + ind[idx] = ind[idx] % (idx + 2); + } + + // Knuth-Durstenfeld Shuffle + for (diff_t i = n - 1; i > 0; --i) { // + std::swap(first[i], first[ind[i - 1]]); + } +} + } // namespace yacl::crypto diff --git a/yacl/crypto/rand/rand_test.cc b/yacl/crypto/rand/rand_test.cc index 3003af1..5f33cc8 100644 --- a/yacl/crypto/rand/rand_test.cc +++ b/yacl/crypto/rand/rand_test.cc @@ -186,4 +186,40 @@ TEST(GenericRandTest, ReplayRandomShuffleTest) { EXPECT_EQ(std::memcmp(vec1.data(), vec4.data(), sizeof(uint128_t) * n), 0); } } + +TEST(GenericRandTest, QuickShuffleTest) { + int n = 257; + + auto vec = FastRandVec(n); + auto ctr = FastRandU64(); + auto seed = SecureRandSeed(); + + // copy + auto vec_bkup = vec; + // copy + auto vec_1 = vec; + auto ctr_1 = ctr; + // copy + auto vec_2 = vec; + auto ctr_2 = FastRandU64(); + + ReplayShuffle(vec.begin(), vec.end(), seed, &ctr); + + // replay + ReplayShuffle(vec_1.begin(), vec_1.end(), seed, &ctr_1); + EXPECT_EQ(std::memcmp(vec.data(), vec_1.data(), sizeof(uint128_t) * n), 0); + + // different state gives different shuffle + ReplayShuffle(vec_2.begin(), vec_2.end(), SecureRandSeed(), &ctr_2); + EXPECT_NE(std::memcmp(vec.data(), vec_2.data(), sizeof(uint128_t) * n), 0); + + // values are exactly the same + std::sort(vec_bkup.begin(), vec_bkup.end()); + std::sort(vec_1.begin(), vec_1.end()); + std::sort(vec_2.begin(), vec_2.end()); + EXPECT_EQ(std::memcmp(vec_bkup.data(), vec_1.data(), sizeof(uint128_t) * n), + 0); + EXPECT_EQ(std::memcmp(vec_bkup.data(), vec_2.data(), sizeof(uint128_t) * n), + 0); +} } // namespace yacl::crypto diff --git a/yacl/crypto/tools/BUILD.bazel b/yacl/crypto/tools/BUILD.bazel index 7a6a3f1..233a2e6 100644 --- a/yacl/crypto/tools/BUILD.bazel +++ b/yacl/crypto/tools/BUILD.bazel @@ -43,6 +43,7 @@ yacl_cc_test( srcs = ["prg_test.cc"], deps = [ ":prg", + "//yacl/crypto/rand", ], ) diff --git a/yacl/crypto/tools/prg.h b/yacl/crypto/tools/prg.h index 42f640a..0416dc0 100644 --- a/yacl/crypto/tools/prg.h +++ b/yacl/crypto/tools/prg.h @@ -227,6 +227,49 @@ uint64_t FillPRandWithMersennePrime(SymmetricCrypto::CryptoType crypto_type, } } +// ----------------------------- +// Fill Pseudorandoms within mod +// ----------------------------- + +// type traits, currently we only support 3 types: +// uint128_t, uint64_t, uint32_t +template +struct IsSupportedLtNContainerType + : public std::disjunction, + std::is_same, + std::is_same> {}; + +template ::value, bool> = true> +uint64_t FillPRandWithLtN(SymmetricCrypto::CryptoType crypto_type, + uint128_t seed, uint64_t iv, uint64_t count, + absl::Span out, T n) { + size_t n_bit_width = 0; + // first, fill all outputs with randomness + if constexpr (std::is_same_v) { + n_bit_width = CountBitWidth(n); + } else { + n_bit_width = absl::bit_width(n); + } + + auto required_size = + (n_bit_width + YACL_MODULE_SECPARAM_S_UINT("prg") + 7) / 8; + Buffer rand_bytes(out.size() * required_size); + auto ret = FillPRand(crypto_type, seed, iv, count, (char*)rand_bytes.data(), + out.size() * required_size); + + // then, perform mod + ByteContainerView rand_view(rand_bytes); + for (size_t i = 0; i < out.size(); ++i) { + math::MPInt r; + r.FromMagBytes(rand_view.subspan(i * required_size, required_size), + Endian::little); + math::MPInt::Mod(r, math::MPInt(n), &r); + out[i] = r.Get(); + } + return ret; +} + // --------------------------- // PRG with cache // --------------------------- diff --git a/yacl/crypto/tools/prg_test.cc b/yacl/crypto/tools/prg_test.cc index 7ad6f63..f623058 100644 --- a/yacl/crypto/tools/prg_test.cc +++ b/yacl/crypto/tools/prg_test.cc @@ -22,6 +22,8 @@ #include "gtest/gtest.h" +#include "yacl/crypto/rand/rand.h" + namespace yacl::crypto { namespace { @@ -324,4 +326,37 @@ TEST(PRTest, MersennePrime8) { } } +TEST(PRTest, Ltn128) { + std::vector out(1000); + uint128_t n = FastRandU128(); + FillPRandWithLtN(SymmetricCrypto::CryptoType::AES128_ECB, 0, 0, 0, + absl::MakeSpan(out), n); + EXPECT_NE(out[0], out[1]); + for (auto e : out) { + EXPECT_LT(e, n); + } +} + +TEST(PRTest, Ltn64) { + std::vector out(1000); + uint64_t n = FastRandU64(); + FillPRandWithLtN(SymmetricCrypto::CryptoType::AES128_ECB, 0, 0, 0, + absl::MakeSpan(out), n); + EXPECT_NE(out[0], out[1]); + for (auto e : out) { + EXPECT_LT(e, n); + } +} + +TEST(PRTest, Ltn32) { + std::vector out(1000); + uint32_t n = FastRandU32(); + FillPRandWithLtN(SymmetricCrypto::CryptoType::AES128_ECB, 0, 0, 0, + absl::MakeSpan(out), n); + EXPECT_NE(out[0], out[1]); + for (auto e : out) { + EXPECT_LT(e, n); + } +} + } // namespace yacl::crypto diff --git a/yacl/math/galois_field/benchmark/BUILD.bazel b/yacl/math/galois_field/benchmark/BUILD.bazel new file mode 100644 index 0000000..02e7a94 --- /dev/null +++ b/yacl/math/galois_field/benchmark/BUILD.bazel @@ -0,0 +1,24 @@ +# Copyright 2024 Ant Group Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("//bazel:yacl.bzl", "yacl_cc_binary") + +yacl_cc_binary( + name = "operation_bench", + srcs = ["operation_bench.cc"], + deps = [ + "//yacl/math/galois_field:gf", + "@com_github_google_benchmark//:benchmark_main", + ], +) diff --git a/yacl/math/galois_field/benchmark/operation_bench.cc b/yacl/math/galois_field/benchmark/operation_bench.cc new file mode 100644 index 0000000..9053693 --- /dev/null +++ b/yacl/math/galois_field/benchmark/operation_bench.cc @@ -0,0 +1,88 @@ +// Copyright 2024 Ant Group Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "benchmark/benchmark.h" + +#include "yacl/math/galois_field/gf.h" + +namespace yacl::math::bench { + +const std::array kLibraryName = {kMPIntLib}; + +class GaloisFieldBM : public benchmark::Fixture { + public: + void SetUp(::benchmark::State &state) override { + const std::string &lib_name = kLibraryName.at(state.range(0)); + int64_t bit_size = state.range(1); + MPInt order; + MPInt::RandPrimeOver(bit_size, &order, PrimeType::Normal); + + gf_ = GaloisFieldFactory::Instance().Create(kPrimeField, ArgLib = lib_name, + ArgMod = order); + } + + protected: + std::unique_ptr gf_; +}; + +BENCHMARK_DEFINE_F(GaloisFieldBM, Add)(benchmark::State &state) { + for (auto _ : state) { + state.PauseTiming(); + auto a = gf_->Random(); + auto b = gf_->Random(); + state.ResumeTiming(); + benchmark::DoNotOptimize(gf_->Add(a, b)); + } +} + +BENCHMARK_DEFINE_F(GaloisFieldBM, Mul)(benchmark::State &state) { + for (auto _ : state) { + state.PauseTiming(); + auto a = gf_->Random(); + auto b = gf_->Random(); + state.ResumeTiming(); + benchmark::DoNotOptimize(gf_->Mul(a, b)); + } +} + +BENCHMARK_DEFINE_F(GaloisFieldBM, Inv)(benchmark::State &state) { + for (auto _ : state) { + state.PauseTiming(); + auto a = gf_->Random(); + state.ResumeTiming(); + benchmark::DoNotOptimize(gf_->Inv(a)); + } +} + +BENCHMARK_DEFINE_F(GaloisFieldBM, Pow)(benchmark::State &state) { + for (auto _ : state) { + state.PauseTiming(); + auto a = gf_->Random(); + MPInt b; + MPInt::RandomLtN(gf_->GetOrder(), &b); + state.ResumeTiming(); + benchmark::DoNotOptimize(gf_->Pow(a, b)); + } +} + +BENCHMARK_REGISTER_F(GaloisFieldBM, Add) + ->ArgsProduct({{0}, benchmark::CreateRange(128, 2048, 2)}); +BENCHMARK_REGISTER_F(GaloisFieldBM, Mul) + ->ArgsProduct({{0}, benchmark::CreateRange(128, 2048, 2)}); +BENCHMARK_REGISTER_F(GaloisFieldBM, Inv) + ->ArgsProduct({{0}, benchmark::CreateRange(128, 2048, 2)}); +BENCHMARK_REGISTER_F(GaloisFieldBM, Pow) + ->ArgsProduct({{0}, benchmark::CreateRange(128, 2048, 2)}); + +} // namespace yacl::math::bench diff --git a/yacl/math/galois_field/factory/BUILD.bazel b/yacl/math/galois_field/factory/BUILD.bazel index ae61608..3c5551f 100644 --- a/yacl/math/galois_field/factory/BUILD.bazel +++ b/yacl/math/galois_field/factory/BUILD.bazel @@ -88,6 +88,9 @@ yacl_cc_library( ], hdrs = ["intel_factory.h"], copts = AES_COPT_FLAGS, + target_compatible_with = [ + "@platforms//cpu:x86_64", + ], visibility = ["//yacl/math/galois_field:__pkg__"], # grant visibility to main target deps = [ ":spi", diff --git a/yacl/math/mpint/benchmark/BUILD.bazel b/yacl/math/mpint/benchmark/BUILD.bazel index 212f4b6..25f0842 100644 --- a/yacl/math/mpint/benchmark/BUILD.bazel +++ b/yacl/math/mpint/benchmark/BUILD.bazel @@ -40,3 +40,12 @@ yacl_cc_binary( "@com_github_google_benchmark//:benchmark_main", ], ) + +yacl_cc_binary( + name = "mod", + srcs = ["mod_bench.cc"], + deps = [ + "//yacl/math/mpint", + "@com_github_google_benchmark//:benchmark_main", + ], +) diff --git a/yacl/math/mpint/benchmark/mod_bench.cc b/yacl/math/mpint/benchmark/mod_bench.cc new file mode 100644 index 0000000..0a05c40 --- /dev/null +++ b/yacl/math/mpint/benchmark/mod_bench.cc @@ -0,0 +1,80 @@ +// Copyright 2024 Ant Group Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "benchmark/benchmark.h" + +#include "yacl/math/mpint/mp_int.h" + +namespace yacl::math::bench { + +class MPIntModBM : public benchmark::Fixture { + public: + void SetUp(::benchmark::State &state) override { + int64_t bit_size = state.range(0); + MPInt::RandPrimeOver(bit_size, &mod_, PrimeType::Normal); + } + + protected: + MPInt mod_; +}; + +BENCHMARK_DEFINE_F(MPIntModBM, AddMod)(benchmark::State &state) { + for (auto _ : state) { + state.PauseTiming(); + MPInt a, b; + MPInt::RandomLtN(mod_, &a); + MPInt::RandomLtN(mod_, &b); + state.ResumeTiming(); + benchmark::DoNotOptimize(a.AddMod(b, mod_)); + } +} + +BENCHMARK_DEFINE_F(MPIntModBM, MulMod)(benchmark::State &state) { + for (auto _ : state) { + state.PauseTiming(); + MPInt a, b; + MPInt::RandomLtN(mod_, &a); + MPInt::RandomLtN(mod_, &b); + state.ResumeTiming(); + benchmark::DoNotOptimize(a.MulMod(b, mod_)); + } +} + +BENCHMARK_DEFINE_F(MPIntModBM, InvMod)(benchmark::State &state) { + for (auto _ : state) { + state.PauseTiming(); + MPInt a; + MPInt::RandomLtN(mod_, &a); + state.ResumeTiming(); + benchmark::DoNotOptimize(a.InvertMod(mod_)); + } +} + +BENCHMARK_DEFINE_F(MPIntModBM, PowMod)(benchmark::State &state) { + for (auto _ : state) { + state.PauseTiming(); + MPInt a, b; + MPInt::RandomLtN(mod_, &a); + MPInt::RandomLtN(mod_, &b); + state.ResumeTiming(); + benchmark::DoNotOptimize(a.PowMod(b, mod_)); + } +} + +BENCHMARK_REGISTER_F(MPIntModBM, AddMod)->RangeMultiplier(2)->Range(128, 2048); +BENCHMARK_REGISTER_F(MPIntModBM, MulMod)->RangeMultiplier(2)->Range(128, 2048); +BENCHMARK_REGISTER_F(MPIntModBM, InvMod)->RangeMultiplier(2)->Range(128, 2048); +BENCHMARK_REGISTER_F(MPIntModBM, PowMod)->RangeMultiplier(2)->Range(128, 2048); + +} // namespace yacl::math::bench