repo-sync-2024-01-23T19:12:47+0800 (#58)

* repo-sync-2024-01-23T19:12:47+0800 * fix unused
secretflow · Jan 23, 2024 · d70b103 · d70b103
1 parent b7d47ae
commit d70b103
Show file tree

Hide file tree

Showing 199 changed files with 4,599 additions and 19,324 deletions.
diff --git a/README.md b/README.md
@@ -26,79 +26,83 @@ receiver.config:
 
 ```json
 {
-    "protocol_config": {
-        "protocol": "PROTOCOL_KKRT",
-        "role": "ROLE_RECEIVER",
-        "broadcast_result": true
+    "psi_config": {
+        "protocol_config": {
+            "protocol": "PROTOCOL_KKRT",
+            "role": "ROLE_RECEIVER",
+            "broadcast_result": true
+        },
+        "input_config": {
+            "type": "IO_TYPE_FILE_CSV",
+            "path": "/root/receiver/receiver_input.csv"
+        },
+        "output_config": {
+            "type": "IO_TYPE_FILE_CSV",
+            "path": "/root/receiver/receiver_output.csv"
+        },
+        "keys": [
+            "id0",
+            "id1"
+        ],
+        "debug_options": {
+            "trace_path": "/root/receiver/receiver.trace"
+        },
+        "link_config": {
+            "parties": [
+                {
+                    "id": "receiver",
+                    "host": "127.0.0.1:5300"
+                },
+                {
+                    "id": "sender",
+                    "host": "127.0.0.1:5400"
+                }
+            ]
+        }
     },
-    "input_config": {
-        "type": "IO_TYPE_FILE_CSV",
-        "path": "/root/receiver/receiver_input.csv"
-    },
-    "output_config": {
-        "type": "IO_TYPE_FILE_CSV",
-        "path": "/root/receiver/receiver_output.csv"
-    },
-    "link_config": {
-        "parties": [
-            {
-                "id": "receiver",
-                "host": "127.0.0.1:5300"
-            },
-            {
-                "id": "sender",
-                "host": "127.0.0.1:5400"
-            }
-        ]
-    },
-    "self_link_party": "receiver",
-    "keys": [
-        "id0",
-        "id1"
-    ],
-    "debug_options": {
-        "trace_path": "/root/receiver/receiver.trace"
-    }
+    "self_link_party": "receiver"
 }
 ```
 
 sender.config:
 
 ```json
 {
-    "protocol_config": {
-        "protocol": "PROTOCOL_KKRT",
-        "role": "ROLE_SENDER",
-        "broadcast_result": true
-    },
-    "input_config": {
-        "type": "IO_TYPE_FILE_CSV",
-        "path": "/root/sender/sender_input.csv"
-    },
-    "output_config": {
-        "type": "IO_TYPE_FILE_CSV",
-        "path": "/root/sender/sender_output.csv"
-    },
-    "link_config": {
-        "parties": [
-            {
-                "id": "receiver",
-                "host": "127.0.0.1:5300"
-            },
-            {
-                "id": "sender",
-                "host": "127.0.0.1:5400"
-            }
-        ]
+    "psi_config": {
+        "protocol_config": {
+            "protocol": "PROTOCOL_KKRT",
+            "role": "ROLE_SENDER",
+            "broadcast_result": true
+        },
+        "input_config": {
+            "type": "IO_TYPE_FILE_CSV",
+            "path": "/root/sender/sender_input.csv"
+        },
+        "output_config": {
+            "type": "IO_TYPE_FILE_CSV",
+            "path": "/root/sender/sender_output.csv"
+        },
+        "keys": [
+            "id0",
+            "id1"
+        ],
+        "debug_options": {
+            "trace_path": "/root/sender/sender.trace"
+        },
+        "link_config": {
+            "parties": [
+                {
+                    "id": "receiver",
+                    "host": "127.0.0.1:5300"
+                },
+                {
+                    "id": "sender",
+                    "host": "127.0.0.1:5400"
+                }
+            ]
+        }
     },
-    "self_link_party": "sender",
-    "keys": [
-        "id0",
-        "id1"
-    ],
-    "debug_options": {
-        "trace_path": "/root/sender/sender.trace"
-    }
+    "self_link_party": "sender"
 }
 ```
 
@@ -115,13 +119,13 @@ sender.config:
 In the first terminal, run the following command
 
 ```bash
-docker run -it  --rm  --network host --mount type=bind,source=/tmp/receiver,target=/root/receiver -w /root  --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=NET_ADMIN --privileged=true secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/psi-anolis8:0.1.0beta bash -c "./main --config receiver/receiver.config"
+docker run -it  --rm  --network host --mount type=bind,source=/tmp/receiver,target=/root/receiver -w /root  --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=NET_ADMIN --privileged=true secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/psi-anolis8:latest bash -c "./main --config receiver/receiver.config"
 ```
 
 In the other terminal, run the following command simultaneously.
 
 ```bash
-docker run -it  --rm  --network host --mount type=bind,source=/tmp/sender,target=/root/sender -w /root  --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=NET_ADMIN --privileged=true secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/psi-anolis8:0.1.0beta bash -c "./main --config sender/sender.config"
+docker run -it  --rm  --network host --mount type=bind,source=/tmp/sender,target=/root/sender -w /root  --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=NET_ADMIN --privileged=true secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/psi-anolis8:latest bash -c "./main --config sender/sender.config"
 ```
 
 ## Building SecretFlow PSI Library
@@ -131,7 +135,7 @@ docker run -it  --rm  --network host --mount type=bind,source=/tmp/sender,target
 
 #### Dev Docker
 
-We use the same dev docker from secretflow/ubuntu-base-ci.
+We use secretflow/ubuntu-base-ci docker image. You may check at [dockerhub](https://hub.docker.com/r/secretflow/ubuntu-base-ci).
 
 ```sh
 # start container
@@ -151,7 +155,7 @@ docker exec -it psi-dev-$(whoami) bash
 #### Linux
 
 ```sh
-Install gcc>=11.2, cmake>=3.26, ninja, nasm>=2.15, python>=3.8, bazel==6.2.1, golang, xxd, lld
+Install gcc>=11.2, cmake>=3.26, ninja, nasm>=2.15, python>=3.8, bazel==6.4.0, golang, xxd, lld
 ```
 
 ### Build & UnitTest

diff --git a/RELEASE.md b/RELEASE.md
@@ -5,6 +5,17 @@
 > - `[API]` prefix for API changes.
 > - `[Improvement]` prefix for implementation improvement.
 
+## v0.2.0.dev240123
+
+- [Feature] add RFC9380 25519 elligator2 hash_to_curve.
+- [Feature] add malicious vole psi.
+- [API] expose ub psi in PSI v2 API.
+- [Improvement] Modify buffer size in sort cmd.
+- [Bugfix] Fix SimpleShuffledBatchProvider.
+- [Bugfix] Fix flakiness in psi_test.
+- [Bugfix] Fix race condition in rr22.
+
+
 ## v0.2.0.dev231228
 
 - [Bugfix] Fix RR22 race condition.
@@ -20,6 +31,7 @@
 - [Bugfix] Fix duplicate key check.
 - [Bugfix] Fix SyncWait.
 
+
 ## v0.1.0beta
 
 - [API] Add PSI v2 API.

diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl
@@ -14,7 +14,6 @@
 
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
-load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
 
 def psi_deps():
     _com_github_nelhage_rules_boost()
@@ -44,9 +43,7 @@ def psi_deps():
     _com_github_floodyberry_curve25519_donna()
     _com_github_ridiculousfish_libdivide()
     _com_github_sparsehash_sparsehash()
-
     _com_github_intel_ipp()
-
     _yacl()
 
 def _yacl():

diff --git a/docker/build.sh b/docker/build.sh
@@ -70,7 +70,7 @@ LATEST_TAG=${DOCKER_REG}/psi-anolis8:latest
 echo -e "Build psi binary ${GREEN}PSI ${PSI_VERSION}${NO_COLOR}..."
 
 if [[ SKIP -eq 0 ]]; then
-    docker run -it  --rm   --mount type=bind,source="$(pwd)/../../psi",target=/home/admin/dev/src -w /home/admin/dev  --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=NET_ADMIN --privileged=true secretflow/release-ci:latest /home/admin/dev/src/docker/entry.sh
+    docker run -it  --rm   --mount type=bind,source="$(pwd)/../../psi",target=/home/admin/dev/src -w /home/admin/dev  --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=NET_ADMIN --privileged=true secretflow/release-ci:1.4 /home/admin/dev/src/docker/entry.sh
     echo -e "Finish building psi binary ${GREEN}${IMAGE_LITE_TAG}${NO_COLOR}"
 fi
 

diff --git a/docker/entry.sh b/docker/entry.sh
@@ -8,6 +8,6 @@ cd src_copied
 
 
 
-bazel build psi/psi:main -c opt --config=linux-release --repository_cache=/tmp/bazel_repo_cache
-chmod 777 bazel-bin/psi/psi/main
-cp bazel-bin/psi/psi/main ../src/docker/
+bazel build psi:main -c opt --config=linux-release --repository_cache=/tmp/bazel_repo_cache
+chmod 777 bazel-bin/psi/main
+cp bazel-bin/psi/main ../src/docker/
diff --git a/docs/getting_started.rst b/docs/getting_started.rst
@@ -7,32 +7,32 @@ Welcome to SecretFlow PSI Library. There are multiple methods to use PSI/PIR.
 * Python packages
 
     * `SPU <https://pypi.org/project/spu/>`_ warps the library as Python bindings. You could call PSI/PIR with spu.
-    * `SecretFlow <https://pypi.org/project/secretflow/>`_ warps spu further.
+    * `SecretFlow <https://pypi.org/project/secretflow/>`_ warps SPU further.
 
 
 * Applications
 
     * `SCQL <https://www.secretflow.org.cn/docs/scql/latest/zh-Hans>`_ integrates this library to do JOIN operations.
     * `SecretPad <https://www.secretflow.org.cn/docs/quickstart/mvp-platform>`_ provides PSI component.
-    * `Easy PSI <https://www.secretflow.org.cn/docs/quickstart/easy-psi>`_ provides most functionality of this library to users.
+    * `Easy PSI <https://www.secretflow.org.cn/docs/quickstart/easy-psi>`_ provides most functionality of this library with User Interface.
 
 
 For PSI, we have a developing v2 PSI.
 
 +------------------------+------------------------------------------------+---------------------------------------------+
 |                        | PSI v1 APIs                                    | PSI v2 APIs                                 |
 +========================+================================================+=============================================+
-| Supported Protocols    | ECDH, BC22, KKRT, ECDH_OPRF_UB, DP_PSI, RR22   | ECDH, KKRT, RR22                            |
+| Supported Protocols    | ECDH, BC22, KKRT, ECDH_OPRF_UB, DP_PSI, RR22   | ECDH, KKRT, RR22, ECDH_OPRF_UB              |
 +------------------------+------------------------------------------------+---------------------------------------------+
 | CSV parser             | Support a subset of csv files.                 | Apache Arrow, support all legal csv files.  |
 +------------------------+------------------------------------------------+---------------------------------------------+
 | Recovery after failure | Unsupported                                    | Supported                                   |
 +------------------------+------------------------------------------------+---------------------------------------------+
-| Inner join             | Unsupported at library level                   | Supported natively                          |
+| Support duplicated keys| Unsupported                                    | Supported                                   |
 +------------------------+------------------------------------------------+---------------------------------------------+
 | Release Docker         | Not provided                                   | Provided                                    |
 +------------------------+------------------------------------------------+---------------------------------------------+
-| Python Binding         | with SPU                                       | Not provided                                |
+| Python Binding         | with SPU                                       | with SPU                                    |
 +------------------------+------------------------------------------------+---------------------------------------------+
 
 
@@ -62,11 +62,31 @@ SPU
 
 Please check `SPU Installation Guidelines <https://www.secretflow.org.cn/docs/spu/latest/en-US/getting_started/install>`_.
 
+APIs: https://www.secretflow.org.cn/docs/spu/latest/en-US/reference/py_api
+
 SecretFlow
 """"""""""
 
 Please check `SecretFlow Installation page <https://www.secretflow.org.cn/docs/secretflow/latest/en-US/getting_started/installation>`_.
 
+APIs:
+
+- PSI v1:
+    - https://www.secretflow.org.cn/docs/secretflow/latest/en-US/source/secretflow#secretflow.SPU.psi_df
+    - https://www.secretflow.org.cn/docs/secretflow/latest/en-US/source/secretflow#secretflow.SPU.psi_csv
+    - https://www.secretflow.org.cn/docs/secretflow/latest/en-US/source/secretflow#secretflow.SPU.psi_join_df
+    - https://www.secretflow.org.cn/docs/secretflow/latest/en-US/source/secretflow#secretflow.SPU.psi_join_csv
+
+- PSI v2:
+    - https://www.secretflow.org.cn/docs/secretflow/latest/en-US/source/secretflow#secretflow.SPU.psi_v2
+
+- PIR:
+    - https://www.secretflow.org.cn/docs/secretflow/latest/en-US/source/secretflow#secretflow.SPU.pir_setup
+    - https://www.secretflow.org.cn/docs/secretflow/latest/en-US/source/secretflow#secretflow.SPU.pir_query
+    - https://www.secretflow.org.cn/docs/secretflow/latest/en-US/source/secretflow#secretflow.SPU.pir_memory_query
+
+- Component: https://www.secretflow.org.cn/docs/secretflow/latest/en-US/component/comp_list#psi
+
 Applications
 ^^^^^^^^^^^^
 
@@ -75,6 +95,10 @@ SCQL
 
 Please check `SCQL Quickstart tutorial <https://www.secretflow.org.cn/docs/scql/latest/en-US/intro/tutorial>`_.
 
+Featured operators using PSI:
+    - https://www.secretflow.org.cn/docs/scql/latest/en-US/reference/operators#in
+    - https://www.secretflow.org.cn/docs/scql/latest/en-US/reference/operators#join
+
 SecretPad
 """""""""
 
@@ -123,7 +147,7 @@ You need to install:
 * ninja
 * nasm>=2.15
 * python>=3.8
-* bazel==6.2.1
+* bazel==6.4.0
 * golang
 * xxd
 * lld
@@ -142,8 +166,6 @@ We use bazel for building and testing::
     # test
     bazel test //...
 
-
-
 Reporting an Issue
 ------------------
 

diff --git a/docs/index.rst b/docs/index.rst
@@ -10,6 +10,8 @@ This is the repo of Private Set Intersection(PSI) and Private Information Retrie
 
 This repo is formerly psi/pir part from `secretflow/spu <https://github.com/secretflow/spu>`_ repo.
 
+We invite you to try `Easy PSI <https://www.secretflow.org.cn/docs/quickstart/easy-psi>`_, a standalone PSI product powered by this library.
+
 .. toctree::
    :maxdepth: 2
    :hidden:

diff --git a/docs/reference/index.rst b/docs/reference/index.rst
@@ -1,10 +1,11 @@
 API reference
 =============
 
-This page covers all public functions and methods.
+This page covers all Protocol Buffers message as APIs.
 
 .. toctree::
    :maxdepth: 1
 
    psi_config
    psi_v2_config
+   launch_config