Convert script from old tensorflow model to onnx (#482)

* tf2onnx convert approach

* add ml_model integrity check

* description added

* Python3.10 checked

* move script to experiment

* removed extra file

* Delete credsweeper/ml_model/ml_model.onnx.md5

.github/workflows/check.yml

@@ -53,40 +53,12 @@ jobs:
         done
         exit ${n}
 
-    # # # git workflow
+    # # # ml_model integrity
 
-    - name: Get latest release tag name
+    - name: Check ml_model.onnx integrity
       if: ${{ always() && steps.code_checkout.conclusion == 'success' }}
       run: |
-        if [ "pull_request" == "${{ github.event_name }}" ]; then
-            API_RELEASE_URL=$(echo "${{ github.event.pull_request.base.repo.releases_url }}")
-        else
-            API_RELEASE_URL=$(echo "${{ github.event.repository.releases_url }}")
-        fi
-        echo "'${API_RELEASE_URL}'" # dbg
-        API_RELEASE_URL=$(echo "${API_RELEASE_URL}" | sed 's|.....$||')
-        echo "'${API_RELEASE_URL}'" # dbg
-        API_RELEASE_URL=$(echo "${API_RELEASE_URL}/latest")
-        echo "'${API_RELEASE_URL}'" # dbg
-        LATEST_RELEASE_TAG=$(
-            curl \
-                --silent \
-                --header "Accept: application/vnd.github.v3+json" \
-                --header "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
-                ${API_RELEASE_URL} \
-            | \
-            jq \
-                --raw-output \
-                '.tag_name' \
-            )
-        echo "LATEST_RELEASE_TAG='${LATEST_RELEASE_TAG}'"
-        export GIT_ANCESTOR=${LATEST_RELEASE_TAG}
-        if bash cicd/git_workflow.sh; then
-            echo "GIT workflow OK"
-        else
-            echo "Please, rebase the branch after ${LATEST_RELEASE_TAG}"
-            exit 1
-        fi
+        md5sum --binary credsweeper/ml_model/ml_model.onnx | grep 8cb870a200d7bc07893aacec38f54033
 
     # # # Python setup
 

cicd/README.md

@@ -2,4 +2,3 @@
 
 The following files are used for:
   * **benchmark.txt** - template scores to compare it with benchmark
-  * **git_workflow.sh** - checks whether branch in is pure rebased after latest release

experiment/tf2onnx/requirements.txt

@@ -0,0 +1,8 @@
+h5py==3.10.0
+keras==2.13.1
+numpy==1.23.5
+onnx==1.15.0
+protobuf==3.20.3
+tensorflow==2.13.1
+tf2onnx==1.16.0
+wrapt==1.14.1

experiment/tf2onnx/tf2onnx.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -e
+
+# tensorflow model may be obtained like this: git restore -s be06d6059f0def4f0fdb50444c08db4ce542173e -- ml_model.h5
+# use virtual environment and the requirements.txt - there are very specific luke combination of packages verions
+# python -m venv .venv
+# . .venv/bin/activate
+# python -m pip install --upgrade pip
+# python -m pip install --requirement requirements.txt
+
+# [optional] thransform model form h5 to saved directory
+python -c 'import tensorflow as tf;model=tf.keras.models.load_model("ml_model.h5");model.save("ml_model")'
+# transform the model
+python -m tf2onnx.convert --saved-model ml_model --output ml_model.onnx --verbose --rename-inputs feature_input,line_input
+# md5sum for integrity
+md5sum --binary ml_model.onnx