Merge pull request #221 from ria-com/v3.1

V3.1

.github/workflows/benchmark.yaml

@@ -1,45 +1,33 @@
+
 name: Nomeroff Net CI GPU Benchmark
 
 on:
-  push:
-    branches: [ master, v3.0, v3.1 ]
   pull_request:
     branches: [  master, v3.0, v3.1 ]
+    paths:
+      - 'nomeroff_net/**'
+      - 'examples/**'
 
 jobs:
   gpu-benchmark:
     runs-on: self-hosted
+    #container: docker://dvcorg/cml-py3:latest
 
     # Timeout: https://stackoverflow.com/a/59076067/4521646
     timeout-minutes: 120
 
     steps:
     - uses: actions/checkout@v2
-
-    # Note: This uses an internal pip API and may not always work
-    # https://github.com/actions/cache/blob/master/examples.md#multiple-oss-in-a-workflow
-    - name: Get pip cache
-      env:
-        repo_token: ${{ secrets.GITHUB_TOKEN }}
-      id: pip-cache
-      run: |
-        python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)"
-
-    - name: Cache pip
-      env:
-        repo_token: ${{ secrets.GITHUB_TOKEN }}
-      uses: actions/[email protected]
+    - uses: actions/setup-node@v2
       with:
-        path: ${{ steps.pip-cache.outputs.dir }}
-        key: ${{ runner.os }}-${{ matrix.python-version }}-pip-${{ hashFiles('requirements.txt') }}
-        restore-keys: |
-          ${{ runner.os }}-${{ matrix.python-version }}-pip-
-
+        node-version: '16'
+    - uses: iterative/setup-cml@v1
     - name: Install dependencies
       env:
         repo_token: ${{ secrets.GITHUB_TOKEN }}
       run: |
 
+        apt-get update
         apt-get install -y libopencv-dev
         apt-get install -y libturbojpeg
         apt-get install -y jupyter-nbconvert
@@ -58,16 +46,52 @@ jobs:
         python --version
         pip --version
         pip list
+
+        cd nomeroff_net/pipes/number_plate_keypoints_detectors/cpp_bindings/
+        . ./build_cpp_bindings_unix.sh
+        cd ../../../../
+
       shell: bash
 
     - name: Run benchmark
       env:
         repo_token: ${{ secrets.GITHUB_TOKEN }}
       run: |
-        
-        echo "## Benchmarks metrics" > report.md
-        CUDA_VISIBLE_DEVICES=0 python examples/py/benchmark/accuracy-test.py >> report.md
-        CUDA_VISIBLE_DEVICES=0 python examples/py/benchmark/runtime-test.py >> report.md
-        
+
+        echo "# Benchmarks" > report.md
+
+        echo "## Hardware сharacteristics" >> report.md
+        echo "### GPU info" >> report.md
+        nvidia-smi --query-gpu=gpu_name,driver_version,memory.total --format=csv --id=0 >> report.md
+        echo "### CPU info" >> report.md
+        lscpu | egrep 'Model name|Socket|Thread|NUMA|CPU\(s\)' >> report.md
+
+        echo "## Benchmark runtime" >> report.md
+        # run first time for download models and overclock gpus
+        CUDA_VISIBLE_DEVICES=0 python examples/py/benchmark/runtime-test.py \
+            --pipeline_name=number_plate_detection_and_reading_runtime \
+            --image_loader_name=turbo \
+            --images_glob=./data/examples/benchmark_oneline_np_images/* \
+            --num_run=1 \
+            --batch_size=1 \
+            --num_workers=1
+        # run second time to write report
+        CUDA_VISIBLE_DEVICES=0 python examples/py/benchmark/runtime-test.py \
+                    --pipeline_name=number_plate_detection_and_reading_runtime \
+                    --image_loader_name=turbo \
+                    --images_glob=./data/examples/benchmark_oneline_np_images/* \
+                    --num_run=10 \
+                    --batch_size=1 \
+                    --num_workers=1 >> report.md
+
+        echo "## Benchmark accuracy" >> report.md
+        echo '```diff' >> report.md
+        CUDA_VISIBLE_DEVICES=0 python examples/py/benchmark/accuracy-test.py \
+            --pipeline_name=number_plate_detection_and_reading \
+            --image_loader_name=turbo \
+            --images_glob=./data/examples/benchmark_oneline_np_images/* \
+            --test_file=./data/examples/accuracy_test_data.json >> report.md
+        echo '```' >> report.md
         cml-send-comment report.md
+
       shell: bash

.github/workflows/nn-ci-cpu-testing.yml

@@ -4,16 +4,16 @@
 name: Nomeroff Net CI CPU Testing
 
 on:
-  push:
-    branches: [ master, v3.0, v3.1 ]
-    paths:
-      - 'nomeroff_net/**'
-      - 'examples/**'
   pull_request:
     branches: [  master, v3.0, v3.1 ]
     paths:
       - 'nomeroff_net/**'
-      - 'examples/**'
+      - 'examples/ju/dataset_convertors/**'
+      - 'examples/ju/dataset_tools/**'
+      - 'examples/ju/inference/**'
+      - 'examples/ju/train/**'
+      - 'examples/py/inference/**'
+      - 'examples/py/rest_examples/**'
   schedule:
     - cron: '10 0 * * *'  # Runs at 00:10 UTC every day
 
@@ -138,7 +138,7 @@ jobs:
         python3 -m nomeroff_net.image_loaders.turbo_loader -f nomeroff_net/image_loaders/turbo_loader.py
 
         # test nnmodels
-        python3 nomeroff_net/nnmodels/numberplate_classification_model.py
+        python3 -m nomeroff_net.nnmodels.numberplate_classification_model -f nomeroff_net/nnmodels/numberplate_classification_model.py
         python3 -m nomeroff_net.nnmodels.numberplate_options_model -f nomeroff_net/nnmodels/numberplate_options_model.py
         python3 -m nomeroff_net.nnmodels.fraud_numberpate_options -f nomeroff_net/nnmodels/fraud_numberpate_options.py
         python3 -m nomeroff_net.nnmodels.numberplate_inverse_model -f nomeroff_net/nnmodels/numberplate_inverse_model.py