Add the test coverage report and reviewer check (#131)

- Could view test coverage report online.
- Need specific user review.

.coveragerc

@@ -0,0 +1,5 @@
+[html]
+directory = coverage
+
+[run]
+data_file = .coverage_$LOCAL_RANK

.github/workflows/approve.yml

@@ -0,0 +1,33 @@
+name: approve
+
+on:
+  pull_request:
+    branches: [ "main" ]
+    types: [opened, synchronize, reopened]
+
+jobs:
+  approve:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@v2
+
+    - name: Get PR Approvals
+      run: |
+        PR_NUMBER=$(jq --raw-output .number "$GITHUB_EVENT_PATH")
+        APPROVERS=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+          "https://api.github.com/repos/${{ github.repository }}/pulls/$PR_NUMBER/reviews" \
+          | jq -r '.[] | select(.state == "APPROVED") | .user.login')
+
+        echo "APPROVERS=$APPROVERS" >> $GITHUB_ENV
+
+    - name: Check for Specific Approver
+      run: |
+        SPECIFIC_APPROVER="aoyulong"
+        if echo "$APPROVERS" | grep -q "$SPECIFIC_APPROVER"; then
+          echo "Specific approver has approved the PR."
+        else
+          echo "The PR has not been approved by the specific approver."
+          exit 1
+        fi

.github/workflows/test.yml

@@ -1,7 +1,7 @@
 # This workflow will install Python dependencies, run tests and lint with a single version of Python
 # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
 
-name: flagscale-test
+name: test
 
 on:
   push:
@@ -10,35 +10,45 @@ on:
     branches: [ "main" ]
 
 jobs:
-  container-test-job:
+  test:
       runs-on: self-hosted
       container:
-        image: localhost:5000/flagscale_cicd:v1.2
+        image: localhost:5000/flagscale_cicd:v1.3
         ports:
           - 80
-        options: --gpus all --hostname flagscale_cicd 
+        volumes:
+          - /home/flagscale_cicd/flask/static:/workspace/report
+        options: --gpus all --hostname flagscale_cicd
       steps:
-        - name: checkout-code
+        - name: Checkout Code
           uses: actions/checkout@v2
 
-        - name: unit_test-megatron
+        - name: Megatron Unit Test
           run: |
             cd megatron
             export PYTHONPATH=..:$PYTHONPATH
-            torchrun --nproc_per_node=8 -m pytest --import-mode=importlib -q -x tests/unit_tests/data
-            torchrun --nproc_per_node=8 -m pytest --import-mode=importlib -q -x tests/unit_tests/dist_checkpointing
-            torchrun --nproc_per_node=8 -m pytest --import-mode=importlib -q -x tests/unit_tests/fusions
-            torchrun --nproc_per_node=8 -m pytest --import-mode=importlib -q -x tests/unit_tests/models
-            torchrun --nproc_per_node=8 -m pytest --import-mode=importlib -q -x tests/unit_tests/pipeline_parallel
-            torchrun --nproc_per_node=8 -m pytest --import-mode=importlib -q -x tests/unit_tests/tensor_parallel
-            torchrun --nproc_per_node=8 -m pytest --import-mode=importlib -q -x tests/unit_tests/transformer
-            torchrun --nproc_per_node=8 -m pytest --import-mode=importlib -q -x tests/unit_tests/*.py
-            
-        - name: unit_test-flagscale
+            torchrun --nproc_per_node=8 -m pytest --import-mode=importlib --cov-append --cov-report=html:/workspace/report/${{github.sha}}/cov-report-megatron --cov=megatron/core -q -x tests/unit_tests/data
+            torchrun --nproc_per_node=8 -m pytest --import-mode=importlib --cov-append --cov-report=html:/workspace/report/${{github.sha}}/cov-report-megatron --cov=megatron/core -q -x tests/unit_tests/dist_checkpointing
+            torchrun --nproc_per_node=8 -m pytest --import-mode=importlib --cov-append --cov-report=html:/workspace/report/${{github.sha}}/cov-report-megatron --cov=megatron/core -q -x tests/unit_tests/fusions
+            torchrun --nproc_per_node=8 -m pytest --import-mode=importlib --cov-append --cov-report=html:/workspace/report/${{github.sha}}/cov-report-megatron --cov=megatron/core -q -x tests/unit_tests/models
+            torchrun --nproc_per_node=8 -m pytest --import-mode=importlib --cov-append --cov-report=html:/workspace/report/${{github.sha}}/cov-report-megatron --cov=megatron/core -q -x tests/unit_tests/pipeline_parallel
+            torchrun --nproc_per_node=8 -m pytest --import-mode=importlib --cov-append --cov-report=html:/workspace/report/${{github.sha}}/cov-report-megatron --cov=megatron/core -q -x tests/unit_tests/tensor_parallel
+            torchrun --nproc_per_node=8 -m pytest --import-mode=importlib --cov-append --cov-report=html:/workspace/report/${{github.sha}}/cov-report-megatron --cov=megatron/core -q -x tests/unit_tests/transformer 
+            torchrun --nproc_per_node=8 -m pytest --import-mode=importlib --cov-append --cov-report=html:/workspace/report/${{github.sha}}/cov-report-megatron --cov=megatron/core -q -x tests/unit_tests/*.py 
+        
+        - name: Megatron Unit Test Coverage Online Report
+          run: |
+            echo "You can access the test coverage report at the http://120.92.110.224:8081/${{github.sha}}/cov-report-megatron/index.html"
+
+        - name: Flagscale Unit Test
           run: |
-            torchrun --nproc_per_node=8 -m pytest --import-mode=importlib -q -x tests/unit_tests/launcher
+            torchrun --nproc_per_node=8 -m pytest --import-mode=importlib --cov-append --cov-report=html:/workspace/report/${{github.sha}}/cov-report-flagscale --cov=flagscale -q -x tests/unit_tests/launcher
         
-        - name: functional_test-flagscale
+        - name: Flagscale Unit Test Coverage Online Report
+          run: |
+            echo "You can access the test coverage report at the http://120.92.110.224:8081/${{github.sha}}/cov-report-flagscale/index.html"
+
+        - name: Flagscale Functional Test
           run: |
             python run.py --config-path tests/functional_tests/aquila/conf --config-name config action=test
             pytest -s tests/functional_tests/test_result.py --test_reaults_path=./tests/functional_tests/aquila/test_result

tests/scripts/unit_test_flagscale.sh

@@ -1 +1 @@
-torchrun --nproc_per_node=8 -m pytest --import-mode=importlib -q -x tests/unit_tests/launcher
+torchrun --nproc_per_node=8 -m pytest --import-mode=importlib --cov-append --cov-report=html:./cov-report-flagscale --cov=flagscale -q -x tests/unit_tests/launcher

tests/scripts/unit_test_megatron.sh

@@ -2,11 +2,11 @@ cd megatron
 
 export PYTHONPATH=..:$PYTHONPATH
 
-torchrun --nproc_per_node=8 -m pytest --import-mode=importlib -q -x tests/unit_tests/data
-torchrun --nproc_per_node=8 -m pytest --import-mode=importlib -q -x tests/unit_tests/dist_checkpointing
-torchrun --nproc_per_node=8 -m pytest --import-mode=importlib -q -x tests/unit_tests/fusions
-torchrun --nproc_per_node=8 -m pytest --import-mode=importlib -q -x tests/unit_tests/models
-torchrun --nproc_per_node=8 -m pytest --import-mode=importlib -q -x tests/unit_tests/pipeline_parallel
-torchrun --nproc_per_node=8 -m pytest --import-mode=importlib -q -x tests/unit_tests/tensor_parallel
-torchrun --nproc_per_node=8 -m pytest --import-mode=importlib -q -x tests/unit_tests/transformer 
-torchrun --nproc_per_node=8 -m pytest --import-mode=importlib -q -x tests/unit_tests/*.py 
+torchrun --nproc_per_node=8 -m pytest --import-mode=importlib --cov-append --cov-report=html:../cov-report-megatron --cov=megatron/core -q -x tests/unit_tests/data
+torchrun --nproc_per_node=8 -m pytest --import-mode=importlib --cov-append --cov-report=html:../cov-report-megatron --cov=megatron/core -q -x tests/unit_tests/dist_checkpointing
+torchrun --nproc_per_node=8 -m pytest --import-mode=importlib --cov-append --cov-report=html:../cov-report-megatron --cov=megatron/core -q -x tests/unit_tests/fusions
+torchrun --nproc_per_node=8 -m pytest --import-mode=importlib --cov-append --cov-report=html:../cov-report-megatron --cov=megatron/core -q -x tests/unit_tests/models
+torchrun --nproc_per_node=8 -m pytest --import-mode=importlib --cov-append --cov-report=html:../cov-report-megatron --cov=megatron/core -q -x tests/unit_tests/pipeline_parallel
+torchrun --nproc_per_node=8 -m pytest --import-mode=importlib --cov-append --cov-report=html:../cov-report-megatron --cov=megatron/core -q -x tests/unit_tests/tensor_parallel
+torchrun --nproc_per_node=8 -m pytest --import-mode=importlib --cov-append --cov-report=html:../cov-report-megatron --cov=megatron/core -q -x tests/unit_tests/transformer 
+torchrun --nproc_per_node=8 -m pytest --import-mode=importlib --cov-append --cov-report=html:../cov-report-megatron --cov=megatron/core -q -x tests/unit_tests/*.py