fix save benchmark

juicedata · May 6, 2024 · 863ca7d · 863ca7d
1 parent 3207f90
commit 863ca7d
Show file tree

Hide file tree

Showing 3 changed files with 37 additions and 11 deletions.
diff --git a/.github/scripts/save_benchmark.sh b/.github/scripts/save_benchmark.sh
@@ -54,7 +54,7 @@ save_benchmark(){
     }
 EOF
     cat result.json
-    # AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID AWS_ACCESS_KEY_SECRET=$AWS_ACCESS_KEY_SECRET ./juicefs sync --force-update result.json s3://juicefs-ci-aws.s3.us-east-1.amazonaws.com/ci-report/fio-test/$created_date/$name
+    AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY ./juicefs sync --force-update result.json s3://juicefs-ci-aws.s3.us-east-1.amazonaws.com/ci-report/fio-test/$created_date/$name
 }
 
 save_benchmark $@
diff --git a/.github/workflows/load.yml b/.github/workflows/load.yml
@@ -87,24 +87,44 @@ jobs:
 
       - name: Load and dump with small directory
         timeout-minutes: 30
+        env:
+          AWS_ACCESS_KEY_ID: ${{secrets.CI_AWS_ACCESS_KEY_ID}}
+          AWS_SECRET_ACCESS_KEY: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}}
+          META: ${{matrix.meta}}
+          START_META: true
         run: |
-          sudo MYSQL_PASSWORD=${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}} META=${{matrix.meta}} START_META=true .github/scripts/command/load_dump_bench.sh test_load_dump_with_small_dir
+          sudo -E .github/scripts/command/load_dump_bench.sh test_load_dump_with_small_dir
 
       - name: Load and dump with big directory
         timeout-minutes: 30
+        env:
+          AWS_ACCESS_KEY_ID: ${{secrets.CI_AWS_ACCESS_KEY_ID}}
+          AWS_SECRET_ACCESS_KEY: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}}
+          META: ${{matrix.meta}}
+          START_META: false
         run: |
-          sudo MYSQL_PASSWORD=${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}} META=${{matrix.meta}} START_META=false .github/scripts/command/load_dump_bench.sh test_load_dump_with_big_dir
+          sudo -E .github/scripts/command/load_dump_bench.sh test_load_dump_with_big_dir
       
       - name: Load and dump subdir with big directory
         if: false
         timeout-minutes: 30
+        env:
+          AWS_ACCESS_KEY_ID: ${{secrets.CI_AWS_ACCESS_KEY_ID}}
+          AWS_SECRET_ACCESS_KEY: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}}
+          META: ${{matrix.meta}} 
+          START_META: false 
         run: |
-          sudo MYSQL_PASSWORD=${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}} META=${{matrix.meta}} START_META=false .github/scripts/command/load_dump_bench.sh test_load_dump_with_big_dir_subdir
+          sudo -E .github/scripts/command/load_dump_bench.sh test_load_dump_with_big_dir_subdir
           
       - name: List big directory
         timeout-minutes: 30
+        env:
+          AWS_ACCESS_KEY_ID: ${{secrets.CI_AWS_ACCESS_KEY_ID}}
+          AWS_SECRET_ACCESS_KEY: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}}
+          META: ${{matrix.meta}} 
+          START_META: false 
         run: | 
-          sudo MYSQL_PASSWORD=${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}} META=${{matrix.meta}} START_META=false .github/scripts/command/load_dump_bench.sh test_list_with_big_dir
+          sudo -E .github/scripts/command/load_dump_bench.sh test_list_with_big_dir
 
       - name: log
         if: always()

diff --git a/.github/workflows/sysbench.yml b/.github/workflows/sysbench.yml
@@ -86,7 +86,8 @@ jobs:
       - name: seq read 1M
         uses: ./.github/actions/sysbench
         with:
-          mysql_password: ${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}}
+          AWS_ACCESS_KEY_ID: ${{secrets.CI_AWS_ACCESS_KEY_ID}}
+          AWS_SECRET_ACCESS_KEY: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}}
           file_num: 1000
           file_total_size: '1G'
           file_test_mode: 'seqrd'
@@ -98,7 +99,8 @@ jobs:
       - name: seq write 1M
         uses: ./.github/actions/sysbench
         with:
-          mysql_password: ${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}}
+          AWS_ACCESS_KEY_ID: ${{secrets.CI_AWS_ACCESS_KEY_ID}}
+          AWS_SECRET_ACCESS_KEY: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}}
           file_num: 1000
           file_total_size: '1G'
           file_test_mode: 'seqwr'
@@ -110,7 +112,8 @@ jobs:
       - name: seq read 100K
         uses: ./.github/actions/sysbench
         with:
-          mysql_password: ${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}}
+          AWS_ACCESS_KEY_ID: ${{secrets.CI_AWS_ACCESS_KEY_ID}}
+          AWS_SECRET_ACCESS_KEY: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}}
           file_num: 10000
           file_total_size: '1G'
           file_test_mode: 'seqrd'
@@ -122,7 +125,8 @@ jobs:
       - name: seq write 100K
         uses: ./.github/actions/sysbench
         with:
-          mysql_password: ${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}}
+          AWS_ACCESS_KEY_ID: ${{secrets.CI_AWS_ACCESS_KEY_ID}}
+          AWS_SECRET_ACCESS_KEY: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}}
           file_num: 10000
           file_total_size: '1G'
           file_test_mode: 'seqwr'
@@ -134,7 +138,8 @@ jobs:
       - name: random readwrite 1M
         uses: ./.github/actions/sysbench
         with:
-          mysql_password: ${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}}
+          AWS_ACCESS_KEY_ID: ${{secrets.CI_AWS_ACCESS_KEY_ID}}
+          AWS_SECRET_ACCESS_KEY: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}}
           file_num: 1000
           file_total_size: '1G'
           file_test_mode: 'rndrw'
@@ -146,7 +151,8 @@ jobs:
       - name: random readwrite 100K
         uses: ./.github/actions/sysbench
         with:
-          mysql_password: ${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}}
+          AWS_ACCESS_KEY_ID: ${{secrets.CI_AWS_ACCESS_KEY_ID}}
+          AWS_SECRET_ACCESS_KEY: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}}
           file_num: 10000
           file_total_size: '1G'
           file_test_mode: 'rndrw'