Enable Framework Benchmark Workflow (#2830)

* enable benchmark. * remove extra label zio-http. * use ubuntu-latest. * remove extra steps temporary. * Revert "remove extra label zio-http." This reverts commit 2ff090e. * test on personal repo. * remove condition. * remove extra labels. * clone the project. * workaround: git init. * git tag. * initial commit. * config git user and email. * do not freeze. * do not freeze plain text * remove extra stuffs. * revert changes. * update ci. * run only on pull request. * reduce the performance floor. * fmt. * remove trigger check on pull request events. * decrease performance floor to 300k/s.
zio · May 11, 2024 · 2667175 · 2667175
1 parent 7b13cb2
commit 2667175
Show file tree

Hide file tree

Showing 4 changed files with 190 additions and 27 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -689,3 +689,159 @@ jobs:
         with:
           path: Main.txt
           key: jmh_benchmarks_${{ github.sha }}
+
+  runBenchmarks-simple:
+    name: Performance Benchmarks (PlainTextBenchmarkServer)
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        scala: [2.13.12]
+        java: [temurin@8]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Clean up
+        id: clean_up
+        env:
+          GITHUB_TOKEN: ${{secrets.ACTIONS_PAT}}
+        run: sudo rm -rf *
+
+      - uses: actions/checkout@v2
+        with:
+          path: zio-http
+
+      - uses: actions/checkout@v2
+        with:
+          repository: khajavi/FrameworkBenchmarks
+          path: FrameworkBenchMarks
+
+      - id: result
+        env:
+          GITHUB_TOKEN: ${{secrets.ACTIONS_PAT}}
+        run: |
+          mkdir -p ./FrameworkBenchMarks/frameworks/Scala/zio-http/src/main/scala
+          cp ./zio-http/zio-http-example/src/main/scala/example/PlainTextBenchmarkServer.scala ./FrameworkBenchMarks/frameworks/Scala/zio-http/src/main/scala/Main.scala
+          cd ./FrameworkBenchMarks
+          cd ./frameworks/Scala/zio-http
+          git init .
+          git config user.email '[email protected]'
+          git config user.name 'ZIO Benchmark'
+          git add build.sbt
+          git commit -m 'initial commit'
+          git tag v1.0.0
+          git clone https://github.com/${{github.event.pull_request.head.repo.owner.login}}/zio-http.git
+          cd zio-http
+          git checkout ${{github.event.pull_request.head.sha}}
+          cd ../../../..
+          ./tfb  --test zio-http | tee result
+          RESULT_REQUEST=$(echo $(grep -B 1 -A 17 "Concurrency: 256 for plaintext" result) | grep -oiE "requests/sec: [0-9]+.[0-9]+" | grep -oiE "[0-9]+" | head -1)
+          RESULT_CONCURRENCY=$(echo $(grep -B 1 -A 17 "Concurrency: 256 for plaintext" result) | grep -oiE "concurrency: [0-9]+" | grep -oiE "[0-9]+")
+          echo "request_per_second=$RESULT_REQUEST" >> $GITHUB_OUTPUT
+          echo "concurrency=$RESULT_CONCURRENCY" >> $GITHUB_OUTPUT
+
+      - if: ${{github.event.pull_request.head.repo.full_name == 'zio/zio-http'}}
+        uses: peter-evans/commit-comment@v2
+        with:
+          sha: ${{github.event.pull_request.head.sha}}
+          body: |
+
+            **🚀 :** Performance Benchmarks (PlainTextBenchmarkServer)
+
+             concurrency: ${{steps.result.outputs.concurrency}}
+             requests/sec:  ${{steps.result.outputs.request_per_second}}
+
+      
+      - name: Performance Report
+        id: perf-report
+        env:
+          REQUESTS_PER_SECOND: ${{steps.result.outputs.request_per_second}}
+          CONCURRENCY: ${{steps.result.outputs.concurrency}}
+          PERFORMANCE_FLOOR: 300000
+        run: |
+          echo "** 🚀 Performance Benchmarks (PlainTextBenchmarkServer) Report 🚀 **" 
+
+          echo "$REQUESTS_PER_SECOND requests/sec for $CONCURRENCY concurrent requests"
+
+          if (( REQUESTS_PER_SECOND > PERFORMANCE_FLOOR )); then
+            echo "Woohoo! Performance is good! $REQUESTS_PER_SECOND requests/sec exceeds the performance floor of $PERFORMANCE_FLOOR requests/sec."
+          else 
+            echo "Performance benchmark failed with $REQUESTS_PER_SECOND req/sec! Performance must exceed $PERFORMANCE_FLOOR req/sec."
+             exit 1
+          fi
+
+  runBenchmarks-effectful:
+    name: Performance Benchmarks (SimpleEffectBenchmarkServer)
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        scala: [2.13.12]
+        java: [temurin@8]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Clean up
+        id: clean_up
+        env:
+          GITHUB_TOKEN: ${{secrets.ACTIONS_PAT}}
+        run: sudo rm -rf *
+
+      - uses: actions/checkout@v2
+        with:
+          path: zio-http
+
+      - uses: actions/checkout@v2
+        with:
+          repository: khajavi/FrameworkBenchmarks
+          path: FrameworkBenchMarks
+
+      - id: result
+        env:
+          GITHUB_TOKEN: ${{secrets.ACTIONS_PAT}}
+        run: |
+          mkdir -p ./FrameworkBenchMarks/frameworks/Scala/zio-http/src/main/scala
+          cp ./zio-http/zio-http-example/src/main/scala/example/SimpleEffectBenchmarkServer.scala ./FrameworkBenchMarks/frameworks/Scala/zio-http/src/main/scala/Main.scala
+          cd ./FrameworkBenchMarks
+          cd ./frameworks/Scala/zio-http
+          git init .
+          git config user.email '[email protected]'
+          git config user.name 'ZIO Benchmark'
+          git add build.sbt
+          git commit -m 'initial commit'
+          git tag v1.0.0
+          git clone https://github.com/${{github.event.pull_request.head.repo.owner.login}}/zio-http.git
+          cd zio-http
+          git checkout ${{github.event.pull_request.head.sha}}
+          cd ../../../..
+          ./tfb  --test zio-http | tee result
+          RESULT_REQUEST=$(echo $(grep -B 1 -A 17 "Concurrency: 256 for plaintext" result) | grep -oiE "requests/sec: [0-9]+.[0-9]+" | grep -oiE "[0-9]+" | head -1)
+          RESULT_CONCURRENCY=$(echo $(grep -B 1 -A 17 "Concurrency: 256 for plaintext" result) | grep -oiE "concurrency: [0-9]+" | grep -oiE "[0-9]+")
+          echo "request_per_second=$RESULT_REQUEST" >> $GITHUB_OUTPUT
+          echo "concurrency=$RESULT_CONCURRENCY" >> $GITHUB_OUTPUT
+
+      - if: ${{github.event.pull_request.head.repo.full_name == 'zio/zio-http'}}
+        uses: peter-evans/commit-comment@v2
+        with:
+          sha: ${{github.event.pull_request.head.sha}}
+          body: |
+
+            **🚀 :** Performance Benchmarks (SimpleEffectBenchmarkServer)
+
+             concurrency: ${{steps.result.outputs.concurrency}}
+             requests/sec:  ${{steps.result.outputs.request_per_second}}
+
+      
+      - name: Performance Report
+        id: perf-report
+        env:
+          REQUESTS_PER_SECOND: ${{steps.result.outputs.request_per_second}}
+          CONCURRENCY: ${{steps.result.outputs.concurrency}}
+          PERFORMANCE_FLOOR: 300000
+        run: |
+          echo "** 🚀 Performance Benchmarks (SimpleEffectBenchmarkServer) Report 🚀 **" 
+
+          echo "$REQUESTS_PER_SECOND requests/sec for $CONCURRENCY concurrent requests"
+
+          if (( REQUESTS_PER_SECOND > PERFORMANCE_FLOOR )); then
+            echo "Woohoo! Performance is good! $REQUESTS_PER_SECOND requests/sec exceeds the performance floor of $PERFORMANCE_FLOOR requests/sec."
+          else 
+            echo "Performance benchmark failed with $REQUESTS_PER_SECOND req/sec! Performance must exceed $PERFORMANCE_FLOOR req/sec."
+             exit 1
+          fi
diff --git a/build.sbt b/build.sbt
@@ -28,7 +28,7 @@ ThisBuild / githubWorkflowAddedJobs    :=
       steps = List(WorkflowStep.Use(UseRef.Public("release-drafter", "release-drafter", s"v${releaseDrafterVersion}"))),
       cond = Option("${{ github.base_ref == 'main' }}"),
     ),
-  ) ++ ScoverageWorkFlow(50, 60) ++ JmhBenchmarkWorkflow(1) // ++ BenchmarkWorkFlow()
+  ) ++ ScoverageWorkFlow(50, 60) ++ JmhBenchmarkWorkflow(1) ++ BenchmarkWorkFlow()
 
 ThisBuild / githubWorkflowTargetTags ++= Seq("v*")
 ThisBuild / githubWorkflowPublishTargetBranches += RefPredicate.StartsWith(Ref.Tag("v"))

diff --git a/project/BenchmarkWorkFlow.scala b/project/BenchmarkWorkFlow.scala
@@ -6,26 +6,26 @@ object BenchmarkWorkFlow {
     makeBenchmarkPass(
       "runBenchmarks-simple",
       "Performance Benchmarks (PlainTextBenchmarkServer)",
-      800000,
+      300000,
       "PlainTextBenchmarkServer",
     ),
     makeBenchmarkPass(
       "runBenchmarks-effectful",
       "Performance Benchmarks (SimpleEffectBenchmarkServer)",
-      500000,
+      300000,
       "SimpleEffectBenchmarkServer",
     ),
   )
 
   private def makeBenchmarkPass(id: String, name: String, performanceFloor: Int, server: String) =
     WorkflowJob(
-      runsOnExtraLabels = List("zio-http"),
+//      runsOnExtraLabels = List("zio-http"),
       id = id,
       name = name,
-      oses = List("centos"),
-      cond = Some(
-        "${{ github.event_name == 'pull_request'}}",
-      ),
+      oses = List("ubuntu-latest"),
+//      cond = Some(
+//        "${{ github.event_name == 'pull_request'}}",
+//      ),
       scalas = List(Scala213),
       steps = List(
         WorkflowStep.Run(
@@ -43,7 +43,7 @@ object BenchmarkWorkFlow {
         WorkflowStep.Use(
           UseRef.Public("actions", "checkout", s"v2"),
           Map(
-            "repository" -> "zio/FrameworkBenchmarks",
+            "repository" -> "khajavi/FrameworkBenchmarks",
             "path"       -> "FrameworkBenchMarks",
           ),
         ),
@@ -54,7 +54,17 @@ object BenchmarkWorkFlow {
             "mkdir -p ./FrameworkBenchMarks/frameworks/Scala/zio-http/src/main/scala",
             s"cp ./zio-http/zio-http-example/src/main/scala/example/${server}.scala ./FrameworkBenchMarks/frameworks/Scala/zio-http/src/main/scala/Main.scala",
             "cd ./FrameworkBenchMarks",
-            """sed -i "s/---COMMIT_SHA---/${{github.event.pull_request.head.repo.owner.login}}\/zio-http.git#${{github.event.pull_request.head.sha}}/g" frameworks/Scala/zio-http/build.sbt""",
+            "cd ./frameworks/Scala/zio-http",
+            "git init .",
+            "git config user.email '[email protected]'",
+            "git config user.name 'ZIO Benchmark'",
+            "git add build.sbt",
+            "git commit -m 'initial commit'",
+            "git tag v1.0.0",
+            "git clone https://github.com/${{github.event.pull_request.head.repo.owner.login}}/zio-http.git",
+            "cd zio-http",
+            "git checkout ${{github.event.pull_request.head.sha}}",
+            "cd ../../../..",
             "./tfb  --test zio-http | tee result",
             """RESULT_REQUEST=$(echo $(grep -B 1 -A 17 "Concurrency: 256 for plaintext" result) | grep -oiE "requests/sec: [0-9]+.[0-9]+" | grep -oiE "[0-9]+" | head -1)""",
             """RESULT_CONCURRENCY=$(echo $(grep -B 1 -A 17 "Concurrency: 256 for plaintext" result) | grep -oiE "concurrency: [0-9]+" | grep -oiE "[0-9]+")""",

diff --git a/zio-http-example/src/main/scala/example/PlainTextBenchmarkServer.scala b/zio-http-example/src/main/scala/example/PlainTextBenchmarkServer.scala
@@ -19,23 +19,20 @@ object PlainTextBenchmarkServer extends ZIOAppDefault {
 
   private val STATIC_SERVER_NAME = "zio-http"
 
-  private val frozenJsonResponse = Response
-    .json(jsonMessage)
-//    .serverTime
-    .addHeader(Header.Server(STATIC_SERVER_NAME))
-
-  private val frozenPlainTextResponse = Response
-    .text(plainTextMessage)
-//    .serverTime
-    .addHeader(Header.Server(STATIC_SERVER_NAME))
-
-  private def plainTextApp(response: Response): Routes[Any, Response] =
-    Routes(Method.GET / plaintextPath -> Handler.fromResponse(response))
-
-  private def jsonApp(json: Response): Routes[Any, Response] =
-    Routes(Method.GET / jsonPath -> Handler.fromResponse(json))
-
-  val app: Routes[Any, Response] = plainTextApp(frozenPlainTextResponse) ++ jsonApp(frozenJsonResponse)
+  val app: Routes[Any, Response] = Routes(
+    Method.GET / plaintextPath ->
+      Handler.fromResponse(
+        Response
+          .text(plainTextMessage)
+          .addHeader(Header.Server(STATIC_SERVER_NAME)),
+      ),
+    Method.GET / jsonPath      ->
+      Handler.fromResponse(
+        Response
+          .json(jsonMessage)
+          .addHeader(Header.Server(STATIC_SERVER_NAME)),
+      ),
+  )
 
   private val config = Server.Config.default
     .port(8080)