Test run e2e passing with gpt-3.5 and multistep

.github/workflows/run-e2e-tests-on-azure.yml

@@ -1,25 +1,28 @@
-name: Continous Integration and Deployment
+name: Run E2E tests on Azure
 
 on:
-  schedule:
-    # Run cron job at 8AM Monday every week.
-    - cron: '0 8 * * 1'
+  push:
+    branches:
+      - run-multistep-with-gpt-3.5
+#  schedule:
+#    # Run cron job at 8AM Monday every week.
+#    - cron: '0 8 * * 1'
   workflow_dispatch:
 
 env:
   DEFAULT_PYTHON_VERSION: '3.9'
   POETRY_VERSION: 1.8.2
 
 jobs:
-  train-model:
+  train-model-and-run-tests:
     name: Train model
     runs-on: ubuntu-22.04
 
     steps:
       - name: Checkout git repository 🕝
         uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
         with:
-          ref: test-azure-openai
+          ref: azure-openai-release-3.9
 
       - name: Setup Python
         uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b
@@ -46,18 +49,6 @@ jobs:
       - name: Set up virtual environment
         run: poetry config virtualenvs.in-project true
 
-      # Authenticate with gcloud for release registry (where Rasa is published)
-      - id: "auth-release"
-        name: Authenticate with gcloud for release registry 🎫
-        uses: "google-github-actions/auth@ef5d53e30bbcd8d0836f4288f5e50ff3e086997d"
-        with:
-          token_format: 'access_token'
-          credentials_json: "${{ secrets.RASA_RELEASES_READ }}"
-
-      - name: Configure OAuth token for poetry
-        run: |
-          poetry config http-basic.rasa-plus oauth2accesstoken $(gcloud auth print-access-token)
-
       - name: Install Dependencies 📦
         run: |
           make install
@@ -77,94 +68,43 @@ jobs:
           name: trained-model
           path: models/${{steps.upload_model.outputs.model}}.tar.gz
 
-  run_e2e_tests:
-    name: Run e2e Tests
-    runs-on: ubuntu-22.04
-    needs: [train-model]
+      - name: Init LLM Cache
+        id: cache-llm
+        uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8
+        with:
+          path: .rasa
+          key: rasa-llm-cache
 
-    steps:
-    - name: Checkout git repository 🕝
-      uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
-      with:
-        ref: test-azure-openai
-
-    - name: Setup Python
-      uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b
-      with:
-        python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
-
-    - name: Install poetry 🦄
-      uses: Gr1N/setup-poetry@15821dc8a61bc630db542ae4baf6a7c19a994844
-      with:
-        poetry-version: ${{ env.POETRY_VERSION }}
-
-    - name: Load Poetry Cached Libraries ⬇
-      id: cache-poetry
-      uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8
-      with:
-        path: .venv
-        key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-${{ env.DEFAULT_PYTHON_VERSION }}-${{ hashFiles('**/poetry.lock') }}
-        restore-keys: ${{ runner.os }}-poetry-${{ env.DEFAULT_PYTHON_VERSION }}
-
-    - name: Create virtual environment
-      if: steps.cache-poetry.outputs.cache-hit != 'true'
-      run: python -m venv create .venv
-
-    - name: Set up virtual environment
-      run: poetry config virtualenvs.in-project true
-
-      # Authenticate with gcloud for release registry (where Rasa is published)
-    - id: "auth-release"
-      name: Authenticate with gcloud for release registry 🎫
-      uses: "google-github-actions/auth@ef5d53e30bbcd8d0836f4288f5e50ff3e086997d"
-      with:
-        token_format: 'access_token'
-        credentials_json: "${{ secrets.RASA_RELEASES_READ }}"
-
-    - name: Configure OAuth token for poetry
-      run: |
-        poetry config http-basic.rasa-plus oauth2accesstoken $(gcloud auth print-access-token)
-
-    - name: Install Dependencies 📦
-      run: |
-        make install
-
-    - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a
-      with:
-        name: trained-model
-        path: models/
-
-    - name: Init LLM Cache
-      id: cache-llm
-      uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8
-      with:
-        path: .rasa
-        key: rasa-llm-cache
-
-    - name: Run action server
-      env:
-        OPENAI_API_KEY: ${{secrets.AZURE_OPENAI_API_KEY}}
-        RASA_PRO_LICENSE: ${{secrets.RASA_PRO_LICENSE}}
-        RASA_DUCKLING_HTTP_URL: ${{secrets.DUCKLING_URL}}
-        RASA_PRO_BETA_INTENTLESS: true
-      run: |
-        make actions &
-
-    - name: Run e2e happy_path tests
-      env:
-        OPENAI_API_KEY: ${{secrets.AZURE_OPENAI_API_KEY}}
-        RASA_PRO_LICENSE: ${{secrets.RASA_PRO_LICENSE}}
-        RASA_DUCKLING_HTTP_URL: ${{secrets.DUCKLING_URL}}
-        RASA_PRO_BETA_INTENTLESS: true
-      run: |
-        poetry run rasa test e2e e2e_tests/passing/happy_path
-
-    - name: Run e2e chitchat tests
-      env:
-        OPENAI_API_KEY: ${{secrets.AZURE_OPENAI_API_KEY}}
-        RASA_PRO_LICENSE: ${{secrets.RASA_PRO_LICENSE}}
-        RASA_DUCKLING_HTTP_URL: ${{secrets.DUCKLING_URL}}
-        RASA_PRO_BETA_INTENTLESS: true
-      run: |
-        poetry run rasa test e2e e2e_tests/passing/chitchat
+      - name: Run action server
+        env:
+          OPENAI_API_KEY: ${{secrets.AZURE_OPENAI_API_KEY}}
+          RASA_PRO_LICENSE: ${{secrets.RASA_PRO_LICENSE}}
+          RASA_DUCKLING_HTTP_URL: ${{secrets.DUCKLING_URL}}
+          RASA_PRO_BETA_INTENTLESS: true
+        run: |
+          make actions &
+
+      - name: Run e2e happy_path tests
+        env:
+          OPENAI_API_KEY: ${{secrets.AZURE_OPENAI_API_KEY}}
+          RASA_PRO_LICENSE: ${{secrets.RASA_PRO_LICENSE}}
+          RASA_DUCKLING_HTTP_URL: ${{secrets.DUCKLING_URL}}
+          RASA_PRO_BETA_INTENTLESS: true
+        run: |
+          poetry run rasa test e2e e2e_tests/passing >> multistep_report.md
+        
+      - name: Persist report
+        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce
+        with:
+          name: multistepprompt_report
+          path: multistep_report.md
+
+#    - name: Run e2e chitchat tests
+#      env:
+#        OPENAI_API_KEY: ${{secrets.AZURE_OPENAI_API_KEY}}
+#        RASA_PRO_LICENSE: ${{secrets.RASA_PRO_LICENSE}}
+#        RASA_DUCKLING_HTTP_URL: ${{secrets.DUCKLING_URL}}
+#        RASA_PRO_BETA_INTENTLESS: true
+#      run: |
+#        poetry run rasa test e2e e2e_tests/passing/chitchat
 

config.yml

@@ -41,6 +41,13 @@ pipeline:
       openai_api_base: "https://devtribe-testing-canada-east.openai.azure.com/"
       openai_api_type: azure
       openai_api_version: "2024-02-01"
+    llm:
+      model_name: "gpt-35-turbo"
+      engine: "test-gpt-35-turbo"
+      api_type: "azure"
+      api_version: "2024-02-01"
+      openai_api_base: "https://devtribe-testing-canada-east.openai.azure.com/"
+      request_timeout: 7
 #- name: SingleStepLLMCommandGenerator
 #  llm:
 #    model_name: "gpt-35-turbo"