Add gpt4 for evaluation in infra, fix article params and prompty, add…

… evaluation logic (#80)

* fix logging when running local

* Evaluate uses gpt4

* add gpt4 eval model to terraform

* check for related searches

* Changes from CI/CD branch (#78)

* troubleshoot bing endpoint/key

* fix typo

* fix typo

* run evaluate on push

* add some print debugging

* more logging

* increase debug logging level

* use gpt-4 for evals

* update max tokens

* put max tokens back to 512

* increase token limit

* updates

* improve article length and token size

* add request to eval result

* revert token change

* undo eval change

* update max tokens

* Improve evaluate print out

* update evaluate

* update evaluate

* update evaluate

* decrease article length, increase tokens

* revert token changes

* increase writer token limit, decrease word count

---------

Co-authored-by: Dan Taylor <[email protected]>

.github/workflows/evaluate.yml

@@ -2,11 +2,11 @@ name: Evaluate
 
 on:
   workflow_dispatch:
-  workflow_run:
-    workflows: ["azd provision and deploy"]
-    branches: [main]
-    types: 
-      - completed
+  push:
+    # Run when commits are pushed to mainline branch (main or master)
+    # Set this to the mainline branch you are using
+    branches:
+      - main
 
 # Set up permissions for deploying with secretless Azure federated credentials
 # https://learn.microsoft.com/en-us/azure/developer/github/connect-from-azure?tabs=azure-portal%2Clinux#set-up-azure-login-with-openid-connect-authentication
@@ -24,6 +24,7 @@ jobs:
       AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }}
       AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }}
       AZURE_OPENAI_API_VERSION: ${{ vars.AZURE_OPENAI_API_VERSION }}
+      AZURE_OPENAI_GPT4_EVAL_DEPLOYMENT: ${{ vars.AZURE_OPENAI_GPT4_EVAL_DEPLOYMENT }}
       AZURE_OPENAI_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME }}
       AZURE_OPENAI_35_TURBO_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_35_TURBO_DEPLOYMENT_NAME }}
       AZURE_SEARCH_ENDPOINT: ${{ vars.AZURE_SEARCH_ENDPOINT }}
@@ -61,7 +62,7 @@ jobs:
           inlineScript: |
             az account set --subscription ${{env.AZURE_SUBSCRIPTION_ID}}
 
-      - name: evaluate chat data
+      - name: evaluate orchestrator
         working-directory: ./src/api
         run: |
           python -m api.evaluate.evaluate

azure.yaml

@@ -60,6 +60,7 @@ infra:
   provider: terraform
 pipeline:
   variables:
+    - AZURE_OPENAI_GPT4_EVAL_DEPLOYMENT
     - AZURE_OPENAI_35_TURBO_DEPLOYMENT_NAME
     - AZURE_OPENAI_35_TURBO_MODEL_NAME
     - AZURE_OPENAI_API_VERSION

data/create-azure-search.py

@@ -0,0 +1,209 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Generating your product search index
+# Thereis notebook is designed to automatically create the product search index for you. It uses the [product catalog](products.csv) file to create the index. In order to do so it needs names ane keys for the following services:
+# 
+# - Azure Search Service
+# - Azure OpenAI Service
+# 
+# You can find the names and keys in the Azure Portal. These need to be entered in a `.env` file in the root of this repository. The `.env` file is not checked in to source control. You can use the [`.env.sample`](../../.env.sample) file as a template.
+
+# In[1]:
+
+
+import os
+import pandas as pd
+from azure.identity import DefaultAzureCredential, get_bearer_token_provider
+from azure.search.documents import SearchClient
+from azure.search.documents.indexes import SearchIndexClient
+from azure.search.documents.indexes.models import (
+    HnswParameters,
+    HnswAlgorithmConfiguration,
+    SemanticPrioritizedFields,
+    SearchableField,
+    SearchField,
+    SearchFieldDataType,
+    SearchIndex,
+    SemanticSearch,
+    SemanticConfiguration,
+    SemanticField,
+    SimpleField,
+    VectorSearch,
+    VectorSearchAlgorithmKind,
+    VectorSearchAlgorithmMetric,
+    ExhaustiveKnnAlgorithmConfiguration,
+    ExhaustiveKnnParameters,
+    VectorSearchProfile,
+)
+from typing import List, Dict
+from openai import AzureOpenAI
+from dotenv import load_dotenv
+
+from pathlib import Path
+
+load_dotenv()
+
+
+# In[2]:
+
+
+def delete_index(search_index_client: SearchIndexClient, search_index: str):
+    print(f"deleting index {search_index}")
+    search_index_client.delete_index(search_index)
+
+
+# In[3]:
+
+
+def create_index_definition(name: str) -> SearchIndex:
+    """
+    Returns an Azure AI Search index with the given name.
+    """
+    # The fields we want to index. The "embedding" field is a vector field that will
+    # be used for vector search.
+    fields = [
+        SimpleField(name="id", type=SearchFieldDataType.String, key=True),
+        SearchableField(name="content", type=SearchFieldDataType.String),
+        SimpleField(name="filepath", type=SearchFieldDataType.String),
+        SearchableField(name="title", type=SearchFieldDataType.String),
+        SimpleField(name="url", type=SearchFieldDataType.String),
+        SearchField(
+            name="contentVector",
+            type=SearchFieldDataType.Collection(SearchFieldDataType.Single),
+            searchable=True,
+            # Size of the vector created by the text-embedding-ada-002 model.
+            vector_search_dimensions=1536,
+            vector_search_profile_name="myHnswProfile",
+        ),
+    ]
+
+    # The "content" field should be prioritized for semantic ranking.
+    semantic_config = SemanticConfiguration(
+        name="default",
+        prioritized_fields=SemanticPrioritizedFields(
+            title_field=SemanticField(field_name="title"),
+            keywords_fields=[],
+            content_fields=[SemanticField(field_name="content")],
+        ),
+    )
+
+    # For vector search, we want to use the HNSW (Hierarchical Navigable Small World)
+    # algorithm (a type of approximate nearest neighbor search algorithm) with cosine
+    # distance.
+    vector_search = VectorSearch(
+        algorithms=[
+            HnswAlgorithmConfiguration(
+                name="myHnsw",
+                kind=VectorSearchAlgorithmKind.HNSW,
+                parameters=HnswParameters(
+                    m=4,
+                    ef_construction=400,
+                    ef_search=500,
+                    metric=VectorSearchAlgorithmMetric.COSINE,
+                ),
+            ),
+            ExhaustiveKnnAlgorithmConfiguration(
+                name="myExhaustiveKnn",
+                kind=VectorSearchAlgorithmKind.EXHAUSTIVE_KNN,
+                parameters=ExhaustiveKnnParameters(
+                    metric=VectorSearchAlgorithmMetric.COSINE
+                ),
+            ),
+        ],
+        profiles=[
+            VectorSearchProfile(
+                name="myHnswProfile",
+                algorithm_configuration_name="myHnsw",
+            ),
+            VectorSearchProfile(
+                name="myExhaustiveKnnProfile",
+                algorithm_configuration_name="myExhaustiveKnn",
+            ),
+        ],
+    )
+
+    # Create the semantic settings with the configuration
+    semantic_search = SemanticSearch(configurations=[semantic_config])
+
+    # Create the search index.
+    index = SearchIndex(
+        name=name,
+        fields=fields,
+        semantic_search=semantic_search,
+        vector_search=vector_search,
+    )
+
+    return index
+
+
+# In[4]:
+
+
+def gen_products(
+    path: str,
+) -> List[Dict[str, any]]:
+    openai_service_endoint = os.environ["AZURE_OPENAI_ENDPOINT"]
+    openai_deployment = "text-embedding-ada-002"
+    # openai.Embedding.create() -> client.embeddings.create()
+    azure_credential = DefaultAzureCredential()
+    token_provider = get_bearer_token_provider(azure_credential,"https://cognitiveservices.azure.com/.default")
+    client = AzureOpenAI(
+        api_version="2023-07-01-preview",
+        azure_endpoint=openai_service_endoint,
+        azure_deployment=openai_deployment,
+        azure_ad_token_provider=token_provider
+    )
+
+    products = pd.read_csv(path)
+    items = []
+    for product in products.to_dict("records"):
+        content = product["description"]
+        id = str(product["id"])
+        title = product["name"]
+        url = f"/products/{title.lower().replace(' ', '-')}"
+        emb = client.embeddings.create(input=content, model=openai_deployment)
+        rec = {
+            "id": id,
+            "content": content,
+            "filepath": f"{title.lower().replace(' ', '-')}",
+            "title": title,
+            "url": url,
+            "contentVector": emb.data[0].embedding,
+        }
+        items.append(rec)
+
+    return items
+
+
+# In[5]:
+
+
+aisearch_endpoint = os.environ["AZURE_SEARCH_ENDPOINT"]
+index_name = "contoso-products"
+
+search_index_client = SearchIndexClient(
+    aisearch_endpoint, DefaultAzureCredential()
+)
+
+delete_index(search_index_client, index_name)
+index = create_index_definition(index_name)
+print(f"creating index {index_name}")
+search_index_client.create_or_update_index(index)
+print(f"index {index_name} created")
+
+
+# In[6]:
+
+
+print(f"indexing documents")
+docs = gen_products("products.csv")
+# Upload our data to the index.
+search_client = SearchClient(
+    endpoint=aisearch_endpoint,
+    index_name=index_name,
+    credential=DefaultAzureCredential(),
+)
+print(f"uploading {len(docs)} documents to index {index_name}")
+ds = search_client.upload_documents(docs)
+

infra/manifests/api/config.tmpl.yaml

@@ -4,6 +4,7 @@ metadata:
   name: api-config
 data:
   AZURE_OPENAI_ENDPOINT: {{.Env.AZURE_OPENAI_ENDPOINT}}
+  AZURE_OPENAI_GPT4_EVAL_DEPLOYMENT: {{.Env.AZURE_OPENAI_GPT4_EVAL_DEPLOYMENT}}
   AZURE_OPENAI_35_TURBO_DEPLOYMENT_NAME: {{.Env.AZURE_OPENAI_35_TURBO_DEPLOYMENT_NAME}}
   AZURE_OPENAI_35_TURBO_MODEL_NAME: {{.Env.AZURE_OPENAI_35_TURBO_MODEL_NAME}}
   AZURE_OPENAI_API_VERSION: {{.Env.AZURE_OPENAI_API_VERSION}}

infra/manifests/api/deployment.tmpl.yaml

@@ -34,6 +34,8 @@ spec:
               value: {{.Env.AZURE_OPENAI_DEPLOYMENT_NAME}}
             - name: AZURE_OPENAI_35_TURBO_DEPLOYMENT_NAME
               value: {{.Env.AZURE_OPENAI_35_TURBO_DEPLOYMENT_NAME}}
+            - name: AZURE_OPENAI_GPT4_EVAL_DEPLOYMENT
+              value: {{.Env.AZURE_OPENAI_GPT4_EVAL_DEPLOYMENT}}
             - name: AZURE_OPENAI_35_TURBO_DEPLOYMENT_NAME
               value: {{.Env.AZURE_OPENAI_35_TURBO_DEPLOYMENT_NAME}}
             - name: AZURE_OPENAI_ENDPOINT

infra/openai.tf

@@ -47,6 +47,22 @@ resource "azurerm_cognitive_deployment" "gpt35_deployment" {
   }
 }
 
+resource "azurerm_cognitive_deployment" "gpt4_deployment" {
+  name                 = var.openai_4_eval_deployment_name
+  cognitive_account_id = azurerm_cognitive_account.cog.id
+
+  model {
+    format  = "OpenAI"
+    name    = var.openai_4_eval_model_name
+    version = var.openai_4_eval_model_version
+  }
+
+  scale {
+    type     = "Standard"
+    capacity = var.openai_4_eval_model_capacity
+  }
+}
+
 resource "azurerm_cognitive_deployment" "embedding_deployment" {
   name                 = var.openai_embedding_model_name
   cognitive_account_id = azurerm_cognitive_account.cog.id

infra/outputs.tf

@@ -22,6 +22,15 @@ output "AZURE_OPENAI_35_TURBO_DEPLOYMENT_NAME" {
   value = var.openai_35_turbo_model_name
 }
 
+output "AZURE_OPENAI_GPT4_EVAL_DEPLOYMENT" {
+  value = var.openai_4_eval_deployment_name
+}
+
+output "AZURE_OPENAI_4_EVAL_MODEL_VERSION" {
+  value = var.openai_4_eval_model_version
+}
+
+
 output "AZURE_OPENAI_35_TURBO_MODEL_NAME" {
   value = var.openai_35_turbo_model_name
 }

infra/variables.tf

@@ -43,6 +43,32 @@ variable "openai_35_turbo_model_name" {
   default     = "gpt-35-turbo"
 }
 
+variable "openai_4_eval_deployment_name" {
+  description = "value of azure openai model name"
+  type        = string
+  default     = "gpt-4-eval"
+}
+
+variable "openai_4_eval_model_name" {
+  description = "value of azure openai model name"
+  type        = string
+  default     = "gpt-4"
+}
+
+
+variable "openai_4_eval_model_version" {
+  description = "value of azure openai model name"
+  type        = string
+  default     = "0613"
+}
+
+variable "openai_4_eval_model_capacity" {
+  description = "value of azure openai model capacity"
+  type        = number
+  default     = 20
+}
+
+
 variable "openai_35_turbo_model_version" {
   description = "value of azure openai model version"
   type        = string

src/api/api/agents/orchestrator.py

@@ -142,11 +142,16 @@ def write_article(request, instructions, evaluate=False):
     # Log final editor response
     log_output("Final editor response: %s", json.dumps(editor_response, indent=2))
 
-if __name__ == "__main__":
-    from api.logging import init_logging
-
-    init_logging()
+@trace
+def test_write_article():
     context = "Can you find the latest camping trends and what folks are doing in the winter?"
     instructions = "Can you find the relevant information needed and good places to visit"
     for result in write_article(context, instructions, evaluate=True):
         print(*result)
+
+if __name__ == "__main__":
+    from api.logging import init_logging
+
+    init_logging()
+    test_write_article()
+

src/api/api/agents/researcher/researcher.py

@@ -7,17 +7,17 @@
 from promptflow.tracing import trace
 from promptflow.core import Prompty, AzureOpenAIModelConfiguration
 
-from api.logging import log_output
-
 from dotenv import load_dotenv
 from pathlib import Path
 
+import base64
+
 folder = Path(__file__).parent.absolute().as_posix()
 load_dotenv()
 
 #bing does not currently support managed identity
-BING_SEARCH_ENDPOINT = os.getenv("BING_SEARCH_ENDPOINT")
-BING_SEARCH_KEY = os.getenv("BING_SEARCH_KEY")
+BING_SEARCH_ENDPOINT = os.environ["BING_SEARCH_ENDPOINT"]
+BING_SEARCH_KEY = os.environ["BING_SEARCH_KEY"]
 BING_HEADERS = {"Ocp-Apim-Subscription-Key": BING_SEARCH_KEY}
 
 
@@ -42,11 +42,10 @@ def find_information(query, market="en-US"):
         {"url": a["url"], "name": a["name"], "description": a["snippet"]}
         for a in items["webPages"]["value"]
     ]
-
     # check if relatedsearches exists
     if "relatedSearches" not in items:
         return {"pages": pages, "related": []}
-
+    
     # else add related searching
     related = [a["text"] for a in items["relatedSearches"]["value"]]
     return {"pages": pages, "related": related}