Merge pull request #255 from marlenezw/new_image_evals

adding changes for image evals.

infra/main.bicepparam

@@ -4,19 +4,18 @@ param environmentName = readEnvironmentVariable('AZURE_ENV_NAME', 'MY_ENV')
 param location = readEnvironmentVariable('AZURE_LOCATION', 'swedencentral')
 param principalId = readEnvironmentVariable('AZURE_PRINCIPAL_ID', '')
 param resourceGroupName = readEnvironmentVariable('AZURE_RESOURCE_GROUP', '')
-param openAILocation = readEnvironmentVariable('AZURE_OPENAI_LOCATION','swedencentral')
 
-param aiHubName = readEnvironmentVariable('AZUREAI_HUB_NAME', 'ignitecreativehub')
-param aiProjectName = readEnvironmentVariable('AZUREAI_PROJECT_NAME', 'ignitecreativeproj')
+param aiHubName = readEnvironmentVariable('AZUREAI_HUB_NAME', '')
+param aiProjectName = readEnvironmentVariable('AZUREAI_PROJECT_NAME', '')
 param endpointName = readEnvironmentVariable('AZUREAI_ENDPOINT_NAME', '')
 
 param openAiName = readEnvironmentVariable('AZURE_OPENAI_NAME', '')
-param searchServiceName = readEnvironmentVariable('AZURE_SEARCH_SERVICE_NAME', 'aisearchcreative')
+param searchServiceName = readEnvironmentVariable('AZURE_SEARCH_SERVICE_NAME', '')
 
-param applicationInsightsName = readEnvironmentVariable('AZURE_APPLICATION_INSIGHTS_NAME', 'ignitecreativeinsights')
-param keyVaultName = readEnvironmentVariable('AZURE_KEYVAULT_NAME', 'creativekey')
-param storageAccountName = readEnvironmentVariable('AZURE_STORAGE_ACCOUNT_NAME', 'creativestorageignite')
-param logAnalyticsWorkspaceName = readEnvironmentVariable('AZURE_LOG_ANALYTICS_WORKSPACE_NAME', 'logcreative')
+param applicationInsightsName = readEnvironmentVariable('AZURE_APPLICATION_INSIGHTS_NAME', '')
+param keyVaultName = readEnvironmentVariable('AZURE_KEYVAULT_NAME', '')
+param storageAccountName = readEnvironmentVariable('AZURE_STORAGE_ACCOUNT_NAME', '')
+param logAnalyticsWorkspaceName = readEnvironmentVariable('AZURE_LOG_ANALYTICS_WORKSPACE_NAME', '')
 
 param useContainerRegistry = bool(readEnvironmentVariable('USE_CONTAINER_REGISTRY', 'true'))
 param useApplicationInsights = bool(readEnvironmentVariable('USE_APPLICATION_INSIGHTS', 'true'))

src/api/agents/product/product.prompty

@@ -7,7 +7,7 @@ model:
   api: chat
   configuration:
     type: azure_openai
-    azure_deployment: gpt-35-turbo
+    azure_deployment: gpt-4-evals
     api_version: 2023-07-01-preview
   parameters:
     max_tokens: 1500

src/api/agents/product/product.py

@@ -21,7 +21,6 @@
 
 AZURE_OPENAI_ENDPOINT = os.getenv("AZURE_OPENAI_ENDPOINT")
 AZURE_OPENAI_VERSION = "2023-07-01-preview"
-AZURE_OPENAI_DEPLOYMENT = "text-embedding-ada-002"
 AZURE_AI_SEARCH_ENDPOINT = os.getenv("AI_SEARCH_ENDPOINT")
 AZURE_AI_SEARCH_INDEX = "contoso-products"
 

src/api/agents/writer/writer.prompty

@@ -7,7 +7,7 @@ model:
   api: chat
   configuration:
     type: azure_openai
-    azure_deployment: gpt-4
+    azure_deployment: gpt-4-evals
     api_version: 2023-07-01-preview
   parameters:
     max_tokens: 2000

src/api/evaluate/data/example_urls.txt

@@ -0,0 +1,8 @@
+
+"http://www.firstaidforfree.com/wp-content/uploads/2017/01/First-Aid-Kit.jpg",
+"https://img.freepik.com/premium-vector/first-aid-reanimation-cpr-training-heart-emergency-revival-ambulance-medical-help-procedures_53562-12289.jpg",
+"https://www.themanual.com/wp-content/uploads/sites/9/2019/04/wilderness-first-aid-bleeding.jpg",
+"https://townsquare.media/site/782/files/2020/06/Elmer-and-Bugs-featured.jpg",
+"http://4.bp.blogspot.com/-fS4RU14NK1g/UTYSopvj7cI/AAAAAAAAIrU/vMqKCoS9wJc/s1600/a-wild-hare-first-appearance-of-bugs-bunny.jpg",
+"https://i.imgflip.com/9a1vlj.jpg",
+

src/api/evaluate/evaluate.py

@@ -9,6 +9,8 @@
 from azure.identity import DefaultAzureCredential
 from azure.ai.project import AIProjectClient
 from azure.ai.project.models import Evaluation, Dataset, EvaluatorConfiguration, ConnectionType
+from openai import AzureOpenAI
+from azure.identity import DefaultAzureCredential, get_bearer_token_provider
 
 from dotenv import load_dotenv
 
@@ -24,9 +26,11 @@ def evaluate_remote(data_path):
     # At the moment, it should be in the format "<HostName>;<AzureSubscriptionId>;<ResourceGroup>;<HubName>"
     # Customer needs to login to Azure subscription via Azure CLI and set the environment variables
 
+    ai_project_conn_str = os.getenv("AZURE_LOCATION")+".api.azureml.ms;"+os.getenv("AZURE_SUBSCRIPTION_ID")+";"+os.getenv("AZURE_RESOURCE_GROUP")+";"+os.getenv("AZURE_AI_PROJECT_NAME")
+
     project_client = AIProjectClient.from_connection_string(
         credential=DefaultAzureCredential(),
-        conn_str=os.getenv("PROJECT_CONNECTION_STRING"),
+        conn_str=ai_project_conn_str,
     )
 
     data_id = project_client.upload_file(data_path)
@@ -73,13 +77,13 @@ def evaluate_remote(data_path):
                     "azure_ai_project": project_client.scope
                 },
             ),
-            "hateunfairness": EvaluatorConfiguration(
+            "hate_unfairness": EvaluatorConfiguration(
                 id="azureml://registries/azureml/models/Hate-and-Unfairness-Evaluator/versions/4",
                 init_params={
                     "azure_ai_project": project_client.scope
                 },
             ),
-            "selfharm": EvaluatorConfiguration(
+            "self_harm": EvaluatorConfiguration(
                 id="azureml://registries/azureml/models/Self-Harm-Related-Content-Evaluator/versions/3",
                 init_params={
                     "azure_ai_project": project_client.scope
@@ -158,7 +162,7 @@ def evaluate_orchestrator(model_config, project_scope,  data_path):
     metrics = {key: [value] for key, value in eval_results['metrics'].items()}
     results_df = pd.DataFrame.from_dict(metrics)
     results_df_gpt_evals = results_df[['relevance.gpt_relevance', 'fluency.gpt_fluency', 'coherence.gpt_coherence','groundedness.gpt_groundedness']]
-    results_df_content_safety = results_df[['violence.violence_defect_rate', 'self-harm.self_harm_defect_rate', 'hate-unfairness.hate_unfairness_defect_rate','sexual.sexual_defect_rate']]
+    results_df_content_safety = results_df[['violence.violence_defect_rate', 'self_harm.self_harm_defect_rate', 'hate_unfairness.hate_unfairness_defect_rate','sexual.sexual_defect_rate']]
 
     mean_df = results_df_gpt_evals.mean()
     print("\nAverage scores:")
@@ -184,34 +188,106 @@ def evaluate_image(project_scope,  image_path):
     import pathlib 
     import base64
 
-    with pathlib.Path(image_path).open("rb") as image_file:
-        encoded_image = base64.b64encode(image_file.read()).decode("utf-8")
+    import validators
+
 
-    conversation = {"conversation":{
-                "messages": [
-        {
-            "role": "system",
-            "content": [
-                {"type": "text", "text": "This is a nature boardwalk at the University of Wisconsin-Madison."}
-            ],
-        },
-        {
-            "role": "user",
-            "content": [
-                {"type": "text", "text": "Can you describe this image?"},
-                {"type": "image_url", "image_url": {"url": f"data:image/jpg;base64,{encoded_image}"}},
-            ],
-        },
-    ]
-    }
-    }
+    if validators.url(image_path):
+        url_path = image_path
+    else:
+        #encode an image or you can add an image file from a url
+        with pathlib.Path(image_path).open("rb") as image_file:
+            encoded_image = base64.b64encode(image_file.read()).decode("utf-8")
+
+            url_path = f"data:image/jpg;base64,{encoded_image}"
+
+    token_provider = get_bearer_token_provider(
+        DefaultAzureCredential(), "https://cognitiveservices.azure.com/.default"
+    )
+
+    client = AzureOpenAI(
+        azure_endpoint = f"{os.getenv('AZURE_OPENAI_ENDPOINT')}", 
+        api_version="2023-07-01-preview",
+        azure_ad_token_provider=token_provider
+    )
+
+    sys_message = "You are an AI assistant that describes images in details."
+
+    messages = []
+
+    print(f"\n===== URL : [{url_path}]")
+    print(f"\n===== Calling Open AI to describe image and retrieve response")
+    completion = client.chat.completions.create(
+    model="gpt-4-evals",
+    messages= [
+                    {
+                        "role": "system", 
+                        "content": [
+                            {"type": "text", "text": sys_message}
+                        ]
+                    },
+                    {
+                        "role": "user",
+                        "content": [
+                            {"type": "text", "text": "Can you describe this image?"},
+                            {"type": "image_url", "image_url": {"url": url_path}},
+                        ],
+                    },
+                ],
+        )
+
+    message = [
+            {
+                "role": "system",
+                "content": [
+                    {"type": "text", "text": sys_message}
+                ],
+            },
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": "Can you describe this image?"},
+                    {"type": "image_url", "image_url": {"url": url_path}},
+                ],
+            },
+            {
+                "role": "assistant",
+                "content": [
+                    {"type": "text", "text": completion.choices[0].message.content},
+                ],
+            },
+        ]
+
+    messages.append(message)
+
+    eval_results = image_evaluator(messages=messages)
+
+    import pandas as pd
 
-    result = image_evaluator(conversation=conversation)
+    print("Image Evaluation summary:\n")
+    print("View in Azure AI Studio at: " + str(eval_results['studio_url']))
+    metrics = {key: [value] for key, value in eval_results['metrics'].items()}
+
+    results_df = pd.DataFrame.from_dict(metrics)
 
-    return result 
+    result_keys = [*metrics.keys()]
+
+    results_df_gpt_evals = results_df[result_keys]
+
+    mean_df = results_df_gpt_evals.mean()
+    print("\nAverage scores:")
+    print(mean_df)
+
+    results_df.to_markdown(folder + '/image_eval_results.md')
+    with open(folder + '/image_eval_results.md', 'a') as file:
+        file.write("\n\nAverages scores:\n\n")
+    mean_df.to_markdown(folder + '/image_eval_results.md', 'a')
+
+    with jsonlines.open(folder + '/image_eval_results.jsonl', 'w') as writer:
+        writer.write(eval_results)
+
+    return eval_results
 
 
-
 
 if __name__ == "__main__":
     import time
@@ -239,14 +315,13 @@ def evaluate_image(project_scope,  image_path):
     eval_result = evaluate_orchestrator(model_config, project_scope, data_path=folder +"/eval_inputs.jsonl")
     evaluate_remote(data_path=folder +"/eval_data.jsonl")
 
-    parent = pathlib.Path(__file__).parent.resolve()
-    path = os.path.join(parent, "data")
-    image_path = os.path.join(path, "image1.jpg")
+    # parent = pathlib.Path(__file__).parent.resolve()
+    # path = os.path.join(parent, "data")
+    # image_path = os.path.join(path, "image1.jpg")
 
-    eval_image_result = evaluate_image(project_scope, image_path)
-    image_results = eval_image_result['rows'][0].pop('inputs.conversation')
+    image_path = "https://i.imgflip.com/9a1vlj.jpg"
 
-    pprint(eval_image_result)
+    eval_image_result = evaluate_image(project_scope, image_path)
 
     end=time.time()
     print(f"Finished evaluate in {end - start}s")

src/api/evaluate/evaluators.py

@@ -8,6 +8,7 @@
 from azure.ai.evaluation import ViolenceEvaluator, HateUnfairnessEvaluator, SelfHarmEvaluator, SexualEvaluator
 from azure.ai.evaluation import evaluate
 from azure.ai.evaluation import ViolenceMultimodalEvaluator, SelfHarmMultimodalEvaluator, HateUnfairnessMultimodalEvaluator, SexualMultimodalEvaluator
+from azure.ai.evaluation import ContentSafetyMultimodalEvaluator, ProtectedMaterialMultimodalEvaluator
 from azure.identity import DefaultAzureCredential
 
 
@@ -41,8 +42,8 @@ def __init__(self, model_config, project_scope):
             "coherence": CoherenceEvaluator(model_config),
             "groundedness": GroundednessEvaluator(model_config),
             "violence": ViolenceEvaluator(azure_ai_project=project_scope, credential=DefaultAzureCredential()),
-            "hate-unfairness": HateUnfairnessEvaluator(azure_ai_project=project_scope, credential=DefaultAzureCredential()),
-            "self-harm": SelfHarmEvaluator(azure_ai_project=project_scope, credential=DefaultAzureCredential()),
+            "hate_unfairness": HateUnfairnessEvaluator(azure_ai_project=project_scope, credential=DefaultAzureCredential()),
+            "self_harm": SelfHarmEvaluator(azure_ai_project=project_scope, credential=DefaultAzureCredential()),
             "sexual": SexualEvaluator(azure_ai_project=project_scope, credential=DefaultAzureCredential()),
             "friendliness": FriendlinessEvaluator(),
         }
@@ -91,14 +92,14 @@ def __call__(self, *, data_path, **kwargs):
                         "query": "${data.query}",
                     },
                 },
-                "self-harm": {
+                "self_harm": {
                     "column_mapping": {
                         "response": "${data.response}",
                         "context": "${data.context}",
                         "query": "${data.query}",
                     },
                 },
-                "hate-unfairness": {
+                "hate_unfairness": {
                     "column_mapping": {
                         "response": "${data.response}",
                         "context": "${data.context}",
@@ -127,36 +128,46 @@ def __call__(self, *, data_path, **kwargs):
 class ImageEvaluator:
     def __init__(self, project_scope):
         self.evaluators = {
-            "violence": ViolenceMultimodalEvaluator(credential=DefaultAzureCredential(), azure_ai_project=project_scope),
-            "sexual": SexualMultimodalEvaluator(credential=DefaultAzureCredential(), azure_ai_project=project_scope),
-            "self-harm": SelfHarmMultimodalEvaluator(credential=DefaultAzureCredential(), azure_ai_project=project_scope),
-            "hate-unfairness": HateUnfairnessMultimodalEvaluator(credential=DefaultAzureCredential(), azure_ai_project=project_scope),
+             "content_safety": ContentSafetyMultimodalEvaluator(credential=DefaultAzureCredential(), azure_ai_project=project_scope), 
+            "protected_material": ProtectedMaterialMultimodalEvaluator(credential=DefaultAzureCredential(), azure_ai_project=project_scope)
         }
         self.project_scope = project_scope
 
 
-    def __call__(self, *, conversation, **kwargs): 
+    def __call__(self, *, messages, **kwargs): 
         import uuid
+        import pathlib
         from pprint import pprint
+        import pandas as pd
 
-        jsonl_path = "datafile.jsonl"
+        file_name="dataset_images.jsonl"    
+        parent = pathlib.Path(__file__).parent.resolve()
+        path = os.path.join(parent, "data")
+        datafile_jsonl_path = os.path.join(path, file_name)
+        with open(datafile_jsonl_path, "w") as outfile:
+            for message in messages:
+                conversation = {"conversation": { "messages" : message}}
+                json_line = json.dumps(conversation)
+                outfile.write(json_line + "\n")
 
-        # Write conversation to JSONL file
-        with open(jsonl_path, "w") as jsonl_file:
-            json.dump(conversation, jsonl_file)
-            jsonl_file.write("\n")
+        print("\n===== Reading Data File =======")
 
+        data_path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data")
+        file_path = os.path.join(data_path, file_name)
+        input_data = pd.read_json(file_path, lines=True)
+        pprint(input_data)
+
+
+        print("\n===== Calling Evaluate API - Content Safety & Protected Material Evaluator for multi-modal =======")
         output = {}
         result = evaluate(
             evaluation_name=f"evaluate-api-multi-modal-eval-dataset-{str(uuid.uuid4())}",
-            data=jsonl_path,
+            data=file_path,
             evaluators=self.evaluators,
             azure_ai_project=self.project_scope,
             evaluator_config={
-                "violence": {"conversation": "${data.conversation}"},
-                "sexual": {"conversation": "${data.conversation}"},
-                "self-harm": {"conversation": "${data.conversation}"},
-                "hate-unfairness": {"conversation": "${data.conversation}"},
+                "content_safety": {"conversation": "${data.conversation}"}, 
+                "protected_material": {"conversation": "${data.conversation}"} 
             }
         )
 
@@ -207,17 +218,17 @@ def evaluate_article_in_background(research_context, product_context, assignment
 
     evaluate_article(eval_data, trace_context)
 
-def evaluate_image(conversation, trace_context):
-    tracer = trace.get_tracer(__name__)
-    with tracer.start_as_current_span("run_image_evaluators", context=trace_context) as span:
-        span.set_attribute("inputs", json.dumps(conversation))
+def evaluate_image(messages):
+    # tracer = trace.get_tracer(__name__)
+    # with tracer.start_as_current_span("run_image_evaluators", context=trace_context) as span:
+    #     span.set_attribute("inputs", json.dumps(conversation))
     project_scope = {
         "subscription_id": os.environ["AZURE_SUBSCRIPTION_ID"],   
         "resource_group_name": os.environ["AZURE_RESOURCE_GROUP"],
         "project_name": os.environ["AZURE_AI_PROJECT_NAME"],        
     }
     evaluator = ImageEvaluator(project_scope)
-    results = evaluator(conversation)
+    results = evaluator(messages)
     resultsJson = json.dumps(results)
 
     print("results: ", resultsJson)