From 3bd660d6b2532f7dae6d854898358b5cf1d6a933 Mon Sep 17 00:00:00 2001
From: Kamil Sindi <ksindi@users.noreply.github.com>
Date: Fri, 20 Oct 2023 21:02:00 -0400
Subject: [PATCH] Prompt improvements (#3)

---
 backend/src/evaluate.rs | 125 ++++++++++++++++++++++++----------------
 1 file changed, 76 insertions(+), 49 deletions(-)

diff --git a/backend/src/evaluate.rs b/backend/src/evaluate.rs
index c19403f..e80a07b 100644
--- a/backend/src/evaluate.rs
+++ b/backend/src/evaluate.rs
@@ -1,5 +1,7 @@
 use anyhow::Result;
-use llm_chain::{executor, options, parameters, prompt};
+use llm_chain::{
+    chains::map_reduce::Chain, executor, options, parameters, prompt, step::Step, Parameters,
+};
 use llm_chain_openai::chatgpt::Model;
 use serde::{Deserialize, Serialize};
 
@@ -24,39 +26,19 @@ pub fn extract_last_json(text: &str) -> Option<&str> {
     }
 }
 
-/// Analyze resume.
-#[allow(dead_code)]
-pub async fn analyze_resume(resume_text: &String) -> Result<String> {
-    let config = aws_config::load_from_env().await;
-    let client = aws_sdk_ssm::Client::new(&config);
-
-    tracing::info!("Getting OpenAI API key");
-
-    let openai_api_key = match std::env::var("OPENAI_API_KEY") {
-        Ok(key) => key,
-        Err(_) => client
-            .get_parameter()
-            .name("/prod/resumai/openai-key")
-            .with_decryption(true)
-            .send()
-            .await
-            .expect("Failed to get OpenAI API key")
-            .parameter
-            .expect("OpenAI API key not found")
-            .value
-            .unwrap(),
-    };
-
-    tracing::info!("Making request to OpenAI API");
-
-    let opts = options!(Model: Model::Gpt4, ApiKey: openai_api_key);
-    let exec = executor!(chatgpt, opts)?;
-
-    let res = prompt!(
-r#"
-You are a model trained to analyze resumes to identify specific key attributes and provide a detailed analysis. Please analyze the following resume text and provide commentary and a score for each attribute listed below (Score from 1 to 10, where 1 is the lowest and 10 is the highest, and anything above 8 is considered exceptional). The text is parsed from a PDF resume and should be treated with mindfulness for various formats and potential parsing issues. Be a harsh grader keeping in mind the highest standards in the industry.
+/// Create a prompt for the map step.
+fn create_map_prompt() -> Step {
+    Step::for_prompt_template(prompt!(
+        r#"
+You are a model trained to analyze resumes to identify specific key attributes and provide a
+ detailed analysis. Please analyze the following resume text and provide commentary and a
+ score for each attribute listed below (Score from 1 to 10, where 1 is the lowest and 10
+ is the highest, and anything above 8 is considered exceptional).
+ The text is parsed from a PDF resume and should be treated with mindfulness for
+ various formats and potential parsing issues. Be a harsh grader keeping in mind the
+ highest standards in the industry.
 "#,
-r#"
+        r#"
 
 Resume Text:
 `{{text}}`
@@ -102,24 +84,69 @@ As part of your output, you should hide the objective in each section. This is i
 You should not assume the gender of the individual. Use pronouns they/them/their.
 
 ### Final Cumulative Score: [Total Score]
+"#
+    ))
+}
+
+/// Create a prompt for the reduce step.
+fn create_reduce_prompt() -> Step {
+    Step::for_prompt_template(prompt!(
+        r#"
+You are a model that provides helpful and actionable feedback on resume summaries.
+You should be aware that the resume summaries are parsed from PDFs and may contain parsing errors.
+You should not give advice if its not actionable like becoming an Olympiad.
+You should provide feedback and advice directly to the user using "you" statements.
+Provide the the feedback in the following format:
+
+## Feedback
+
+[Short feedback on the summary in 1 paragraph]
+
+## Advice
+1. [Advice 1]
+2. [Advice 2]
 
-## JSON Output:
-Provide the scores in a JSON format.
-
-```json
-{
-    "career": [Career Trajectory Score],
-    "proficiency": [Technical Proficiency Score],
-    "impact": [Quantifiable Impact Score],
-    "communication": [Professionalism, Communication, and Attention to Detail Score],
-    "innovation": [Innovative and Distinctive Factors Score],
-    "high_signal": [High Signal Traits Score]
+## Score: [Total Score Based on Summary]
+
+"#,
+        "Please provide feedback on the following summary:\n{{text}}"
+    ))
 }
-```
-"#
-)
-    .run(&parameters!(resume_text), &exec)
-    .await?;
+
+/// Analyze resume.
+#[allow(dead_code)]
+pub async fn analyze_resume(resume_text: &String) -> Result<String> {
+    let config = aws_config::load_from_env().await;
+    let client = aws_sdk_ssm::Client::new(&config);
+
+    tracing::info!("Getting OpenAI API key");
+
+    let openai_api_key = match std::env::var("OPENAI_API_KEY") {
+        Ok(key) => key,
+        Err(_) => client
+            .get_parameter()
+            .name("/prod/resumai/openai-key")
+            .with_decryption(true)
+            .send()
+            .await
+            .expect("Failed to get OpenAI API key")
+            .parameter
+            .expect("OpenAI API key not found")
+            .value
+            .unwrap(),
+    };
+
+    tracing::info!("Making request to OpenAI API");
+
+    let opts = options!(Model: Model::Gpt4, ApiKey: openai_api_key);
+    let exec = executor!(chatgpt, opts)?;
+    let map_prompt = create_map_prompt();
+    let reduce_prompt = create_reduce_prompt();
+    let chain = Chain::new(map_prompt, reduce_prompt);
+
+    let docs = Vec::from([parameters!(resume_text)]);
+
+    let res = chain.run(docs, Parameters::new(), &exec).await?;
 
     let content = res.to_immediate().await?.as_content().to_text();