Skip to content

Commit

Permalink
Prompt improvements (#3)
Browse files Browse the repository at this point in the history
  • Loading branch information
ksindi authored Oct 21, 2023
1 parent 3c86718 commit 3bd660d
Showing 1 changed file with 76 additions and 49 deletions.
125 changes: 76 additions & 49 deletions backend/src/evaluate.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
use anyhow::Result;
use llm_chain::{executor, options, parameters, prompt};
use llm_chain::{
chains::map_reduce::Chain, executor, options, parameters, prompt, step::Step, Parameters,
};
use llm_chain_openai::chatgpt::Model;
use serde::{Deserialize, Serialize};

Expand All @@ -24,39 +26,19 @@ pub fn extract_last_json(text: &str) -> Option<&str> {
}
}

/// Analyze resume.
#[allow(dead_code)]
pub async fn analyze_resume(resume_text: &String) -> Result<String> {
let config = aws_config::load_from_env().await;
let client = aws_sdk_ssm::Client::new(&config);

tracing::info!("Getting OpenAI API key");

let openai_api_key = match std::env::var("OPENAI_API_KEY") {
Ok(key) => key,
Err(_) => client
.get_parameter()
.name("/prod/resumai/openai-key")
.with_decryption(true)
.send()
.await
.expect("Failed to get OpenAI API key")
.parameter
.expect("OpenAI API key not found")
.value
.unwrap(),
};

tracing::info!("Making request to OpenAI API");

let opts = options!(Model: Model::Gpt4, ApiKey: openai_api_key);
let exec = executor!(chatgpt, opts)?;

let res = prompt!(
r#"
You are a model trained to analyze resumes to identify specific key attributes and provide a detailed analysis. Please analyze the following resume text and provide commentary and a score for each attribute listed below (Score from 1 to 10, where 1 is the lowest and 10 is the highest, and anything above 8 is considered exceptional). The text is parsed from a PDF resume and should be treated with mindfulness for various formats and potential parsing issues. Be a harsh grader keeping in mind the highest standards in the industry.
/// Create a prompt for the map step.
fn create_map_prompt() -> Step {
Step::for_prompt_template(prompt!(
r#"
You are a model trained to analyze resumes to identify specific key attributes and provide a
detailed analysis. Please analyze the following resume text and provide commentary and a
score for each attribute listed below (Score from 1 to 10, where 1 is the lowest and 10
is the highest, and anything above 8 is considered exceptional).
The text is parsed from a PDF resume and should be treated with mindfulness for
various formats and potential parsing issues. Be a harsh grader keeping in mind the
highest standards in the industry.
"#,
r#"
r#"
Resume Text:
`{{text}}`
Expand Down Expand Up @@ -102,24 +84,69 @@ As part of your output, you should hide the objective in each section. This is i
You should not assume the gender of the individual. Use pronouns they/them/their.
### Final Cumulative Score: [Total Score]
"#
))
}

/// Create a prompt for the reduce step.
fn create_reduce_prompt() -> Step {
Step::for_prompt_template(prompt!(
r#"
You are a model that provides helpful and actionable feedback on resume summaries.
You should be aware that the resume summaries are parsed from PDFs and may contain parsing errors.
You should not give advice if its not actionable like becoming an Olympiad.
You should provide feedback and advice directly to the user using "you" statements.
Provide the the feedback in the following format:
## Feedback
[Short feedback on the summary in 1 paragraph]
## Advice
1. [Advice 1]
2. [Advice 2]
## JSON Output:
Provide the scores in a JSON format.
```json
{
"career": [Career Trajectory Score],
"proficiency": [Technical Proficiency Score],
"impact": [Quantifiable Impact Score],
"communication": [Professionalism, Communication, and Attention to Detail Score],
"innovation": [Innovative and Distinctive Factors Score],
"high_signal": [High Signal Traits Score]
## Score: [Total Score Based on Summary]
"#,
"Please provide feedback on the following summary:\n{{text}}"
))
}
```
"#
)
.run(&parameters!(resume_text), &exec)
.await?;

/// Analyze resume.
#[allow(dead_code)]
pub async fn analyze_resume(resume_text: &String) -> Result<String> {
let config = aws_config::load_from_env().await;
let client = aws_sdk_ssm::Client::new(&config);

tracing::info!("Getting OpenAI API key");

let openai_api_key = match std::env::var("OPENAI_API_KEY") {
Ok(key) => key,
Err(_) => client
.get_parameter()
.name("/prod/resumai/openai-key")
.with_decryption(true)
.send()
.await
.expect("Failed to get OpenAI API key")
.parameter
.expect("OpenAI API key not found")
.value
.unwrap(),
};

tracing::info!("Making request to OpenAI API");

let opts = options!(Model: Model::Gpt4, ApiKey: openai_api_key);
let exec = executor!(chatgpt, opts)?;
let map_prompt = create_map_prompt();
let reduce_prompt = create_reduce_prompt();
let chain = Chain::new(map_prompt, reduce_prompt);

let docs = Vec::from([parameters!(resume_text)]);

let res = chain.run(docs, Parameters::new(), &exec).await?;

let content = res.to_immediate().await?.as_content().to_text();

Expand Down

0 comments on commit 3bd660d

Please sign in to comment.