evaluate_with_chatgpt.py

import os
import openai
 
openai.api_key = os.environ["OPENAI_API_KEY"]

generator_list = ["fair_wmt20", "gpt1", "gpt2_small", "gpt2_medium", "gpt2_large", "gpt2_pytorch", "gpt2_xl", "gpt3", "grover_base",\
    "grover_large", "grover_mega", "pplm_distil", "pplm_gpt2", "transfo_xl", "xlm", "xlnet_base", "xlnet_large"]


model_engine = "gpt-3.5-turbo"
max_tokens = 1024

for generator in generator_list:
    responses = []

    path = "ENTER PATH HERE"
    filename1 = "ENTER FILENAME FOR JSONL FILE HERE"

    ground_truth_label = 1.0  ## for ai

    test_df =  pd.read_json(path + filename1, lines=True, orient='records')

    test_df = test_df[:2000] ## only testing on first 2000 samples
    final_df = pd.DataFrame(columns = ['text', 'ground_truth'])
    final_df['text'] = test_df['text']
    final_df = final_df.assign(ground_truth=ground_truth_label)

    for i in range(len(final_df)):
        time.sleep(1.5)
        text = final_df.iloc[i].text
        prompt = "Is the following passage generated by an AI or written by a human: '"+text+"'"
        # print(i)
        try:
            completion = openai.ChatCompletion.create(
            model=model_engine,
            messages=[
                {'role': 'user', 'content': prompt}
            ],
            max_tokens=max_tokens,
            temperature=0,
            top_p=1,
            frequency_penalty=0,
            presence_penalty=0
            )
            response = completion.choices[0].message.content
        except Exception as e:
            response = "null"
        responses.append(response)

    temp_df = pd.DataFrame(responses, columns=['chatgpt_response'])
    frames = [final_df, temp_df]
    final_df_to_save = pd.concat(frames, axis = 1)
    final_df_to_save.to_csv("/chatgpt_responses_"+"generator", index=False) 
    ## this file will be processed for getting the labels "yes", "no", "unclear"

    del test_df, temp_df, final_df, final_df_to_save