Skip to content

Commit

Permalink
[Bug] Fix the checking error in gpt4v generation (#274)
Browse files Browse the repository at this point in the history
* fix

* Fix error handling in GPT4V class
  • Loading branch information
pufanyi authored Sep 22, 2024
1 parent cf8a9c1 commit 7fc3fbb
Showing 1 changed file with 5 additions and 3 deletions.
8 changes: 5 additions & 3 deletions lmms_eval/models/gpt4v.py
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,9 @@ def generate_until(self, requests) -> List[str]:
payload["max_tokens"] = gen_kwargs["max_new_tokens"]
payload["temperature"] = gen_kwargs["temperature"]

for attempt in range(5):
NUM_ATTEMPTS = 5

for attempt in range(NUM_ATTEMPTS):
try:
response = url_requests.post(API_URL, headers=headers, json=payload, timeout=self.timeout)
response_data = response.json()
Expand All @@ -200,10 +202,10 @@ def generate_until(self, requests) -> List[str]:
error_msg = ""

eval_logger.info(f"Attempt {attempt + 1} failed with error: {str(e)}.\nReponse: {error_msg}")
if attempt <= 5:
if attempt < NUM_ATTEMPTS - 1:
time.sleep(NUM_SECONDS_TO_SLEEP)
else: # If this was the last attempt, log and return empty string
eval_logger.error(f"All 5 attempts failed. Last error message: {str(e)}.\nResponse: {response.json()}")
eval_logger.error(f"All {NUM_ATTEMPTS} attempts failed. Last error message: {str(e)}.\nResponse: {response.json()}")
response_text = ""
res.append(response_text)
pbar.update(1)
Expand Down

0 comments on commit 7fc3fbb

Please sign in to comment.