Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add wandb logging to BootstrapFewShot's metric_val #544

Closed
wants to merge 3 commits into from
Closed

Add wandb logging to BootstrapFewShot's metric_val #544

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
da2c480
Add wandb logging to `BootstrapFewShot's` `metric_val`
CShorten Mar 4, 2024
6775321
Whoops update
CShorten Mar 4, 2024
3167807
Automatic Style fixes
CShorten Mar 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 10 additions & 6 deletions dspy/teleprompt/bootstrap.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import threading

import tqdm
import wandb

import dsp
import dspy
Expand Down Expand Up @@ -29,7 +30,6 @@

# TODO: Add baselines=[...]


class BootstrapFewShot(Teleprompter):
def __init__(self, metric=None, metric_threshold=None, teacher_settings={}, max_bootstrapped_demos=4, max_labeled_demos=16, max_rounds=1, max_errors=5):
self.metric = metric
Expand All @@ -43,13 +43,13 @@ def __init__(self, metric=None, metric_threshold=None, teacher_settings={}, max_
self.error_count = 0
self.error_lock = threading.Lock()

def compile(self, student, *, teacher=None, trainset, valset=None):
def compile(self, student, *, teacher=None, trainset, valset=None, wandb_enabled=False):
self.trainset = trainset
self.valset = valset

self._prepare_student_and_teacher(student, teacher)
self._prepare_predictor_mappings()
self._bootstrap()
self._bootstrap(wandb_enabled=wandb_enabled)

self.student = self._train()
self.student._compiled = True
Expand Down Expand Up @@ -94,7 +94,7 @@ def _prepare_predictor_mappings(self):
self.name2predictor = name2predictor
self.predictor2name = predictor2name

def _bootstrap(self, *, max_bootstraps=None):
def _bootstrap(self, *, max_bootstraps=None, wandb_enabled=False):
max_bootstraps = max_bootstraps or self.max_bootstrapped_demos

bootstrapped = {}
Expand All @@ -106,7 +106,7 @@ def _bootstrap(self, *, max_bootstraps=None):
break

if example_idx not in bootstrapped:
success = self._bootstrap_one_example(example, round_idx)
success = self._bootstrap_one_example(example, round_idx, wandb_enabled)

if success:
bootstrapped[example_idx] = True
Expand All @@ -124,7 +124,7 @@ def _bootstrap(self, *, max_bootstraps=None):
# evaluate = Evaluate(program=self.teacher, metric=self.metric, num_threads=12)
# score = evaluate(self.metric, display_table=False, display_progress=True)

def _bootstrap_one_example(self, example, round_idx=0):
def _bootstrap_one_example(self, example, round_idx=0, wandb_enabled=False):
name2traces = self.name2traces
teacher = self.teacher #.deepcopy()
predictor_cache = {}
Expand All @@ -148,6 +148,10 @@ def _bootstrap_one_example(self, example, round_idx=0):

if self.metric:
metric_val = self.metric(example, prediction, trace)
if wandb_enabled:
wandb.log({
"metric_val": metric_val,
})
if self.metric_threshold:
success = metric_val >= self.metric_threshold
else:
Expand Down