diff --git a/openweights/worker/main.py b/openweights/worker/main.py index d01871c..161f45e 100644 --- a/openweights/worker/main.py +++ b/openweights/worker/main.py @@ -342,14 +342,16 @@ def _execute_job(self, job): stderr=subprocess.STDOUT, cwd=tmp_dir, env=env, - preexec_fn=os.setsid # Allow us to send signals to the process group + preexec_fn=os.setsid, # Allow us to send signals to the process group + bufsize=1, # Line buffered + universal_newlines=True # Text mode ) # Stream logs to both file and stdout - for line in iter(self.current_process.stdout.readline, b''): - decoded = line.decode().rstrip('\n') - print(decoded) - log_file.write(decoded + '\n') + for line in iter(self.current_process.stdout.readline, ''): + print(line.rstrip('\n'), flush=True) # Immediate stdout flush + log_file.write(line) + log_file.flush() # Force immediate write to file self.current_process.wait() diff --git a/openweights/worker/sft.py b/openweights/worker/sft.py index 9714705..5fdae5b 100644 --- a/openweights/worker/sft.py +++ b/openweights/worker/sft.py @@ -99,7 +99,7 @@ def apply_chat_template(examples): instruction_part, response_part = get_instruct_response_part(tokenizer) trainer_kwargs['data_collator'] = DataCollatorForSeq2Seq(tokenizer = tokenizer) trainer = train_on_responses_only( - SFTTrainer(**trainer), + SFTTrainer(**trainer_kwargs), instruction_part=instruction_part, response_part=response_part )