From 1c607ba5ac8c6c3c516fa96d77accfc13dc76c9e Mon Sep 17 00:00:00 2001 From: mohammadrezapourreza Date: Wed, 29 Nov 2023 17:50:41 -0500 Subject: [PATCH 1/6] DH-5033/ the new endpoints for finetuning --- dataherald/finetuning/openai_finetuning.py | 188 +++++++++++++++++++++ 1 file changed, 188 insertions(+) create mode 100644 dataherald/finetuning/openai_finetuning.py diff --git a/dataherald/finetuning/openai_finetuning.py b/dataherald/finetuning/openai_finetuning.py new file mode 100644 index 00000000..838814bb --- /dev/null +++ b/dataherald/finetuning/openai_finetuning.py @@ -0,0 +1,188 @@ +import json +import os +import time +import uuid +from typing import Any, List + +import openai +from bson.objectid import ObjectId + +from dataherald.db_scanner.models.types import TableDescription, TableDescriptionStatus +from dataherald.db_scanner.repository.base import TableDescriptionRepository +from dataherald.repositories.database_connections import DatabaseConnectionRepository +from dataherald.repositories.finetunings import FinetuningsRepository +from dataherald.repositories.golden_records import GoldenRecordRepository +from dataherald.types import Finetuning +from dataherald.utils.agent_prompts import FINETUNING_SYSTEM_INFORMATION + +FILE_PROCESSING_ATTEMPTS = 20 + + +class OpenAIFineTuning: + finetuning_dataset_path: str + + def format_columns(self, table: TableDescription, top_k: int = 100) -> str: + """ + format_columns formats the columns. + + Args: + table: The table to format. + top_k: The number of categories to show. + + Returns: + The formatted columns in string format. + """ + columns_information = "" + for column in table.columns: + name = column.name + is_primary_key = column.is_primary_key + if is_primary_key: + primary_key_text = ( + f"this column is a primary key of the table {table.table_name}," + ) + else: + primary_key_text = "" + foreign_key = column.foreign_key + if foreign_key: + foreign_key_text = ( + f"this column has a foreign key to the table {foreign_key}," + ) + else: + foreign_key_text = "" + categories = column.categories + if categories: + if len(categories) <= top_k: + categories_text = f"Categories: {categories}," + else: + categories_text = "" + else: + categories_text = "" + if primary_key_text or foreign_key_text or categories_text: + columns_information += ( + f"{name}: {primary_key_text}{foreign_key_text}{categories_text}\n" + ) + return columns_information + + @staticmethod + def format_dataset(self, db_scan: List[TableDescription]) -> str: + schema_of_database = "" + for table in db_scan: + tables_schema = table.table_schema + schema_of_database += f"{tables_schema}\n" + schema_of_database += "# Categorical Columns:\n" + columns_information = self.format_columns(table) + schema_of_database += columns_information + sample_rows = table.examples + schema_of_database += "# Sample rows:\n" + for item in sample_rows: + for key, value in item.items(): + schema_of_database += f"{key}: {value}, " + schema_of_database += "\n" + schema_of_database += "\n\n" + return schema_of_database + + @classmethod + def create_fintuning_dataset(cls, fine_tuning_request: Finetuning, storage: Any): + db_connection_id = fine_tuning_request.db_connection_id + repository = TableDescriptionRepository(storage) + db_scan = repository.get_all_tables_by_db( + { + "db_connection_id": ObjectId(db_connection_id), + "status": TableDescriptionStatus.SYNCHRONIZED.value, + } + ) + golden_records_repository = GoldenRecordRepository(storage) + database_schema = cls.format_dataset(db_scan) + cls.finetuning_dataset_path = f"tmp/{str(uuid.uuid4())}.jsonl" + for golden_record_id in fine_tuning_request.golden_records: + golden_record = golden_records_repository.find_by_id(golden_record_id) + question = golden_record.question + query = golden_record.sql_query + system_prompt = FINETUNING_SYSTEM_INFORMATION + database_schema + user_prompt = "User Question: " + question + "\n SQL: " + assistant_prompt = query + "\n" + with open(cls.finetuning_dataset_path, "a") as outfile: + messages = { + "messages": [ + {"role": "system", "content": f"{system_prompt}"}, + {"role": "user", "content": f"Question : {user_prompt}"}, + {"role": "assistant", "content": f"{assistant_prompt}"}, + ] + } + json.dump(messages, outfile) + outfile.write("\n") + db_connection_repository = DatabaseConnectionRepository(storage) + db_connection = db_connection_repository.find_by_id( + fine_tuning_request.db_connection_id + ) + openai.api_key = db_connection.decrypt_api_key() + model_repository = FinetuningsRepository(storage) + model = model_repository.find_by_id(fine_tuning_request.id) + model.finetuning_file_id = openai.File.create( + file=open(cls.finetuning_dataset_path, purpose="fine-tune") + )["id"] + model_repository.update(model) + os.remove(cls.finetuning_dataset_path) + + @classmethod + def create_fine_tuning_job(cls, fine_tuning_request: Finetuning, storage: Any): + db_connection_repository = DatabaseConnectionRepository(storage) + db_connection = db_connection_repository.find_by_id( + fine_tuning_request.db_connection_id + ) + openai.api_key = db_connection.decrypt_api_key() + model_repository = FinetuningsRepository(storage) + model = model_repository.find_by_id(fine_tuning_request.id) + retrieve_file_attempt = 0 + while True: + if ( + openai.File.retrieve(id=model.finetuning_file_id)["status"] + == "processed" + ): + break + time.sleep(5) + retrieve_file_attempt += 1 + if retrieve_file_attempt == FILE_PROCESSING_ATTEMPTS: + model.status = "failed" + model.error = "File processing failed" + model_repository.update(model) + return + finetuning_request = openai.FineTune.create( + training_file=model.finetuning_file_id, + model=model.base_llm.model_name, + hyperparameters=model.base_llm.model_parameters, + ) + model.finetuning_job_id = finetuning_request["id"] + if finetuning_request["status"] == "failed": + model.error = "Fine tuning failed before starting" + model.status = finetuning_request["status"] + model_repository.update(model) + + @classmethod + def retrieve_finetuning_job(cls, fine_tuning_request: Finetuning, storage: Any): + db_connection_repository = DatabaseConnectionRepository(storage) + db_connection = db_connection_repository.find_by_id( + fine_tuning_request.db_connection_id + ) + openai.api_key = db_connection.decrypt_api_key() + model_repository = FinetuningsRepository(storage) + model = model_repository.find_by_id(fine_tuning_request.id) + finetuning_request = openai.FineTune.retrieve(id=model.finetuning_job_id) + if finetuning_request["status"] == "failed": + model.error = "Fine tuning failed during processing by OpenAI" + model.status = finetuning_request["status"] + model_repository.update(model) + + @classmethod + def cancel_finetuning_job(cls, fine_tuning_request: Finetuning, storage: Any): + db_connection_repository = DatabaseConnectionRepository(storage) + db_connection = db_connection_repository.find_by_id( + fine_tuning_request.db_connection_id + ) + openai.api_key = db_connection.decrypt_api_key() + model_repository = FinetuningsRepository(storage) + model = model_repository.find_by_id(fine_tuning_request.id) + finetuning_request = openai.FineTune.cancel(id=model.finetuning_job_id) + model.status = finetuning_request["status"] + model.error = "Fine tuning cancelled by the user" + model_repository.update(model) From 9922110847352e388732f616bfa4ac4892ced15d Mon Sep 17 00:00:00 2001 From: mohammadrezapourreza Date: Wed, 29 Nov 2023 17:52:06 -0500 Subject: [PATCH 2/6] DH-5033/new endpoints for finetuning --- dataherald/api/__init__.py | 19 +++++ dataherald/api/fastapi.py | 90 ++++++++++++++++++++++ dataherald/finetuning/__init__py | 2 + dataherald/finetuning/openai_finetuning.py | 17 ++-- dataherald/repositories/finetunings.py | 61 +++++++++++++++ dataherald/server/fastapi/__init__.py | 41 ++++++++++ dataherald/types.py | 40 ++++++++++ 7 files changed, 262 insertions(+), 8 deletions(-) create mode 100644 dataherald/finetuning/__init__py create mode 100644 dataherald/repositories/finetunings.py diff --git a/dataherald/api/__init__.py b/dataherald/api/__init__.py index 84140c46..fdbc8f96 100644 --- a/dataherald/api/__init__.py +++ b/dataherald/api/__init__.py @@ -9,8 +9,11 @@ from dataherald.db_scanner.models.types import TableDescription from dataherald.sql_database.models.types import DatabaseConnection, SSHSettings from dataherald.types import ( + CancelFineTuningRequest, CreateResponseRequest, DatabaseConnectionRequest, + Finetuning, + FineTuningRequest, GoldenRecord, GoldenRecordRequest, Instruction, @@ -167,3 +170,19 @@ def update_instruction( instruction_request: UpdateInstruction, ) -> Instruction: pass + + @abstractmethod + def create_finetuning_job( + self, fine_tuning_request: FineTuningRequest, background_tasks: BackgroundTasks + ) -> Finetuning: + pass + + @abstractmethod + def cancel_finetuning_job( + self, cancel_fine_tuning_request: CancelFineTuningRequest + ) -> Finetuning: + pass + + @abstractmethod + def get_finetuning_job(self, finetuning_job_id: str) -> Finetuning: + pass diff --git a/dataherald/api/fastapi.py b/dataherald/api/fastapi.py index 26409845..54495afb 100644 --- a/dataherald/api/fastapi.py +++ b/dataherald/api/fastapi.py @@ -27,6 +27,7 @@ from dataherald.eval import Evaluator from dataherald.repositories.base import ResponseRepository from dataherald.repositories.database_connections import DatabaseConnectionRepository +from dataherald.repositories.finetunings import FinetuningsRepository from dataherald.repositories.golden_records import GoldenRecordRepository from dataherald.repositories.instructions import InstructionRepository from dataherald.repositories.question import QuestionRepository @@ -39,8 +40,11 @@ from dataherald.sql_generator import SQLGenerator from dataherald.sql_generator.generates_nl_answer import GeneratesNlAnswer from dataherald.types import ( + CancelFineTuningRequest, CreateResponseRequest, DatabaseConnectionRequest, + Finetuning, + FineTuningRequest, GoldenRecord, GoldenRecordRequest, Instruction, @@ -68,6 +72,10 @@ def async_scanning(scanner, database, scanner_request, storage): ) +def async_fine_tuning(): + pass + + def delete_file(file_location: str): os.remove(file_location) @@ -638,3 +646,85 @@ def update_instruction( ) instruction_repository.update(updated_instruction) return json.loads(json_util.dumps(updated_instruction)) + + @override + def create_finetuning_job( + self, fine_tuning_request: FineTuningRequest, background_tasks: BackgroundTasks + ) -> Finetuning: + db_connection_repository = DatabaseConnectionRepository(self.storage) + + db_connection = db_connection_repository.find_by_id( + fine_tuning_request.db_connection_id + ) + if not db_connection: + raise HTTPException(status_code=404, detail="Database connection not found") + + golden_records_repository = GoldenRecordRepository(self.storage) + golden_records = [] + if fine_tuning_request.golden_records: + for golden_record_id in fine_tuning_request.golden_records: + golden_record = golden_records_repository.find_by_id(golden_record_id) + if not golden_record: + raise HTTPException( + status_code=404, detail="Golden record not found" + ) + golden_records.append(golden_record) + else: + golden_records = golden_records_repository.find_by( + {"db_connection_id": ObjectId(fine_tuning_request.db_connection_id)}, + page=0, + limit=0, + ) + if not golden_records: + raise HTTPException(status_code=404, detail="No golden records found") + + model_repository = FinetuningsRepository(self.storage) + model = model_repository.insert( + Finetuning( + db_connection_id=fine_tuning_request.db_connection_id, + base_llm=fine_tuning_request.base_llm, + golden_records=[ + str(golden_record.id) for golden_record in golden_records + ], + ) + ) + + background_tasks.add_task( + async_fine_tuning, fine_tuning_request, self.storage, golden_records + ) + + return model + + @override + def cancel_finetuning_job( + self, cancel_fine_tuning_request: CancelFineTuningRequest + ) -> Finetuning: + model_repository = FinetuningsRepository(self.storage) + model = model_repository.find_by_id(cancel_fine_tuning_request.finetuning_id) + if not model: + raise HTTPException(status_code=404, detail="Model not found") + + if model.status == "succeeded": + raise HTTPException( + status_code=400, detail="Model has already succeeded. Cannot cancel." + ) + if model.status == "failed": + raise HTTPException( + status_code=400, detail="Model has already failed. Cannot cancel." + ) + if model.status == "cancelled": + raise HTTPException( + status_code=400, detail="Model has already been cancelled." + ) + + # Todo: Add code to cancel the fine tuning job + + return model + + @override + def get_finetuning_job(self, finetuning_job_id: str) -> Finetuning: + model_repository = FinetuningsRepository(self.storage) + model = model_repository.find_by_id(finetuning_job_id) + if not model: + raise HTTPException(status_code=404, detail="Model not found") + return model diff --git a/dataherald/finetuning/__init__py b/dataherald/finetuning/__init__py new file mode 100644 index 00000000..4979f15d --- /dev/null +++ b/dataherald/finetuning/__init__py @@ -0,0 +1,2 @@ +class Finetuning: + pass diff --git a/dataherald/finetuning/openai_finetuning.py b/dataherald/finetuning/openai_finetuning.py index 838814bb..5f7f9cb8 100644 --- a/dataherald/finetuning/openai_finetuning.py +++ b/dataherald/finetuning/openai_finetuning.py @@ -1,4 +1,5 @@ import json +import logging import os import time import uuid @@ -17,6 +18,7 @@ FILE_PROCESSING_ATTEMPTS = 20 +logger = logging.getLogger(__name__) class OpenAIFineTuning: finetuning_dataset_path: str @@ -118,12 +120,11 @@ def create_fintuning_dataset(cls, fine_tuning_request: Finetuning, storage: Any) openai.api_key = db_connection.decrypt_api_key() model_repository = FinetuningsRepository(storage) model = model_repository.find_by_id(fine_tuning_request.id) - model.finetuning_file_id = openai.File.create( - file=open(cls.finetuning_dataset_path, purpose="fine-tune") - )["id"] + model.finetuning_file_id = openai.File.create(file=open(cls.finetuning_dataset_path,purpose='fine-tune'))['id'] model_repository.update(model) os.remove(cls.finetuning_dataset_path) + @classmethod def create_fine_tuning_job(cls, fine_tuning_request: Finetuning, storage: Any): db_connection_repository = DatabaseConnectionRepository(storage) @@ -135,10 +136,7 @@ def create_fine_tuning_job(cls, fine_tuning_request: Finetuning, storage: Any): model = model_repository.find_by_id(fine_tuning_request.id) retrieve_file_attempt = 0 while True: - if ( - openai.File.retrieve(id=model.finetuning_file_id)["status"] - == "processed" - ): + if openai.File.retrieve(id=model.finetuning_file_id)["status"] == "processed": break time.sleep(5) retrieve_file_attempt += 1 @@ -150,7 +148,7 @@ def create_fine_tuning_job(cls, fine_tuning_request: Finetuning, storage: Any): finetuning_request = openai.FineTune.create( training_file=model.finetuning_file_id, model=model.base_llm.model_name, - hyperparameters=model.base_llm.model_parameters, + hyperparameters= model.base_llm.model_parameters ) model.finetuning_job_id = finetuning_request["id"] if finetuning_request["status"] == "failed": @@ -186,3 +184,6 @@ def cancel_finetuning_job(cls, fine_tuning_request: Finetuning, storage: Any): model.status = finetuning_request["status"] model.error = "Fine tuning cancelled by the user" model_repository.update(model) + + + diff --git a/dataherald/repositories/finetunings.py b/dataherald/repositories/finetunings.py new file mode 100644 index 00000000..baf3aef9 --- /dev/null +++ b/dataherald/repositories/finetunings.py @@ -0,0 +1,61 @@ +from bson.objectid import ObjectId + +from dataherald.types import Finetuning + +DB_COLLECTION = "finetunings" + + +class FinetuningsRepository: + def __init__(self, storage): + self.storage = storage + + def insert(self, model: Finetuning) -> Finetuning: + model.id = str( + self.storage.insert_one(DB_COLLECTION, model.dict(exclude={"id"})) + ) + return model + + def find_one(self, query: dict) -> Finetuning | None: + row = self.storage.find_one(DB_COLLECTION, query) + if not row: + return None + obj = Finetuning(**row) + obj.id = str(row["_id"]) + return obj + + def update(self, model: Finetuning) -> Finetuning: + self.storage.update_or_create( + DB_COLLECTION, + {"_id": ObjectId(model.id)}, + model.dict(exclude={"id"}), + ) + return model + + def find_by_id(self, id: str) -> Finetuning | None: + row = self.storage.find_one(DB_COLLECTION, {"_id": ObjectId(id)}) + if not row: + return None + obj = Finetuning(**row) + obj.id = str(row["_id"]) + return obj + + def find_by(self, query: dict, page: int = 1, limit: int = 10) -> list[Finetuning]: + rows = self.storage.find(DB_COLLECTION, query, page=page, limit=limit) + result = [] + for row in rows: + obj = Finetuning(**row) + obj.id = str(row["_id"]) + result.append(obj) + return result + + def find_all(self, page: int = 0, limit: int = 0) -> list[Finetuning]: + rows = self.storage.find_all(DB_COLLECTION, page=page, limit=limit) + result = [] + for row in rows: + obj = Finetuning(**row) + obj.id = str(row["_id"]) + result.append(obj) + return result + + def delete_by_id(self, id: str) -> int: + return self.storage.delete_by_id(DB_COLLECTION, id) diff --git a/dataherald/server/fastapi/__init__.py b/dataherald/server/fastapi/__init__.py index 82be2092..fe180f56 100644 --- a/dataherald/server/fastapi/__init__.py +++ b/dataherald/server/fastapi/__init__.py @@ -13,8 +13,11 @@ from dataherald.db_scanner.models.types import TableDescription from dataherald.sql_database.models.types import DatabaseConnection, SSHSettings from dataherald.types import ( + CancelFineTuningRequest, CreateResponseRequest, DatabaseConnectionRequest, + Finetuning, + FineTuningRequest, GoldenRecord, GoldenRecordRequest, Instruction, @@ -215,6 +218,28 @@ def __init__(self, settings: Settings): tags=["Instructions"], ) + self.router.add_api_route( + "/api/v1/finetunings", + self.create_finetuning_job, + methods=["POST"], + status_code=201, + tags=["Finetunings"], + ) + + self.router.add_api_route( + "/api/v1/finetunings/{finetuning_id}", + self.get_finetuning_job, + methods=["GET"], + tags=["Finetunings"], + ) + + self.router.add_api_route( + "/api/v1/finetunings/{finetuning_id}/cancel", + self.cancel_finetuning_job, + methods=["POST"], + tags=["Finetunings"], + ) + self.router.add_api_route( "/api/v1/heartbeat", self.heartbeat, methods=["GET"], tags=["System"] ) @@ -379,3 +404,19 @@ def update_instruction( ) -> Instruction: """Updates an instruction""" return self._api.update_instruction(instruction_id, instruction_request) + + def create_finetuning_job( + self, fine_tuning_request: FineTuningRequest, background_tasks: BackgroundTasks + ) -> Finetuning: + """Creates a fine tuning job""" + return self._api.create_finetuning_job(fine_tuning_request, background_tasks) + + def cancel_finetuning_job( + self, cancel_fine_tuning_request: CancelFineTuningRequest + ) -> Finetuning: + """Cancels a fine tuning job""" + return self._api.cancel_finetuning_job(cancel_fine_tuning_request) + + def get_finetuning_job(self, finetuning_job_id: str) -> Finetuning: + """Gets fine tuning jobs""" + return self._api.get_finetuning_job(finetuning_job_id) diff --git a/dataherald/types.py b/dataherald/types.py index 6cddcf9a..3d69a340 100644 --- a/dataherald/types.py +++ b/dataherald/types.py @@ -139,3 +139,43 @@ class ColumnDescriptionRequest(BaseModel): class TableDescriptionRequest(BaseModel): description: str | None columns: list[ColumnDescriptionRequest] | None + + +class FineTuningStatus(Enum): + QUEUED = "queued" + RUNNING = "running" + SUCCEEDED = "succeeded" + FAILED = "failed" + CANCELLED = "cancelled" + + +class BaseLLM(BaseModel): + model_provider: str | None = None + model_name: str | None = None + model_parameters: dict[str, str] | None = None + + +class Finetuning(BaseModel): + id: str | None = None + db_connection_id: str | None = None + status: str = "queued" + error: str | None = None + base_llm: BaseLLM | None = None + finetuning_file_id: str | None = None + finetuning_job_id: str | None = None + model_id: str | None = None + created_at: datetime = Field(default_factory=datetime.now) + golden_records: list[str] | None = None + metadata: dict[str, str] | None = None + + +class FineTuningRequest(BaseModel): + db_connection_id: str + base_llm: BaseLLM + golden_records: list[str] | None = None + metadata: dict[str, str] | None = None + + +class CancelFineTuningRequest(BaseModel): + finetuning_id: str + metadata: dict[str, str] | None = None From d683a32de7adf2d968b04f305cf434a42fad65b7 Mon Sep 17 00:00:00 2001 From: mohammadrezapourreza Date: Thu, 30 Nov 2023 16:45:26 -0500 Subject: [PATCH 3/6] DH-5033/ finalized llm finetuning with openai --- dataherald/api/fastapi.py | 25 ++- dataherald/finetuning/__init__.py | 29 +++ dataherald/finetuning/__init__py | 2 - dataherald/finetuning/openai_finetuning.py | 194 ++++++++++++--------- dataherald/types.py | 1 + requirements.txt | 4 +- 6 files changed, 165 insertions(+), 90 deletions(-) create mode 100644 dataherald/finetuning/__init__.py delete mode 100644 dataherald/finetuning/__init__py diff --git a/dataherald/api/fastapi.py b/dataherald/api/fastapi.py index 54495afb..dc1c50a9 100644 --- a/dataherald/api/fastapi.py +++ b/dataherald/api/fastapi.py @@ -25,6 +25,7 @@ TableDescriptionRepository, ) from dataherald.eval import Evaluator +from dataherald.finetuning.openai_finetuning import OpenAIFineTuning from dataherald.repositories.base import ResponseRepository from dataherald.repositories.database_connections import DatabaseConnectionRepository from dataherald.repositories.finetunings import FinetuningsRepository @@ -56,6 +57,7 @@ TableDescriptionRequest, UpdateInstruction, ) +from dataherald.utils.models_context_window import OPENAI_CONTEXT_WIDNOW_SIZES from dataherald.utils.s3 import S3 logger = logging.getLogger(__name__) @@ -72,8 +74,10 @@ def async_scanning(scanner, database, scanner_request, storage): ) -def async_fine_tuning(): - pass +def async_fine_tuning(storage, model): + openai_fine_tuning = OpenAIFineTuning(storage, model) + openai_fine_tuning.create_fintuning_dataset() + openai_fine_tuning.create_fine_tuning_job() def delete_file(file_location: str): @@ -678,6 +682,12 @@ def create_finetuning_job( if not golden_records: raise HTTPException(status_code=404, detail="No golden records found") + if fine_tuning_request.base_llm.model_name not in OPENAI_CONTEXT_WIDNOW_SIZES: + raise HTTPException( + status_code=400, + detail=f"Model {fine_tuning_request.base_llm.model_name} not supported", + ) + model_repository = FinetuningsRepository(self.storage) model = model_repository.insert( Finetuning( @@ -689,9 +699,7 @@ def create_finetuning_job( ) ) - background_tasks.add_task( - async_fine_tuning, fine_tuning_request, self.storage, golden_records - ) + background_tasks.add_task(async_fine_tuning, self.storage, model) return model @@ -717,9 +725,9 @@ def cancel_finetuning_job( status_code=400, detail="Model has already been cancelled." ) - # Todo: Add code to cancel the fine tuning job + openai_fine_tuning = OpenAIFineTuning(self.storage, model) - return model + return openai_fine_tuning.cancel_finetuning_job() @override def get_finetuning_job(self, finetuning_job_id: str) -> Finetuning: @@ -727,4 +735,5 @@ def get_finetuning_job(self, finetuning_job_id: str) -> Finetuning: model = model_repository.find_by_id(finetuning_job_id) if not model: raise HTTPException(status_code=404, detail="Model not found") - return model + openai_fine_tuning = OpenAIFineTuning(self.storage, model) + return openai_fine_tuning.retrieve_finetuning_job() diff --git a/dataherald/finetuning/__init__.py b/dataherald/finetuning/__init__.py new file mode 100644 index 00000000..4521214b --- /dev/null +++ b/dataherald/finetuning/__init__.py @@ -0,0 +1,29 @@ +from abc import ABC, abstractmethod + +from dataherald.config import Component +from dataherald.types import Finetuning + + +class FinetuningModel(Component, ABC): + def __init__(self, storage): + self.storage = storage + + @abstractmethod + def count_tokens(self, messages: dict) -> int: + pass + + @abstractmethod + def create_fintuning_dataset(self): + pass + + @abstractmethod + def create_fine_tuning_job(self): + pass + + @abstractmethod + def retrieve_finetuning_job(self) -> Finetuning: + pass + + @abstractmethod + def cancel_finetuning_job(self) -> Finetuning: + pass diff --git a/dataherald/finetuning/__init__py b/dataherald/finetuning/__init__py deleted file mode 100644 index 4979f15d..00000000 --- a/dataherald/finetuning/__init__py +++ /dev/null @@ -1,2 +0,0 @@ -class Finetuning: - pass diff --git a/dataherald/finetuning/openai_finetuning.py b/dataherald/finetuning/openai_finetuning.py index 5f7f9cb8..e4028ab2 100644 --- a/dataherald/finetuning/openai_finetuning.py +++ b/dataherald/finetuning/openai_finetuning.py @@ -5,25 +5,47 @@ import uuid from typing import Any, List -import openai +import tiktoken from bson.objectid import ObjectId +from openai import OpenAI +from overrides import override +from tiktoken import Encoding from dataherald.db_scanner.models.types import TableDescription, TableDescriptionStatus from dataherald.db_scanner.repository.base import TableDescriptionRepository +from dataherald.finetuning import FinetuningModel from dataherald.repositories.database_connections import DatabaseConnectionRepository from dataherald.repositories.finetunings import FinetuningsRepository from dataherald.repositories.golden_records import GoldenRecordRepository from dataherald.types import Finetuning from dataherald.utils.agent_prompts import FINETUNING_SYSTEM_INFORMATION +from dataherald.utils.models_context_window import OPENAI_CONTEXT_WIDNOW_SIZES FILE_PROCESSING_ATTEMPTS = 20 logger = logging.getLogger(__name__) -class OpenAIFineTuning: - finetuning_dataset_path: str - def format_columns(self, table: TableDescription, top_k: int = 100) -> str: +class OpenAIFineTuning(FinetuningModel): + encoding: Encoding + fine_tuning_model: Finetuning + storage: Any + client: OpenAI + + def __init__(self, storage: Any, fine_tuning_model: Finetuning): + self.storage = storage + self.fine_tuning_model = fine_tuning_model + db_connection_repository = DatabaseConnectionRepository(storage) + db_connection = db_connection_repository.find_by_id( + fine_tuning_model.db_connection_id + ) + self.encoding = tiktoken.encoding_for_model( + fine_tuning_model.base_llm.model_name + ) + self.client = OpenAI(api_key=db_connection.decrypt_api_key()) + + @classmethod + def format_columns(cls, table: TableDescription, top_k: int = 100) -> str: """ format_columns formats the columns. @@ -65,14 +87,14 @@ def format_columns(self, table: TableDescription, top_k: int = 100) -> str: ) return columns_information - @staticmethod - def format_dataset(self, db_scan: List[TableDescription]) -> str: + @classmethod + def format_dataset(cls, db_scan: List[TableDescription]) -> str: schema_of_database = "" for table in db_scan: tables_schema = table.table_schema schema_of_database += f"{tables_schema}\n" schema_of_database += "# Categorical Columns:\n" - columns_information = self.format_columns(table) + columns_information = cls.format_columns(table) schema_of_database += columns_information sample_rows = table.examples schema_of_database += "# Sample rows:\n" @@ -83,27 +105,36 @@ def format_dataset(self, db_scan: List[TableDescription]) -> str: schema_of_database += "\n\n" return schema_of_database - @classmethod - def create_fintuning_dataset(cls, fine_tuning_request: Finetuning, storage: Any): - db_connection_id = fine_tuning_request.db_connection_id - repository = TableDescriptionRepository(storage) + @override + def count_tokens(self, messages: dict) -> int: + prompt = "" + for message in messages["messages"]: + prompt += message["content"] + return len(self.encoding.encode(prompt)) + + @override + def create_fintuning_dataset(self): + db_connection_id = self.fine_tuning_model.db_connection_id + repository = TableDescriptionRepository(self.storage) db_scan = repository.get_all_tables_by_db( { "db_connection_id": ObjectId(db_connection_id), "status": TableDescriptionStatus.SYNCHRONIZED.value, } ) - golden_records_repository = GoldenRecordRepository(storage) - database_schema = cls.format_dataset(db_scan) - cls.finetuning_dataset_path = f"tmp/{str(uuid.uuid4())}.jsonl" - for golden_record_id in fine_tuning_request.golden_records: + golden_records_repository = GoldenRecordRepository(self.storage) + database_schema = self.format_dataset(db_scan) + finetuning_dataset_path = f"tmp/{str(uuid.uuid4())}.jsonl" + model_repository = FinetuningsRepository(self.storage) + model = model_repository.find_by_id(self.fine_tuning_model.id) + for golden_record_id in self.fine_tuning_model.golden_records: golden_record = golden_records_repository.find_by_id(golden_record_id) question = golden_record.question query = golden_record.sql_query system_prompt = FINETUNING_SYSTEM_INFORMATION + database_schema user_prompt = "User Question: " + question + "\n SQL: " assistant_prompt = query + "\n" - with open(cls.finetuning_dataset_path, "a") as outfile: + with open(finetuning_dataset_path, "a") as outfile: messages = { "messages": [ {"role": "system", "content": f"{system_prompt}"}, @@ -111,79 +142,86 @@ def create_fintuning_dataset(cls, fine_tuning_request: Finetuning, storage: Any) {"role": "assistant", "content": f"{assistant_prompt}"}, ] } + number_of_tokens = self.count_tokens(messages) + if ( + number_of_tokens + > OPENAI_CONTEXT_WIDNOW_SIZES[ + self.fine_tuning_model.base_llm.model_name + ] + ): + model.status = "failed" + model.error = "The number of tokens in the prompt is too large" + model_repository.update(model) + os.remove(finetuning_dataset_path) + return json.dump(messages, outfile) outfile.write("\n") - db_connection_repository = DatabaseConnectionRepository(storage) - db_connection = db_connection_repository.find_by_id( - fine_tuning_request.db_connection_id - ) - openai.api_key = db_connection.decrypt_api_key() - model_repository = FinetuningsRepository(storage) - model = model_repository.find_by_id(fine_tuning_request.id) - model.finetuning_file_id = openai.File.create(file=open(cls.finetuning_dataset_path,purpose='fine-tune'))['id'] + model.finetuning_file_id = self.client.files.create( + file=open(finetuning_dataset_path, "rb"), purpose="fine-tune" + ).id model_repository.update(model) - os.remove(cls.finetuning_dataset_path) - + os.remove(finetuning_dataset_path) - @classmethod - def create_fine_tuning_job(cls, fine_tuning_request: Finetuning, storage: Any): - db_connection_repository = DatabaseConnectionRepository(storage) - db_connection = db_connection_repository.find_by_id( - fine_tuning_request.db_connection_id - ) - openai.api_key = db_connection.decrypt_api_key() - model_repository = FinetuningsRepository(storage) - model = model_repository.find_by_id(fine_tuning_request.id) + def check_file_status(self, file_id: str) -> bool: retrieve_file_attempt = 0 while True: - if openai.File.retrieve(id=model.finetuning_file_id)["status"] == "processed": - break + file_info = self.client.files.retrieve(file_id=file_id) + if file_info.status == "processed": + return True time.sleep(5) retrieve_file_attempt += 1 if retrieve_file_attempt == FILE_PROCESSING_ATTEMPTS: - model.status = "failed" - model.error = "File processing failed" - model_repository.update(model) - return - finetuning_request = openai.FineTune.create( - training_file=model.finetuning_file_id, - model=model.base_llm.model_name, - hyperparameters= model.base_llm.model_parameters + return False + + @override + def create_fine_tuning_job(self): + model_repository = FinetuningsRepository(self.storage) + model = model_repository.find_by_id(self.fine_tuning_model.id) + if self.check_file_status(model.finetuning_file_id): + finetuning_request = self.client.fine_tuning.jobs.create( + training_file=model.finetuning_file_id, + model=model.base_llm.model_name, + hyperparameters=model.base_llm.model_parameters + if model.base_llm.model_parameters + else { + "batch_size": 1, + "learning_rate_multiplier": "auto", + "n_epochs": 3, + }, + ) + model.finetuning_job_id = finetuning_request.id + if finetuning_request.status == "failed": + model.error = "Fine tuning failed before starting" + model.status = finetuning_request.status + model_repository.update(model) + else: + model.status = "failed" + model.error = "File processing failed" + model_repository.update(model) + + @override + def retrieve_finetuning_job(self) -> Finetuning: + model_repository = FinetuningsRepository(self.storage) + model = model_repository.find_by_id(self.fine_tuning_model.id) + finetuning_request = self.client.fine_tuning.jobs.retrieve( + fine_tuning_job_id=model.finetuning_job_id ) - model.finetuning_job_id = finetuning_request["id"] - if finetuning_request["status"] == "failed": - model.error = "Fine tuning failed before starting" - model.status = finetuning_request["status"] + if finetuning_request.status == "failed": + model.error = finetuning_request.error.message + model.status = finetuning_request.status + if finetuning_request.fine_tuned_model: + model.model_id = finetuning_request.fine_tuned_model model_repository.update(model) - - @classmethod - def retrieve_finetuning_job(cls, fine_tuning_request: Finetuning, storage: Any): - db_connection_repository = DatabaseConnectionRepository(storage) - db_connection = db_connection_repository.find_by_id( - fine_tuning_request.db_connection_id + return model + + @override + def cancel_finetuning_job(self) -> Finetuning: + model_repository = FinetuningsRepository(self.storage) + model = model_repository.find_by_id(self.fine_tuning_model.id) + finetuning_request = self.client.fine_tuning.jobs.cancel( + fine_tuning_job_id=model.finetuning_job_id ) - openai.api_key = db_connection.decrypt_api_key() - model_repository = FinetuningsRepository(storage) - model = model_repository.find_by_id(fine_tuning_request.id) - finetuning_request = openai.FineTune.retrieve(id=model.finetuning_job_id) - if finetuning_request["status"] == "failed": - model.error = "Fine tuning failed during processing by OpenAI" - model.status = finetuning_request["status"] - model_repository.update(model) - - @classmethod - def cancel_finetuning_job(cls, fine_tuning_request: Finetuning, storage: Any): - db_connection_repository = DatabaseConnectionRepository(storage) - db_connection = db_connection_repository.find_by_id( - fine_tuning_request.db_connection_id - ) - openai.api_key = db_connection.decrypt_api_key() - model_repository = FinetuningsRepository(storage) - model = model_repository.find_by_id(fine_tuning_request.id) - finetuning_request = openai.FineTune.cancel(id=model.finetuning_job_id) - model.status = finetuning_request["status"] + model.status = finetuning_request.status model.error = "Fine tuning cancelled by the user" model_repository.update(model) - - - + return model diff --git a/dataherald/types.py b/dataherald/types.py index 3d69a340..99efe9e4 100644 --- a/dataherald/types.py +++ b/dataherald/types.py @@ -147,6 +147,7 @@ class FineTuningStatus(Enum): SUCCEEDED = "succeeded" FAILED = "failed" CANCELLED = "cancelled" + VALIDATING_FILES = "validating_files" class BaseLLM(BaseModel): diff --git a/requirements.txt b/requirements.txt index d102da95..4e4d9173 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,10 @@ dnspython==2.3.0 fastapi==0.98.0 httpx==0.24.1 -langchain==0.0.312 +langchain==0.0.335 load-dotenv==0.1.0 mypy-extensions==1.0.0 -openai==0.27.8 +openai==1.3.6 openapi-schema-pydantic==1.2.4 overrides==7.3.1 packaging==23.1 From 3d86840f272328b5c9e463070303b5fb2d0fc366 Mon Sep 17 00:00:00 2001 From: mohammadrezapourreza Date: Wed, 13 Dec 2023 13:19:30 -0500 Subject: [PATCH 4/6] DH-5033/adding the documents for finetuning --- docs/api.cancel_finetuning.rst | 101 +++++++++++++++++++++++++++ docs/api.finetuning.rst | 120 +++++++++++++++++++++++++++++++++ docs/api.get_finetuning.rst | 86 +++++++++++++++++++++++ docs/api.rst | 41 +++++++++++ 4 files changed, 348 insertions(+) create mode 100644 docs/api.cancel_finetuning.rst create mode 100644 docs/api.finetuning.rst create mode 100644 docs/api.get_finetuning.rst diff --git a/docs/api.cancel_finetuning.rst b/docs/api.cancel_finetuning.rst new file mode 100644 index 00000000..52f514dc --- /dev/null +++ b/docs/api.cancel_finetuning.rst @@ -0,0 +1,101 @@ +Cancel a finetuning job +======================== + +This endpoint allows you to cancel a finetuning job, which will stop the training process but will not delete the job from the collection of finetuning jobs. + +request this ``POST`` endpoint to cancel a finetuning job. + + /api/v1/finetuning-jobs//cancel + +**Request body** + +.. code-block:: rst + + { + "finetuning_id": "finetuing-job-id", + "metadata": dict[str, str] | None + } + +**Responses** + +HTTP 200 code response + +.. code-block:: rst + + { + "id": "finetuing-job-id", + "db_connection_id": "database_connection_id" + "status": "finetuning_job_status" # queued is default other possible values are [queued, running, succeeded, failed, validating_files, or cancelled] + "error": "The error message if the job failed" # optional default value is None + "base_llm": { + "model_provider": "model_provider_name" # right now openai is the only provider. + "model_name": "model_name" # right now gpt-3.5-turbo and gpt-4 are suported. + "model_parameters": { + "n_epochs": int or string, #optional, default value 3 + "batch_size": int or string, #optional, default value 1 + "learning_rate_multiplier", int or string, #optional, default value "auto" + } + } + "finetuning_file_id": "This is the file id that is assigned by model provider when uploading the finetuning file" + "finetuning_job_id": "The is the finetuning job id that is assigned by model provider when creating the finetuning job" + "model_id": "The is the model id that is assigned by model provider after finetuning job is done" + "created_at": datetime, + "golden_records": array[ids], # default value is none which means use all of the golden records + "metadata": dict[str, str] | None} #optional, default value None + } + +**Request example** + +.. code-block:: rst + + curl -X 'POST' \ + 'http://localhost/api/v1/finetunings/{finetuning_id}/cancel' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ + "finetuning_id": "finetuning-job-id" + }' + + +**Response example** + +.. code-block:: rst + + { + "id": "finetuning-job-id", + "db_connection_id": "database_connection_id", + "status": "cancelled", + "error": "Fine tuning cancelled by the user", + "base_llm": { + "model_provider": "openai", + "model_name": "gpt-3.5-turbo-1106", + "model_parameters": { + "n_epochs": "1" + } + }, + "finetuning_file_id": null, + "finetuning_job_id": null, + "model_id": null, + "created_at": "2023-12-13T17:35:53.263485", + "golden_records": [ # a list of golden record ids, keep in mind that at least 10 golden records are required for openai models finetuning + "656777c9eb4e22533dc43fce", + "656777ceeb4e22533dc43fcf", + "656777ceeb4e22533dc43fd0", + "656777ceeb4e22533dc43fd1", + "656777ceeb4e22533dc43fd2", + "656777ceeb4e22533dc43fd3", + "656777ceeb4e22533dc43fd4", + "656777ceeb4e22533dc43fd5", + "656777ceeb4e22533dc43fd6", + "656777ceeb4e22533dc43fd7", + "656777ceeb4e22533dc43fd8", + "656777ceeb4e22533dc43fd9", + "656777ceeb4e22533dc43fda", + "656777ceeb4e22533dc43fdb", + "656777ceeb4e22533dc43fdc", + "656777ceeb4e22533dc43fdd", + "656777ceeb4e22533dc43fde" + ], + "metadata": null + } + diff --git a/docs/api.finetuning.rst b/docs/api.finetuning.rst new file mode 100644 index 00000000..95f2770e --- /dev/null +++ b/docs/api.finetuning.rst @@ -0,0 +1,120 @@ +Create a finetuning job +========================= + +One of the features that our framework provides is the ability to finetune a model on your golden records. This model is going to be used inside an Agent for generating SQL queries for the given input. The finetuning job is going to be created asynchronously and you can check the status of the job by using the GET /api/v1/finetuning-jobs/ endpoint. + +This endpoint will create a file that is expected to be uploaded to the model provider. After the file is uploaded, we will create a finetuning job and train the model. After the model is trained, the model provider will assign a model id to the model and you can use this model id inside our agent. + + +Request this ``POST`` endpoint to create a finetuning job:: + + /api/v1/finetunings + +**Request body** + +.. code-block:: rst + + { + "db_connection_id": "database_connection_id" + "base_llm": { + "model_provider": "model_provider_name" # right now openai is the only provider. + "model_name": "model_name" # right now gpt-3.5-turbo and gpt-4 are suported. + "model_parameters": { + "n_epochs": int or string, #optional, default value 3 + "batch_size": int or string, #optional, default value 1 + "learning_rate_multiplier", int or string, #optional, default value "auto" + } + } + "golden_records": array[ids] #optional, default value is none which means use all of the golden records + "metadata": dict[str, str] | None} #optional, default value None + } + +**Responses** + +HTTP 201 code response + +.. code-block:: rst + + { + "id": "finetuing-job-id", + "db_connection_id": "database_connection_id" + "status": "finetuning_job_status" # queued is default other possible values are [queued, running, succeeded, failed, validating_files, or cancelled] + "error": "The error message if the job failed" # optional default value is None + "base_llm": { + "model_provider": "model_provider_name" # right now openai is the only provider. + "model_name": "model_name" # right now gpt-3.5-turbo and gpt-4 are suported. + "model_parameters": { + "n_epochs": int or string, #optional, default value 3 + "batch_size": int or string, #optional, default value 1 + "learning_rate_multiplier", int or string, #optional, default value "auto" + } + } + "finetuning_file_id": "This is the file id that is assigned by model provider when uploading the finetuning file" + "finetuning_job_id": "The is the finetuning job id that is assigned by model provider when creating the finetuning job" + "model_id": "The is the model id that is assigned by model provider after finetuning job is done" + "created_at": datetime, + "golden_records": array[ids], # default value is none which means use all of the golden records + "metadata": dict[str, str] | None} #optional, default value None + } + +**Request example** + +.. code-block:: rst + + curl -X 'POST' \ + 'http://localhost/api/v1/finetunings' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ + "db_connection_id": "database_connection_id", + "base_llm": { + "model_provider": "openai", + "model_name": "gpt-3.5-turbo-1106", + "model_parameters": { + "n_epochs": 1 + } + } + }' + + +**Response example** + +.. code-block:: rst + + { + "id": "finetuning-job-id", + "db_connection_id": "database_connection_id", + "status": "queued", + "error": null, + "base_llm": { + "model_provider": "openai", + "model_name": "gpt-3.5-turbo-1106", + "model_parameters": { + "n_epochs": "1" + } + }, + "finetuning_file_id": null, + "finetuning_job_id": null, + "model_id": null, + "created_at": "2023-12-13T17:35:53.263485", + "golden_records": [ # a list of golden record ids, keep in mind that at least 10 golden records are required for openai models finetuning + "656777c9eb4e22533dc43fce", + "656777ceeb4e22533dc43fcf", + "656777ceeb4e22533dc43fd0", + "656777ceeb4e22533dc43fd1", + "656777ceeb4e22533dc43fd2", + "656777ceeb4e22533dc43fd3", + "656777ceeb4e22533dc43fd4", + "656777ceeb4e22533dc43fd5", + "656777ceeb4e22533dc43fd6", + "656777ceeb4e22533dc43fd7", + "656777ceeb4e22533dc43fd8", + "656777ceeb4e22533dc43fd9", + "656777ceeb4e22533dc43fda", + "656777ceeb4e22533dc43fdb", + "656777ceeb4e22533dc43fdc", + "656777ceeb4e22533dc43fdd", + "656777ceeb4e22533dc43fde" + ], + "metadata": null + } diff --git a/docs/api.get_finetuning.rst b/docs/api.get_finetuning.rst new file mode 100644 index 00000000..2481d30f --- /dev/null +++ b/docs/api.get_finetuning.rst @@ -0,0 +1,86 @@ +Retrieving a fine-tuning job +============================ + +After creating a fine-tuning job, you can retrive the status of the job by calling the following API. When the status is `succeeded` you can use this model inside the agent. + +request this ``GET`` endpoint to retrieve the status of the fine-tuning job. + + /api/v1/finetuning-jobs/ + +**Responses** + +HTTP 200 code response + +.. code-block:: rst + + { + "id": "finetuing-job-id", + "db_connection_id": "database_connection_id" + "status": "finetuning_job_status" # queued is default other possible values are [queued, running, succeeded, failed, validating_files, or cancelled] + "error": "The error message if the job failed" # optional default value is None + "base_llm": { + "model_provider": "model_provider_name" # right now openai is the only provider. + "model_name": "model_name" # right now gpt-3.5-turbo and gpt-4 are suported. + "model_parameters": { + "n_epochs": int or string, #optional, default value 3 + "batch_size": int or string, #optional, default value 1 + "learning_rate_multiplier", int or string, #optional, default value "auto" + } + } + "finetuning_file_id": "This is the file id that is assigned by model provider when uploading the finetuning file" + "finetuning_job_id": "The is the finetuning job id that is assigned by model provider when creating the finetuning job" + "model_id": "The is the model id that is assigned by model provider after finetuning job is done" + "created_at": datetime, + "golden_records": array[ids], # default value is none which means use all of the golden records + "metadata": dict[str, str] | None} #optional, default value None + } + +**Request example** + +.. code-block:: rst + + curl -X 'GET' \ + 'http://localhost/api/v1/finetunings/{finetuning_id}?finetuning_job_id={finetuing-job-id}' \ + -H 'accept: application/json' + +**Response example** + +.. code-block:: rst + + { + "id": "finetuning-job-id", + "db_connection_id": "database_connection_id", + "status": "validating_files", + "error": null, + "base_llm": { + "model_provider": "openai", + "model_name": "gpt-3.5-turbo-1106", + "model_parameters": { + "n_epochs": "1" + } + }, + "finetuning_file_id": null, + "finetuning_job_id": null, + "model_id": null, + "created_at": "2023-12-13T17:35:53.263485", + "golden_records": [ # a list of golden record ids, keep in mind that at least 10 golden records are required for openai models finetuning + "656777c9eb4e22533dc43fce", + "656777ceeb4e22533dc43fcf", + "656777ceeb4e22533dc43fd0", + "656777ceeb4e22533dc43fd1", + "656777ceeb4e22533dc43fd2", + "656777ceeb4e22533dc43fd3", + "656777ceeb4e22533dc43fd4", + "656777ceeb4e22533dc43fd5", + "656777ceeb4e22533dc43fd6", + "656777ceeb4e22533dc43fd7", + "656777ceeb4e22533dc43fd8", + "656777ceeb4e22533dc43fd9", + "656777ceeb4e22533dc43fda", + "656777ceeb4e22533dc43fdb", + "656777ceeb4e22533dc43fdc", + "656777ceeb4e22533dc43fdd", + "656777ceeb4e22533dc43fde" + ], + "metadata": null + } \ No newline at end of file diff --git a/docs/api.rst b/docs/api.rst index 0072039a..e6ec1229 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -130,6 +130,43 @@ Related endpoints are: "instruction": "string", } +Finetuning jobs +--------------------- +The ``finetuning`` object is used to finetune the LLM to your data. This is an asynchronous process that uploads your golden records to model provider servers and creates a finetuning job. +The finetuned model is going to be used inside an agent for generating SQL queries. + +Related endpoints are: + +* :doc:`Finetuning job create ` -- ``POST api/v1/finetunings`` +* :doc:`Finetuning job get ` -- ``GET api/v1/finetunings/{finetuning_id}`` +* :doc:`Finetuning job cancel ` -- ``POST api/v1/finetunings/{finetuning_id}/cancel`` + + +**Finetuning resource example:** + +.. code-block:: json + + { + "id": "finetuing-job-id", + "db_connection_id": "database_connection_id", + "status": "finetuning_job_status", // Possible values: queued, running, succeeded, validating_files, failed, or cancelled + "error": "The error message if the job failed", // Optional, default is None + "base_llm": { + "model_provider": "model_provider_name", // Currently, only 'openai' + "model_name": "model_name", // Supported: gpt-3.5-turbo, gpt-4 + "model_parameters": { + "n_epochs": "int or string", // Optional, default 3 + "batch_size": "int or string", // Optional, default 1 + "learning_rate_multiplier": "int or string" // Optional, default "auto" + } + }, + "finetuning_file_id": "File ID for finetuning file", + "finetuning_job_id": "Finetuning job ID", + "model_id": "Model ID after finetuning", + "created_at": "datetime", + "golden_records": "array[ids]", // Default is None, meaning use all golden records + "metadata": "dict[str, str] | None" // Optional, default None + } .. toctree:: :hidden: @@ -158,3 +195,7 @@ Related endpoints are: api.list_responses api.get_response api.get_response_file + + api.finetuning + api.get_finetuning + api.cancel_finetuning From def6fba9d336d44a17b86d9617ad4108eaf5e2a9 Mon Sep 17 00:00:00 2001 From: mohammadrezapourreza Date: Wed, 13 Dec 2023 13:22:55 -0500 Subject: [PATCH 5/6] DH-5033/update the docs --- docs/api.cancel_finetuning.rst | 2 +- docs/api.get_finetuning.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/api.cancel_finetuning.rst b/docs/api.cancel_finetuning.rst index 52f514dc..eb2de7b8 100644 --- a/docs/api.cancel_finetuning.rst +++ b/docs/api.cancel_finetuning.rst @@ -5,7 +5,7 @@ This endpoint allows you to cancel a finetuning job, which will stop the trainin request this ``POST`` endpoint to cancel a finetuning job. - /api/v1/finetuning-jobs//cancel + /api/v1/finetunings//cancel **Request body** diff --git a/docs/api.get_finetuning.rst b/docs/api.get_finetuning.rst index 2481d30f..6a87b432 100644 --- a/docs/api.get_finetuning.rst +++ b/docs/api.get_finetuning.rst @@ -5,7 +5,7 @@ After creating a fine-tuning job, you can retrive the status of the job by calli request this ``GET`` endpoint to retrieve the status of the fine-tuning job. - /api/v1/finetuning-jobs/ + /api/v1/finetunings/ **Responses** From 7a307f25c4cbb35ab15e9d234aaf859549945682 Mon Sep 17 00:00:00 2001 From: mohammadrezapourreza Date: Wed, 13 Dec 2023 16:08:56 -0500 Subject: [PATCH 6/6] DH-5033/add alias --- dataherald/api/fastapi.py | 1 + dataherald/types.py | 2 ++ docs/api.cancel_finetuning.rst | 2 ++ docs/api.finetuning.rst | 6 +++++- docs/api.get_finetuning.rst | 2 ++ docs/api.rst | 1 + 6 files changed, 13 insertions(+), 1 deletion(-) diff --git a/dataherald/api/fastapi.py b/dataherald/api/fastapi.py index dc1c50a9..5fe9173a 100644 --- a/dataherald/api/fastapi.py +++ b/dataherald/api/fastapi.py @@ -692,6 +692,7 @@ def create_finetuning_job( model = model_repository.insert( Finetuning( db_connection_id=fine_tuning_request.db_connection_id, + alias=fine_tuning_request.alias, base_llm=fine_tuning_request.base_llm, golden_records=[ str(golden_record.id) for golden_record in golden_records diff --git a/dataherald/types.py b/dataherald/types.py index 99efe9e4..df67afe1 100644 --- a/dataherald/types.py +++ b/dataherald/types.py @@ -158,6 +158,7 @@ class BaseLLM(BaseModel): class Finetuning(BaseModel): id: str | None = None + alias: str | None = None db_connection_id: str | None = None status: str = "queued" error: str | None = None @@ -172,6 +173,7 @@ class Finetuning(BaseModel): class FineTuningRequest(BaseModel): db_connection_id: str + alias: str base_llm: BaseLLM golden_records: list[str] | None = None metadata: dict[str, str] | None = None diff --git a/docs/api.cancel_finetuning.rst b/docs/api.cancel_finetuning.rst index eb2de7b8..a4798068 100644 --- a/docs/api.cancel_finetuning.rst +++ b/docs/api.cancel_finetuning.rst @@ -25,6 +25,7 @@ HTTP 200 code response { "id": "finetuing-job-id", "db_connection_id": "database_connection_id" + "alias": "model name" "status": "finetuning_job_status" # queued is default other possible values are [queued, running, succeeded, failed, validating_files, or cancelled] "error": "The error message if the job failed" # optional default value is None "base_llm": { @@ -63,6 +64,7 @@ HTTP 200 code response { "id": "finetuning-job-id", + "alias": "my_model", "db_connection_id": "database_connection_id", "status": "cancelled", "error": "Fine tuning cancelled by the user", diff --git a/docs/api.finetuning.rst b/docs/api.finetuning.rst index 95f2770e..f5ab211e 100644 --- a/docs/api.finetuning.rst +++ b/docs/api.finetuning.rst @@ -16,6 +16,7 @@ Request this ``POST`` endpoint to create a finetuning job:: { "db_connection_id": "database_connection_id" + "alias": "model name", "base_llm": { "model_provider": "model_provider_name" # right now openai is the only provider. "model_name": "model_name" # right now gpt-3.5-turbo and gpt-4 are suported. @@ -37,7 +38,8 @@ HTTP 201 code response { "id": "finetuing-job-id", - "db_connection_id": "database_connection_id" + "db_connection_id": "database_connection_id", + "alias": "model name", "status": "finetuning_job_status" # queued is default other possible values are [queued, running, succeeded, failed, validating_files, or cancelled] "error": "The error message if the job failed" # optional default value is None "base_llm": { @@ -67,6 +69,7 @@ HTTP 201 code response -H 'Content-Type: application/json' \ -d '{ "db_connection_id": "database_connection_id", + "alias": "my_model", "base_llm": { "model_provider": "openai", "model_name": "gpt-3.5-turbo-1106", @@ -83,6 +86,7 @@ HTTP 201 code response { "id": "finetuning-job-id", + "alias": "my_model", "db_connection_id": "database_connection_id", "status": "queued", "error": null, diff --git a/docs/api.get_finetuning.rst b/docs/api.get_finetuning.rst index 6a87b432..af3528c5 100644 --- a/docs/api.get_finetuning.rst +++ b/docs/api.get_finetuning.rst @@ -16,6 +16,7 @@ HTTP 200 code response { "id": "finetuing-job-id", "db_connection_id": "database_connection_id" + "alias": "model name" "status": "finetuning_job_status" # queued is default other possible values are [queued, running, succeeded, failed, validating_files, or cancelled] "error": "The error message if the job failed" # optional default value is None "base_llm": { @@ -50,6 +51,7 @@ HTTP 200 code response { "id": "finetuning-job-id", "db_connection_id": "database_connection_id", + "alias": "my_model", "status": "validating_files", "error": null, "base_llm": { diff --git a/docs/api.rst b/docs/api.rst index e6ec1229..d546b2fc 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -149,6 +149,7 @@ Related endpoints are: { "id": "finetuing-job-id", "db_connection_id": "database_connection_id", + "alias": "model name", "status": "finetuning_job_status", // Possible values: queued, running, succeeded, validating_files, failed, or cancelled "error": "The error message if the job failed", // Optional, default is None "base_llm": {