From 8557efca913528251eabdfcd9f89583b1232e36b Mon Sep 17 00:00:00 2001 From: KATTA-00 Date: Tue, 13 Aug 2024 11:28:24 +0530 Subject: [PATCH 01/32] Add a temp example --- examples/temp/.devcontainer.json | 4 + examples/temp/.gitignore | 11 + examples/temp/Ballerina.toml | 8 + examples/temp/Dependencies.toml | 310 ++++++++++++++++++++++++++ examples/temp/client.bal | 222 +++++++++++++++++++ examples/temp/main.bal | 106 +++++++++ examples/temp/training.jsonl | 10 + examples/temp/types.bal | 361 +++++++++++++++++++++++++++++++ examples/temp/utils.bal | 252 +++++++++++++++++++++ examples/temp/validation.jsonl | 8 + 10 files changed, 1292 insertions(+) create mode 100644 examples/temp/.devcontainer.json create mode 100644 examples/temp/.gitignore create mode 100644 examples/temp/Ballerina.toml create mode 100644 examples/temp/Dependencies.toml create mode 100644 examples/temp/client.bal create mode 100644 examples/temp/main.bal create mode 100644 examples/temp/training.jsonl create mode 100644 examples/temp/types.bal create mode 100644 examples/temp/utils.bal create mode 100644 examples/temp/validation.jsonl diff --git a/examples/temp/.devcontainer.json b/examples/temp/.devcontainer.json new file mode 100644 index 0000000..fd48abe --- /dev/null +++ b/examples/temp/.devcontainer.json @@ -0,0 +1,4 @@ +{ + "image": "ballerina/ballerina-devcontainer:2201.9.2", + "extensions": ["WSO2.ballerina"], +} diff --git a/examples/temp/.gitignore b/examples/temp/.gitignore new file mode 100644 index 0000000..d5fc29a --- /dev/null +++ b/examples/temp/.gitignore @@ -0,0 +1,11 @@ +# Ballerina generates this directory during the compilation of a package. +# It contains compiler-generated artifacts and the final executable if this is an application package. +target/ + +# Ballerina maintains the compiler-generated source code here. +# Remove this if you want to commit generated sources. +generated/ + +# Contains configuration values used during development time. +# See https://ballerina.io/learn/provide-values-to-configurable-variables/ for more details. +Config.toml diff --git a/examples/temp/Ballerina.toml b/examples/temp/Ballerina.toml new file mode 100644 index 0000000..d7bf1b8 --- /dev/null +++ b/examples/temp/Ballerina.toml @@ -0,0 +1,8 @@ +[package] +org = "kanishka" +name = "temp" +version = "0.1.0" +distribution = "2201.9.2" + +[build-options] +observabilityIncluded = true diff --git a/examples/temp/Dependencies.toml b/examples/temp/Dependencies.toml new file mode 100644 index 0000000..d7d14c3 --- /dev/null +++ b/examples/temp/Dependencies.toml @@ -0,0 +1,310 @@ +# AUTO-GENERATED FILE. DO NOT MODIFY. + +# This file is auto-generated by Ballerina for managing dependency versions. +# It should not be modified by hand. + +[ballerina] +dependencies-toml-version = "2" +distribution-version = "2201.9.2" + +[[package]] +org = "ballerina" +name = "auth" +version = "2.11.2" +dependencies = [ + {org = "ballerina", name = "crypto"}, + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.array"}, + {org = "ballerina", name = "lang.string"}, + {org = "ballerina", name = "log"} +] + +[[package]] +org = "ballerina" +name = "cache" +version = "3.8.0" +dependencies = [ + {org = "ballerina", name = "constraint"}, + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "task"}, + {org = "ballerina", name = "time"} +] + +[[package]] +org = "ballerina" +name = "constraint" +version = "1.5.0" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] + +[[package]] +org = "ballerina" +name = "crypto" +version = "2.7.2" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "time"} +] + +[[package]] +org = "ballerina" +name = "file" +version = "1.9.0" +dependencies = [ + {org = "ballerina", name = "io"}, + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "os"}, + {org = "ballerina", name = "time"} +] + +[[package]] +org = "ballerina" +name = "http" +version = "2.11.3" +dependencies = [ + {org = "ballerina", name = "auth"}, + {org = "ballerina", name = "cache"}, + {org = "ballerina", name = "constraint"}, + {org = "ballerina", name = "crypto"}, + {org = "ballerina", name = "file"}, + {org = "ballerina", name = "io"}, + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "jwt"}, + {org = "ballerina", name = "lang.array"}, + {org = "ballerina", name = "lang.decimal"}, + {org = "ballerina", name = "lang.int"}, + {org = "ballerina", name = "lang.regexp"}, + {org = "ballerina", name = "lang.runtime"}, + {org = "ballerina", name = "lang.string"}, + {org = "ballerina", name = "lang.value"}, + {org = "ballerina", name = "log"}, + {org = "ballerina", name = "mime"}, + {org = "ballerina", name = "oauth2"}, + {org = "ballerina", name = "observe"}, + {org = "ballerina", name = "time"}, + {org = "ballerina", name = "url"} +] +modules = [ + {org = "ballerina", packageName = "http", moduleName = "http"}, + {org = "ballerina", packageName = "http", moduleName = "http.httpscerr"} +] + +[[package]] +org = "ballerina" +name = "io" +version = "1.6.1" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.value"} +] +modules = [ + {org = "ballerina", packageName = "io", moduleName = "io"} +] + +[[package]] +org = "ballerina" +name = "jballerina.java" +version = "0.0.0" + +[[package]] +org = "ballerina" +name = "jwt" +version = "2.11.0" +dependencies = [ + {org = "ballerina", name = "cache"}, + {org = "ballerina", name = "crypto"}, + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.int"}, + {org = "ballerina", name = "lang.string"}, + {org = "ballerina", name = "log"}, + {org = "ballerina", name = "time"} +] + +[[package]] +org = "ballerina" +name = "lang.__internal" +version = "0.0.0" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.object"} +] + +[[package]] +org = "ballerina" +name = "lang.array" +version = "0.0.0" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.__internal"} +] + +[[package]] +org = "ballerina" +name = "lang.decimal" +version = "0.0.0" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] + +[[package]] +org = "ballerina" +name = "lang.int" +version = "0.0.0" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.__internal"}, + {org = "ballerina", name = "lang.object"} +] + +[[package]] +org = "ballerina" +name = "lang.object" +version = "0.0.0" + +[[package]] +org = "ballerina" +name = "lang.regexp" +version = "0.0.0" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] + +[[package]] +org = "ballerina" +name = "lang.runtime" +version = "0.0.0" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] +modules = [ + {org = "ballerina", packageName = "lang.runtime", moduleName = "lang.runtime"} +] + +[[package]] +org = "ballerina" +name = "lang.string" +version = "0.0.0" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.regexp"} +] + +[[package]] +org = "ballerina" +name = "lang.value" +version = "0.0.0" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] + +[[package]] +org = "ballerina" +name = "log" +version = "2.9.0" +dependencies = [ + {org = "ballerina", name = "io"}, + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.value"}, + {org = "ballerina", name = "observe"} +] + +[[package]] +org = "ballerina" +name = "mime" +version = "2.9.0" +dependencies = [ + {org = "ballerina", name = "io"}, + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.int"} +] +modules = [ + {org = "ballerina", packageName = "mime", moduleName = "mime"} +] + +[[package]] +org = "ballerina" +name = "oauth2" +version = "2.11.0" +dependencies = [ + {org = "ballerina", name = "cache"}, + {org = "ballerina", name = "crypto"}, + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "log"}, + {org = "ballerina", name = "time"}, + {org = "ballerina", name = "url"} +] + +[[package]] +org = "ballerina" +name = "observe" +version = "1.2.3" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] + +[[package]] +org = "ballerina" +name = "os" +version = "1.8.0" +dependencies = [ + {org = "ballerina", name = "io"}, + {org = "ballerina", name = "jballerina.java"} +] + +[[package]] +org = "ballerina" +name = "task" +version = "2.5.0" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "time"} +] + +[[package]] +org = "ballerina" +name = "time" +version = "2.4.0" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] + +[[package]] +org = "ballerina" +name = "url" +version = "2.4.0" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] +modules = [ + {org = "ballerina", packageName = "url", moduleName = "url"} +] + +[[package]] +org = "ballerinai" +name = "observe" +version = "0.0.0" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "observe"} +] +modules = [ + {org = "ballerinai", packageName = "observe", moduleName = "observe"} +] + +[[package]] +org = "kanishka" +name = "temp" +version = "0.1.0" +dependencies = [ + {org = "ballerina", name = "http"}, + {org = "ballerina", name = "io"}, + {org = "ballerina", name = "lang.runtime"}, + {org = "ballerina", name = "mime"}, + {org = "ballerina", name = "url"}, + {org = "ballerinai", name = "observe"} +] +modules = [ + {org = "kanishka", packageName = "temp", moduleName = "temp"} +] + diff --git a/examples/temp/client.bal b/examples/temp/client.bal new file mode 100644 index 0000000..72c21eb --- /dev/null +++ b/examples/temp/client.bal @@ -0,0 +1,222 @@ +// AUTO-GENERATED FILE. DO NOT MODIFY. +// This file is auto-generated by the Ballerina OpenAPI tool. + +// Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import ballerina/http; +import ballerina/mime; + +# The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. +public isolated client class Client { + final http:Client clientEp; + # Gets invoked to initialize the `connector`. + # + # + config - The configurations to be used when initializing the `connector` + # + serviceUrl - URL of the target service + # + return - An error if connector initialization failed + public isolated function init(ConnectionConfig config, string serviceUrl = "https://api.openai.com/v1") returns error? { + http:ClientConfiguration httpClientConfig = {auth: config.auth, httpVersion: config.httpVersion, timeout: config.timeout, forwarded: config.forwarded, poolConfig: config.poolConfig, compression: config.compression, circuitBreaker: config.circuitBreaker, retryConfig: config.retryConfig, validation: config.validation}; + do { + if config.http1Settings is ClientHttp1Settings { + ClientHttp1Settings settings = check config.http1Settings.ensureType(ClientHttp1Settings); + httpClientConfig.http1Settings = {...settings}; + } + if config.http2Settings is http:ClientHttp2Settings { + httpClientConfig.http2Settings = check config.http2Settings.ensureType(http:ClientHttp2Settings); + } + if config.cache is http:CacheConfig { + httpClientConfig.cache = check config.cache.ensureType(http:CacheConfig); + } + if config.responseLimits is http:ResponseLimitConfigs { + httpClientConfig.responseLimits = check config.responseLimits.ensureType(http:ResponseLimitConfigs); + } + if config.secureSocket is http:ClientSecureSocket { + httpClientConfig.secureSocket = check config.secureSocket.ensureType(http:ClientSecureSocket); + } + if config.proxy is http:ProxyConfig { + httpClientConfig.proxy = check config.proxy.ensureType(http:ProxyConfig); + } + } + http:Client httpEp = check new (serviceUrl, httpClientConfig); + self.clientEp = httpEp; + return; + } + + # Delete a file. + # + # + file_id - The ID of the file to use for this request. + # + headers - Headers to be sent with the request + # + return - OK + resource isolated function delete files/[string file_id](map headers = {}) returns DeleteFileResponse|error { + string resourcePath = string `/files/${getEncodedUri(file_id)}`; + return self.clientEp->delete(resourcePath, headers = headers); + } + + # Delete a fine-tuned model. You must have the Owner role in your organization to delete a model. + # + # + model - The model to delete + # + headers - Headers to be sent with the request + # + return - OK + resource isolated function delete models/[string model](map headers = {}) returns DeleteModelResponse|error { + string resourcePath = string `/models/${getEncodedUri(model)}`; + return self.clientEp->delete(resourcePath, headers = headers); + } + + # Returns a list of files that belong to the user's organization. + # + # + headers - Headers to be sent with the request + # + queries - Queries to be sent with the request + # + return - OK + resource isolated function get files(map headers = {}, *ListFilesQueries queries) returns ListFilesResponse|error { + string resourcePath = string `/files`; + resourcePath = resourcePath + check getPathForQueryParam(queries); + return self.clientEp->get(resourcePath, headers); + } + + # Returns information about a specific file. + # + # + file_id - The ID of the file to use for this request. + # + headers - Headers to be sent with the request + # + return - OK + resource isolated function get files/[string file_id](map headers = {}) returns OpenAIFile|error { + string resourcePath = string `/files/${getEncodedUri(file_id)}`; + return self.clientEp->get(resourcePath, headers); + } + + # Returns the contents of the specified file. + # + # + file_id - The ID of the file to use for this request. + # + headers - Headers to be sent with the request + # + return - OK + resource isolated function get files/[string file_id]/content(map headers = {}) returns byte[]|error { + string resourcePath = string `/files/${getEncodedUri(file_id)}/content`; + return self.clientEp->get(resourcePath, headers); + } + + # List your organization's fine-tuning jobs + # + # + headers - Headers to be sent with the request + # + queries - Queries to be sent with the request + # + return - OK + resource isolated function get fine_tuning/jobs(map headers = {}, *ListPaginatedFineTuningJobsQueries queries) returns ListPaginatedFineTuningJobsResponse|error { + string resourcePath = string `/fine_tuning/jobs`; + resourcePath = resourcePath + check getPathForQueryParam(queries); + return self.clientEp->get(resourcePath, headers); + } + + # Get info about a fine-tuning job. + # + # [Learn more about fine-tuning](/docs/guides/fine-tuning) + # + # + fine_tuning_job_id - The ID of the fine-tuning job. + # + headers - Headers to be sent with the request + # + return - OK + resource isolated function get fine_tuning/jobs/[string fine_tuning_job_id](map headers = {}) returns FineTuningJob|error { + string resourcePath = string `/fine_tuning/jobs/${getEncodedUri(fine_tuning_job_id)}`; + return self.clientEp->get(resourcePath, headers); + } + + # List checkpoints for a fine-tuning job. + # + # + fine_tuning_job_id - The ID of the fine-tuning job to get checkpoints for. + # + headers - Headers to be sent with the request + # + queries - Queries to be sent with the request + # + return - OK + resource isolated function get fine_tuning/jobs/[string fine_tuning_job_id]/checkpoints(map headers = {}, *ListFineTuningJobCheckpointsQueries queries) returns ListFineTuningJobCheckpointsResponse|error { + string resourcePath = string `/fine_tuning/jobs/${getEncodedUri(fine_tuning_job_id)}/checkpoints`; + resourcePath = resourcePath + check getPathForQueryParam(queries); + return self.clientEp->get(resourcePath, headers); + } + + # Get status updates for a fine-tuning job. + # + # + fine_tuning_job_id - The ID of the fine-tuning job to get events for. + # + headers - Headers to be sent with the request + # + queries - Queries to be sent with the request + # + return - OK + resource isolated function get fine_tuning/jobs/[string fine_tuning_job_id]/events(map headers = {}, *ListFineTuningEventsQueries queries) returns ListFineTuningJobEventsResponse|error { + string resourcePath = string `/fine_tuning/jobs/${getEncodedUri(fine_tuning_job_id)}/events`; + resourcePath = resourcePath + check getPathForQueryParam(queries); + return self.clientEp->get(resourcePath, headers); + } + + # Lists the currently available models, and provides basic information about each one such as the owner and availability. + # + # + headers - Headers to be sent with the request + # + return - OK + resource isolated function get models(map headers = {}) returns ListModelsResponse|error { + string resourcePath = string `/models`; + return self.clientEp->get(resourcePath, headers); + } + + # Retrieves a model instance, providing basic information about the model such as the owner and permissioning. + # + # + model - The ID of the model to use for this request + # + headers - Headers to be sent with the request + # + return - OK + resource isolated function get models/[string model](map headers = {}) returns Model|error { + string resourcePath = string `/models/${getEncodedUri(model)}`; + return self.clientEp->get(resourcePath, headers); + } + + # Upload a file that can be used across various endpoints. Individual files can be up to 512 MB, and the size of all files uploaded by one organization can be up to 100 GB. + # + # The Assistants API supports files up to 2 million tokens and of specific file types. See the [Assistants Tools guide](/docs/assistants/tools) for details. + # + # The Fine-tuning API only supports `.jsonl` files. The input also has certain required formats for fine-tuning [chat](/docs/api-reference/fine-tuning/chat-input) or [completions](/docs/api-reference/fine-tuning/completions-input) models. + # + # The Batch API only supports `.jsonl` files up to 100 MB in size. The input also has a specific required [format](/docs/api-reference/batch/request-input). + # + # Please [contact us](https://help.openai.com/) if you need to increase these storage limits. + # + # + headers - Headers to be sent with the request + # + return - OK + resource isolated function post files(CreateFileRequest payload, map headers = {}) returns OpenAIFile|error { + string resourcePath = string `/files`; + http:Request request = new; + mime:Entity[] bodyParts = check createBodyParts(payload); + request.setBodyParts(bodyParts); + return self.clientEp->post(resourcePath, request, headers); + } + + # Creates a fine-tuning job which begins the process of creating a new model from a given dataset. + # + # Response includes details of the enqueued job including job status and the name of the fine-tuned models once complete. + # + # [Learn more about fine-tuning](/docs/guides/fine-tuning) + # + # + headers - Headers to be sent with the request + # + return - OK + resource isolated function post fine_tuning/jobs(CreateFineTuningJobRequest payload, map headers = {}) returns FineTuningJob|error { + string resourcePath = string `/fine_tuning/jobs`; + http:Request request = new; + json jsonBody = payload.toJson(); + request.setPayload(jsonBody, "application/json"); + return self.clientEp->post(resourcePath, request, headers); + } + + # Immediately cancel a fine-tune job. + # + # + fine_tuning_job_id - The ID of the fine-tuning job to cancel. + # + headers - Headers to be sent with the request + # + return - OK + resource isolated function post fine_tuning/jobs/[string fine_tuning_job_id]/cancel(map headers = {}) returns FineTuningJob|error { + string resourcePath = string `/fine_tuning/jobs/${getEncodedUri(fine_tuning_job_id)}/cancel`; + http:Request request = new; + return self.clientEp->post(resourcePath, request, headers); + } +} diff --git a/examples/temp/main.bal b/examples/temp/main.bal new file mode 100644 index 0000000..215742a --- /dev/null +++ b/examples/temp/main.bal @@ -0,0 +1,106 @@ +import ballerina/io; +import ballerina/lang.runtime; + +// Configurable variables for environment setup. +configurable string token = ?; +string serviceUrl = "https://api.openai.com/v1"; + +// Configurable variables for file names. +string trainingFileName = "training.jsonl"; +string validationFileName = "validation.jsonl"; + +// Initialize the connection configuration and client. +final ConnectionConfig config = {auth: {token: token}}; +final Client openAIFinetunes = check new Client(config, serviceUrl); + +public function main() returns error? { + + // Read the training file content. + // byte[] trainingFileContent = check io:fileReadBytes(trainingFileName); + // byte[] validationFileContent = check io:fileReadBytes(validationFileName); + + // CreateFileRequest trainingFileRequest = { + // file: {fileContent: trainingFileContent, fileName: trainingFileName}, + // purpose: "fine-tune" + // }; + + // CreateFileRequest validationFileRequest = { + // file: {fileContent: validationFileContent, fileName: validationFileName}, + // purpose: "fine-tune" + // }; + + // OpenAIFile trainingFileResponse = + // check openAIFinetunes->/files.post(trainingFileRequest); + + // OpenAIFile validationFileResponse = + // check openAIFinetunes->/files.post(validationFileRequest); + + // string trainingFileId = trainingFileResponse.id; + // string validationFileId = validationFileResponse.id; + + // io:println("Training file id: " + trainingFileId); + // io:println("Validation file id: " + validationFileId); + + // // Create a fine-tuning job. + // CreateFineTuningJobRequest fineTuneRequest = { + // model: "gpt-3.5-turbo", + // training_file: trainingFileId, + // validation_file: validationFileId, + // hyperparameters: { + // n_epochs: 15, + // batch_size: 3, + // learning_rate_multiplier: 0.3 + // } + // }; + + // FineTuningJob fineTuneResponse = + // check openAIFinetunes->/fine_tuning/jobs.post(fineTuneRequest); + + // string fineTuneJobId = fineTuneResponse.id; + + string fineTuneJobId = "ftjob-IeZPVQ2AloE42006RyNDtFKR"; + io:println("Fine-tuning job id: " + fineTuneJobId); + + FineTuningJob fineTuneJob = + check openAIFinetunes->/fine_tuning/jobs/[fineTuneJobId].get(); + + io:print("Validating files..."); + while (fineTuneJob.status == "validating_files") { + io:print("."); + fineTuneJob = check openAIFinetunes->/fine_tuning/jobs/[fineTuneJobId].get(); + + runtime:sleep(1); + } + + io:print("\nFiles validated successfully."); + + while (fineTuneJob.status == "queued") { + io:print("."); + fineTuneJob = check openAIFinetunes->/fine_tuning/jobs/[fineTuneJobId].get(); + + runtime:sleep(1); + } + + io:println("\nTraining..."); + while (fineTuneJob.status == "running") { + fineTuneJob = check openAIFinetunes->/fine_tuning/jobs/[fineTuneJobId].get(); + + ListFineTuningJobEventsResponse eventsResponse = + check openAIFinetunes->/fine_tuning/jobs/[fineTuneJobId]/events.get(); + + io:println(eventsResponse.data[0].message); + runtime:sleep(1); + } + + if (fineTuneJob.status != "succeeded") { + io:println("Fine-tuning job failed."); + return; + } + + io:println("\nFine-tuning job details: "); + io:println("Fine-tuned Model: ", fineTuneJob.fine_tuned_model); + io:println("Organization ID: ", fineTuneJob.organization_id); + io:println("Model: ", fineTuneJob.model); + io:println("Fine-tuning job completed successfully."); + +} diff --git a/examples/temp/training.jsonl b/examples/temp/training.jsonl new file mode 100644 index 0000000..c1a9c70 --- /dev/null +++ b/examples/temp/training.jsonl @@ -0,0 +1,10 @@ +{"messages": [{"role": "user", "content": "What is the capital of France?"}, {"role": "assistant", "content": "The capital of France is Paris."}]} +{"messages": [{"role": "user", "content": "What is the primary function of the heart?"}, {"role": "assistant", "content": "The primary function of the heart is to pump blood throughout the body."}]} +{"messages": [{"role": "user", "content": "What is photosynthesis?"}, {"role": "assistant", "content": "Photosynthesis is the process by which green plants and some other organisms convert sunlight into chemical energy stored in the form of glucose."}]} +{"messages": [{"role": "user", "content": "Who wrote the play 'Romeo and Juliet'?"}, {"role": "assistant", "content": "William Shakespeare wrote the play 'Romeo and Juliet'."}]} +{"messages": [{"role": "user", "content": "Which element has the atomic number 1?"}, {"role": "assistant", "content": "Hydrogen has the atomic number 1."}]} +{"messages": [{"role": "user", "content": "What is the largest planet in our solar system?"}, {"role": "assistant", "content": "Jupiter is the largest planet in our solar system."}]} +{"messages": [{"role": "user", "content": "What is the freezing point of water in Celsius?"}, {"role": "assistant", "content": "The freezing point of water in Celsius is 0 degrees."}]} +{"messages": [{"role": "user", "content": "What is the square root of 144?"}, {"role": "assistant", "content": "The square root of 144 is 12."}]} +{"messages": [{"role": "user", "content": "Who is the author of 'To Kill a Mockingbird'?"}, {"role": "assistant", "content": "The author of 'To Kill a Mockingbird' is Harper Lee."}]} +{"messages": [{"role": "user", "content": "What is the smallest unit of life?"}, {"role": "assistant", "content": "The smallest unit of life is the cell."}]} diff --git a/examples/temp/types.bal b/examples/temp/types.bal new file mode 100644 index 0000000..093632d --- /dev/null +++ b/examples/temp/types.bal @@ -0,0 +1,361 @@ +// AUTO-GENERATED FILE. DO NOT MODIFY. +// This file is auto-generated by the Ballerina OpenAPI tool. + +// Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import ballerina/http; + +# The `File` object represents a document that has been uploaded to OpenAI. +public type OpenAIFile record { + # The file identifier, which can be referenced in the API endpoints. + string id; + # The size of the file, in bytes. + int bytes; + # The Unix timestamp (in seconds) for when the file was created. + int created_at; + # The name of the file. + string filename; + # The object type, which is always `file`. + "file" 'object; + # The intended purpose of the file. Supported values are `assistants`, `assistants_output`, `batch`, `batch_output`, `fine-tune`, `fine-tune-results` and `vision`. + "assistants"|"assistants_output"|"batch"|"batch_output"|"fine-tune"|"fine-tune-results"|"vision" purpose; + # Deprecated. The current status of the file, which can be either `uploaded`, `processed`, or `error`. + # + # # Deprecated + @deprecated + "uploaded"|"processed"|"error" status; + # Deprecated. For details on why a fine-tuning training file failed validation, see the `error` field on `fine_tuning.job`. + string? status_details?; +}; + +public type CreateFineTuningJobRequest record { + # The name of the model to fine-tune. You can select one of the + # [supported models](/docs/guides/fine-tuning/what-models-can-be-fine-tuned). + string|"babbage-002"|"davinci-002"|"gpt-3.5-turbo" model; + # The ID of an uploaded file that contains training data. + # + # See [upload file](/docs/api-reference/files/create) for how to upload a file. + # + # Your dataset must be formatted as a JSONL file. Additionally, you must upload your file with the purpose `fine-tune`. + # + # The contents of the file should differ depending on if the model uses the [chat](/docs/api-reference/fine-tuning/chat-input) or [completions](/docs/api-reference/fine-tuning/completions-input) format. + # + # See the [fine-tuning guide](/docs/guides/fine-tuning) for more details. + string training_file; + CreateFineTuningJobRequest_hyperparameters hyperparameters?; + # A string of up to 18 characters that will be added to your fine-tuned model name. + # + # For example, a `suffix` of "custom-model-name" would produce a model name like `ft:gpt-3.5-turbo:openai:custom-model-name:7p4lURel`. + string? suffix?; + # The ID of an uploaded file that contains validation data. + # + # If you provide this file, the data is used to generate validation + # metrics periodically during fine-tuning. These metrics can be viewed in + # the fine-tuning results file. + # The same data should not be present in both train and validation files. + # + # Your dataset must be formatted as a JSONL file. You must upload your file with the purpose `fine-tune`. + # + # See the [fine-tuning guide](/docs/guides/fine-tuning) for more details. + string? validation_file?; + # A list of integrations to enable for your fine-tuning job. + CreateFineTuningJobRequest_integrations[]? integrations?; + # The seed controls the reproducibility of the job. Passing in the same seed and job parameters should produce the same results, but may differ in rare cases. + # If a seed is not specified, one will be generated for you. + int? seed?; +}; + +# The `fine_tuning.job.checkpoint` object represents a model checkpoint for a fine-tuning job that is ready to use. +public type FineTuningJobCheckpoint record { + # The checkpoint identifier, which can be referenced in the API endpoints. + string id; + # The Unix timestamp (in seconds) for when the checkpoint was created. + int created_at; + # The name of the fine-tuned checkpoint model that is created. + string fine_tuned_model_checkpoint; + # The step number that the checkpoint was created at. + int step_number; + FineTuningJobCheckpoint_metrics metrics; + # The name of the fine-tuning job that this checkpoint was created from. + string fine_tuning_job_id; + # The object type, which is always "fine_tuning.job.checkpoint". + "fine_tuning.job.checkpoint" 'object; +}; + +public type ListPaginatedFineTuningJobsResponse record { + FineTuningJob[] data; + boolean has_more; + "list" 'object; +}; + +# The hyperparameters used for the fine-tuning job. +public type CreateFineTuningJobRequest_hyperparameters record { + # Number of examples in each batch. A larger batch size means that model parameters + # are updated less frequently, but with lower variance. + "auto"|int batch_size = "auto"; + # Scaling factor for the learning rate. A smaller learning rate may be useful to avoid + # overfitting. + "auto"|decimal learning_rate_multiplier = "auto"; + # The number of epochs to train the model for. An epoch refers to one full cycle + # through the training dataset. + "auto"|int n_epochs = "auto"; +}; + +# Represents the Queries record for the operation: listFineTuningEvents +public type ListFineTuningEventsQueries record { + # Number of events to retrieve. + int 'limit = 20; + # Identifier for the last event from the previous pagination request. + string after?; +}; + +# Represents the Queries record for the operation: listFineTuningJobCheckpoints +public type ListFineTuningJobCheckpointsQueries record { + # Number of checkpoints to retrieve. + int 'limit = 10; + # Identifier for the last checkpoint ID from the previous pagination request. + string after?; +}; + +# The settings for your integration with Weights and Biases. This payload specifies the project that +# metrics will be sent to. Optionally, you can set an explicit display name for your run, add tags +# to your run, and set a default entity (team, username, etc) to be associated with your run. +public type CreateFineTuningJobRequest_wandb record { + # The name of the project that the new run will be created under. + string project; + # A display name to set for the run. If not set, we will use the Job ID as the name. + string? name?; + # The entity to use for the run. This allows you to set the team or username of the WandB user that you would + # like associated with the run. If not set, the default entity for the registered WandB API key is used. + string? entity?; + # A list of tags to be attached to the newly created run. These tags are passed through directly to WandB. Some + # default tags are generated by OpenAI: "openai/finetune", "openai/{base-model}", "openai/{ftjob-abcdef}". + string[] tags?; +}; + +# The hyperparameters used for the fine-tuning job. See the [fine-tuning guide](/docs/guides/fine-tuning) for more details. +public type FineTuningJob_hyperparameters record { + # The number of epochs to train the model for. An epoch refers to one full cycle through the training dataset. + # "auto" decides the optimal number of epochs based on the size of the dataset. If setting the number manually, we support any number between 1 and 50 epochs. + "auto"|int n_epochs; +}; + +# Represents the Queries record for the operation: listFiles +public type ListFilesQueries record { + # Only return files with the given purpose. + string purpose?; +}; + +public type ListFineTuningJobCheckpointsResponse record { + FineTuningJobCheckpoint[] data; + "list" 'object; + string? first_id?; + string? last_id?; + boolean has_more; +}; + +public type DeleteModelResponse record { + string id; + boolean deleted; + string 'object; +}; + +public type FineTuningIntegration record { + # The type of the integration being enabled for the fine-tuning job + "wandb" 'type; + CreateFineTuningJobRequest_wandb wandb; +}; + +# Provides a set of configurations for controlling the behaviours when communicating with a remote HTTP endpoint. +@display {label: "Connection Config"} +public type ConnectionConfig record {| + # Configurations related to client authentication + http:BearerTokenConfig auth; + # The HTTP version understood by the client + http:HttpVersion httpVersion = http:HTTP_2_0; + # Configurations related to HTTP/1.x protocol + ClientHttp1Settings http1Settings?; + # Configurations related to HTTP/2 protocol + http:ClientHttp2Settings http2Settings?; + # The maximum time to wait (in seconds) for a response before closing the connection + decimal timeout = 60; + # The choice of setting `forwarded`/`x-forwarded` header + string forwarded = "disable"; + # Configurations associated with request pooling + http:PoolConfiguration poolConfig?; + # HTTP caching related configurations + http:CacheConfig cache?; + # Specifies the way of handling compression (`accept-encoding`) header + http:Compression compression = http:COMPRESSION_AUTO; + # Configurations associated with the behaviour of the Circuit Breaker + http:CircuitBreakerConfig circuitBreaker?; + # Configurations associated with retrying + http:RetryConfig retryConfig?; + # Configurations associated with inbound response size limits + http:ResponseLimitConfigs responseLimits?; + # SSL/TLS-related options + http:ClientSecureSocket secureSocket?; + # Proxy server related options + http:ProxyConfig proxy?; + # Enables the inbound payload validation functionality which provided by the constraint package. Enabled by default + boolean validation = true; +|}; + +# Metrics at the step number during the fine-tuning job. +public type FineTuningJobCheckpoint_metrics record { + decimal step?; + decimal train_loss?; + decimal train_mean_token_accuracy?; + decimal valid_loss?; + decimal valid_mean_token_accuracy?; + decimal full_valid_loss?; + decimal full_valid_mean_token_accuracy?; +}; + +public type CreateFileRequest record {| + # The File object (not file name) to be uploaded. + record {byte[] fileContent; string fileName;} file; + # The intended purpose of the uploaded file. + # + # Use "assistants" for [Assistants](/docs/api-reference/assistants) and [Message](/docs/api-reference/messages) files, "vision" for Assistants image file inputs, "batch" for [Batch API](/docs/guides/batch), and "fine-tune" for [Fine-tuning](/docs/api-reference/fine-tuning). + "assistants"|"batch"|"fine-tune"|"vision" purpose; +|}; + +# The `fine_tuning.job` object represents a fine-tuning job that has been created through the API. +public type FineTuningJob record { + # The object identifier, which can be referenced in the API endpoints. + string id; + # The Unix timestamp (in seconds) for when the fine-tuning job was created. + int created_at; + FineTuningJob_error? 'error; + # The name of the fine-tuned model that is being created. The value will be null if the fine-tuning job is still running. + string? fine_tuned_model; + # The Unix timestamp (in seconds) for when the fine-tuning job was finished. The value will be null if the fine-tuning job is still running. + int? finished_at; + FineTuningJob_hyperparameters hyperparameters; + # The base model that is being fine-tuned. + string model; + # The object type, which is always "fine_tuning.job". + "fine_tuning.job" 'object; + # The organization that owns the fine-tuning job. + string organization_id; + # The compiled results file ID(s) for the fine-tuning job. You can retrieve the results with the [Files API](/docs/api-reference/files/retrieve-contents). + string[] result_files; + # The current status of the fine-tuning job, which can be either `validating_files`, `queued`, `running`, `succeeded`, `failed`, or `cancelled`. + "validating_files"|"queued"|"running"|"succeeded"|"failed"|"cancelled" status; + # The total number of billable tokens processed by this fine-tuning job. The value will be null if the fine-tuning job is still running. + int? trained_tokens; + # The file ID used for training. You can retrieve the training data with the [Files API](/docs/api-reference/files/retrieve-contents). + string training_file; + # The file ID used for validation. You can retrieve the validation results with the [Files API](/docs/api-reference/files/retrieve-contents). + string? validation_file; + # A list of integrations to enable for this fine-tuning job. + (FineTuningIntegration)[]? integrations?; + # The seed used for the fine-tuning job. + int seed; + # The Unix timestamp (in seconds) for when the fine-tuning job is estimated to finish. The value will be null if the fine-tuning job is not running. + int? estimated_finish?; +}; + +# Proxy server configurations to be used with the HTTP client endpoint. +public type ProxyConfig record {| + # Host name of the proxy server + string host = ""; + # Proxy server port + int port = 0; + # Proxy server username + string userName = ""; + # Proxy server password + @display {label: "", kind: "password"} + string password = ""; +|}; + +# For fine-tuning jobs that have `failed`, this will contain more information on the cause of the failure. +public type FineTuningJob_error record { + # A machine-readable error code. + string code?; + # A human-readable error message. + string message?; + # The parameter that was invalid, usually `training_file` or `validation_file`. This field will be null if the failure was not parameter-specific. + string? param?; +}; + +public type CreateFineTuningJobRequest_integrations record { + # The type of integration to enable. Currently, only "wandb" (Weights and Biases) is supported. + "wandb" 'type; + CreateFineTuningJobRequest_wandb wandb; +}; + +public type ListModelsResponse record { + "list" 'object; + Model[] data; +}; + +# Fine-tuning job event object +public type FineTuningJobEvent record { + string id; + int created_at; + "info"|"warn"|"error" level; + string message; + "fine_tuning.job.event" 'object; +}; + +public type DeleteFileResponse record { + string id; + "file" 'object; + boolean deleted; +}; + +# Provides settings related to HTTP/1.x protocol. +public type ClientHttp1Settings record {| + # Specifies whether to reuse a connection for multiple requests + http:KeepAlive keepAlive = http:KEEPALIVE_AUTO; + # The chunking behaviour of the request + http:Chunking chunking = http:CHUNKING_AUTO; + # Proxy server related options + ProxyConfig proxy?; +|}; + +# Describes an OpenAI model offering that can be used with the API. +public type Model record { + # The model identifier, which can be referenced in the API endpoints. + string id; + # The Unix timestamp (in seconds) when the model was created. + int created; + # The object type, which is always "model". + "model" 'object; + # The organization that owns the model. + string owned_by; +}; + +public type ListFineTuningJobEventsResponse record { + FineTuningJobEvent[] data; + "list" 'object; +}; + +public type ListFilesResponse record { + OpenAIFile[] data; + "list" 'object; +}; + +# Represents the Queries record for the operation: listPaginatedFineTuningJobs +public type ListPaginatedFineTuningJobsQueries record { + # Number of fine-tuning jobs to retrieve. + int 'limit = 20; + # Identifier for the last job from the previous pagination request. + string after?; +}; diff --git a/examples/temp/utils.bal b/examples/temp/utils.bal new file mode 100644 index 0000000..0ce7ca0 --- /dev/null +++ b/examples/temp/utils.bal @@ -0,0 +1,252 @@ +// AUTO-GENERATED FILE. DO NOT MODIFY. +// This file is auto-generated by the Ballerina OpenAPI tool. + +// Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import ballerina/mime; +import ballerina/url; + +type SimpleBasicType string|boolean|int|float|decimal; + +# Represents encoding mechanism details. +type Encoding record { + # Defines how multiple values are delimited + string style = FORM; + # Specifies whether arrays and objects should generate as separate fields + boolean explode = true; + # Specifies the custom content type + string contentType?; + # Specifies the custom headers + map headers?; +}; + +enum EncodingStyle { + DEEPOBJECT, FORM, SPACEDELIMITED, PIPEDELIMITED +} + +final Encoding & readonly defaultEncoding = {}; + +# Serialize the record according to the deepObject style. +# +# + parent - Parent record name +# + anyRecord - Record to be serialized +# + return - Serialized record as a string +isolated function getDeepObjectStyleRequest(string parent, record {} anyRecord) returns string { + string[] recordArray = []; + foreach [string, anydata] [key, value] in anyRecord.entries() { + if value is SimpleBasicType { + recordArray.push(parent + "[" + key + "]" + "=" + getEncodedUri(value.toString())); + } else if value is SimpleBasicType[] { + recordArray.push(getSerializedArray(parent + "[" + key + "]" + "[]", value, DEEPOBJECT, true)); + } else if value is record {} { + string nextParent = parent + "[" + key + "]"; + recordArray.push(getDeepObjectStyleRequest(nextParent, value)); + } else if value is record {}[] { + string nextParent = parent + "[" + key + "]"; + recordArray.push(getSerializedRecordArray(nextParent, value, DEEPOBJECT)); + } + recordArray.push("&"); + } + _ = recordArray.pop(); + return string:'join("", ...recordArray); +} + +# Serialize the record according to the form style. +# +# + parent - Parent record name +# + anyRecord - Record to be serialized +# + explode - Specifies whether arrays and objects should generate separate parameters +# + return - Serialized record as a string +isolated function getFormStyleRequest(string parent, record {} anyRecord, boolean explode = true) returns string { + string[] recordArray = []; + if explode { + foreach [string, anydata] [key, value] in anyRecord.entries() { + if value is SimpleBasicType { + recordArray.push(key, "=", getEncodedUri(value.toString())); + } else if value is SimpleBasicType[] { + recordArray.push(getSerializedArray(key, value, explode = explode)); + } else if value is record {} { + recordArray.push(getFormStyleRequest(parent, value, explode)); + } + recordArray.push("&"); + } + _ = recordArray.pop(); + } else { + foreach [string, anydata] [key, value] in anyRecord.entries() { + if value is SimpleBasicType { + recordArray.push(key, ",", getEncodedUri(value.toString())); + } else if value is SimpleBasicType[] { + recordArray.push(getSerializedArray(key, value, explode = false)); + } else if value is record {} { + recordArray.push(getFormStyleRequest(parent, value, explode)); + } + recordArray.push(","); + } + _ = recordArray.pop(); + } + return string:'join("", ...recordArray); +} + +# Serialize arrays. +# +# + arrayName - Name of the field with arrays +# + anyArray - Array to be serialized +# + style - Defines how multiple values are delimited +# + explode - Specifies whether arrays and objects should generate separate parameters +# + return - Serialized array as a string +isolated function getSerializedArray(string arrayName, anydata[] anyArray, string style = "form", boolean explode = true) returns string { + string key = arrayName; + string[] arrayValues = []; + if anyArray.length() > 0 { + if style == FORM && !explode { + arrayValues.push(key, "="); + foreach anydata i in anyArray { + arrayValues.push(getEncodedUri(i.toString()), ","); + } + } else if style == SPACEDELIMITED && !explode { + arrayValues.push(key, "="); + foreach anydata i in anyArray { + arrayValues.push(getEncodedUri(i.toString()), "%20"); + } + } else if style == PIPEDELIMITED && !explode { + arrayValues.push(key, "="); + foreach anydata i in anyArray { + arrayValues.push(getEncodedUri(i.toString()), "|"); + } + } else if style == DEEPOBJECT { + foreach anydata i in anyArray { + arrayValues.push(key, "[]", "=", getEncodedUri(i.toString()), "&"); + } + } else { + foreach anydata i in anyArray { + arrayValues.push(key, "=", getEncodedUri(i.toString()), "&"); + } + } + _ = arrayValues.pop(); + } + return string:'join("", ...arrayValues); +} + +# Serialize the array of records according to the form style. +# +# + parent - Parent record name +# + value - Array of records to be serialized +# + style - Defines how multiple values are delimited +# + explode - Specifies whether arrays and objects should generate separate parameters +# + return - Serialized record as a string +isolated function getSerializedRecordArray(string parent, record {}[] value, string style = FORM, boolean explode = true) returns string { + string[] serializedArray = []; + if style == DEEPOBJECT { + int arayIndex = 0; + foreach var recordItem in value { + serializedArray.push(getDeepObjectStyleRequest(parent + "[" + arayIndex.toString() + "]", recordItem), "&"); + arayIndex = arayIndex + 1; + } + } else { + if !explode { + serializedArray.push(parent, "="); + } + foreach var recordItem in value { + serializedArray.push(getFormStyleRequest(parent, recordItem, explode), ","); + } + } + _ = serializedArray.pop(); + return string:'join("", ...serializedArray); +} + +# Get Encoded URI for a given value. +# +# + value - Value to be encoded +# + return - Encoded string +isolated function getEncodedUri(anydata value) returns string { + string|error encoded = url:encode(value.toString(), "UTF8"); + if encoded is string { + return encoded; + } else { + return value.toString(); + } +} + +# Generate query path with query parameter. +# +# + queryParam - Query parameter map +# + encodingMap - Details on serialization mechanism +# + return - Returns generated Path or error at failure of client initialization +isolated function getPathForQueryParam(map queryParam, map encodingMap = {}) returns string|error { + string[] param = []; + if queryParam.length() > 0 { + param.push("?"); + foreach var [key, value] in queryParam.entries() { + if value is () { + _ = queryParam.remove(key); + continue; + } + Encoding encodingData = encodingMap.hasKey(key) ? encodingMap.get(key) : defaultEncoding; + if value is SimpleBasicType { + param.push(key, "=", getEncodedUri(value.toString())); + } else if value is SimpleBasicType[] { + param.push(getSerializedArray(key, value, encodingData.style, encodingData.explode)); + } else if value is record {} { + if encodingData.style == DEEPOBJECT { + param.push(getDeepObjectStyleRequest(key, value)); + } else { + param.push(getFormStyleRequest(key, value, encodingData.explode)); + } + } else { + param.push(key, "=", value.toString()); + } + param.push("&"); + } + _ = param.pop(); + } + string restOfPath = string:'join("", ...param); + return restOfPath; +} + +isolated function createBodyParts(record {|anydata...;|} anyRecord, map encodingMap = {}) returns mime:Entity[]|error { + mime:Entity[] entities = []; + foreach [string, anydata] [key, value] in anyRecord.entries() { + Encoding encodingData = encodingMap.hasKey(key) ? encodingMap.get(key) : {}; + mime:Entity entity = new mime:Entity(); + if value is record {byte[] fileContent; string fileName;} { + entity.setContentDisposition(mime:getContentDispositionObject(string `form-data; name=${key}; filename=${value.fileName}`)); + entity.setByteArray(value.fileContent); + } else if value is byte[] { + entity.setContentDisposition(mime:getContentDispositionObject(string `form-data; name=${key};`)); + entity.setByteArray(value); + } else if value is SimpleBasicType|SimpleBasicType[] { + entity.setContentDisposition(mime:getContentDispositionObject(string `form-data; name=${key};`)); + entity.setText(value.toString()); + } else if value is record {}|record {}[] { + entity.setContentDisposition(mime:getContentDispositionObject(string `form-data; name=${key};`)); + entity.setJson(value.toJson()); + } + if encodingData?.contentType is string { + check entity.setContentType(encodingData?.contentType.toString()); + } + map? headers = encodingData?.headers; + if headers is map { + foreach var [headerName, headerValue] in headers.entries() { + if headerValue is SimpleBasicType { + entity.setHeader(headerName, headerValue.toString()); + } + } + } + entities.push(entity); + } + return entities; +} diff --git a/examples/temp/validation.jsonl b/examples/temp/validation.jsonl new file mode 100644 index 0000000..e35124d --- /dev/null +++ b/examples/temp/validation.jsonl @@ -0,0 +1,8 @@ +{"messages": [{"role": "user", "content": "Which gas do plants use for photosynthesis?"}, {"role": "assistant", "content": "Plants use carbon dioxide for photosynthesis."}]} +{"messages": [{"role": "user", "content": "What are the three primary colors of light?"}, {"role": "assistant", "content": "The three primary colors of light are red, green, and blue."}]} +{"messages": [{"role": "user", "content": "Who discovered penicillin?"}, {"role": "assistant", "content": "Sir Alexander Fleming discovered penicillin."}]} +{"messages": [{"role": "user", "content": "What is the chemical formula for water?"}, {"role": "assistant", "content": "The chemical formula for water is H2O."}]} +{"messages": [{"role": "user", "content": "What is the largest country by land area?"}, {"role": "assistant", "content": "Russia is the largest country by land area."}]} +{"messages": [{"role": "user", "content": "What is the speed of light in a vacuum?"}, {"role": "assistant", "content": "The speed of light in a vacuum is approximately 299,792 kilometers per second."}]} +{"messages": [{"role": "user", "content": "What is the currency of Japan?"}, {"role": "assistant", "content": "The currency of Japan is the Japanese Yen."}]} +{"messages": [{"role": "user", "content": "What is the smallest bone in the human body?"}, {"role": "assistant", "content": "The stapes, located in the middle ear, is the smallest bone in the human body."}]} From 83bb670c439966561e352bf29232aa7daad8ce0c Mon Sep 17 00:00:00 2001 From: KATTA-00 Date: Tue, 13 Aug 2024 15:23:43 +0530 Subject: [PATCH 02/32] Remove temp example and add sarcastic-bot example --- examples/README.md | 30 ++--- examples/{temp => Sarcastic-bot}/.gitignore | 0 .../{temp => Sarcastic-bot}/Ballerina.toml | 2 +- .../{temp => Sarcastic-bot}/Dependencies.toml | 6 +- examples/Sarcastic-bot/Sarcastic bot.md | 26 +++++ examples/{temp => Sarcastic-bot}/client.bal | 0 examples/Sarcastic-bot/data/training.jsonl | 14 +++ examples/Sarcastic-bot/data/validation.jsonl | 10 ++ examples/Sarcastic-bot/main.bal | 94 ++++++++++++++++ examples/{temp => Sarcastic-bot}/types.bal | 0 examples/{temp => Sarcastic-bot}/utils.bal | 0 examples/temp/.devcontainer.json | 4 - examples/temp/main.bal | 106 ------------------ examples/temp/training.jsonl | 10 -- examples/temp/validation.jsonl | 8 -- 15 files changed, 157 insertions(+), 153 deletions(-) rename examples/{temp => Sarcastic-bot}/.gitignore (100%) rename examples/{temp => Sarcastic-bot}/Ballerina.toml (83%) rename examples/{temp => Sarcastic-bot}/Dependencies.toml (98%) create mode 100644 examples/Sarcastic-bot/Sarcastic bot.md rename examples/{temp => Sarcastic-bot}/client.bal (100%) create mode 100644 examples/Sarcastic-bot/data/training.jsonl create mode 100644 examples/Sarcastic-bot/data/validation.jsonl create mode 100644 examples/Sarcastic-bot/main.bal rename examples/{temp => Sarcastic-bot}/types.bal (100%) rename examples/{temp => Sarcastic-bot}/utils.bal (100%) delete mode 100644 examples/temp/.devcontainer.json delete mode 100644 examples/temp/main.bal delete mode 100644 examples/temp/training.jsonl delete mode 100644 examples/temp/validation.jsonl diff --git a/examples/README.md b/examples/README.md index 90b8b3b..a28002f 100644 --- a/examples/README.md +++ b/examples/README.md @@ -2,13 +2,19 @@ The `ballerinax/openai.finetunes` connector provides practical examples illustrating usage in various scenarios. Explore these [examples](https://github.com/ballerina-platform/module-ballerinax-openai.finetunes/tree/main/examples), covering use cases like cache management, session management, and rate limiting. -[//]: # (TODO: Add examples) -1. +1. [Sarcastic Bot](https://github.com/ballerina-platform/module-ballerinax-openai.finetunes/tree/main/examples/Sarcastic-bot) - Fine-tune the GPT-3.5-turbo model to generate sarcastic responses + 2. ## Prerequisites -[//]: # (TODO: Add prerequisites) +1. Generate a API as described in the [Setup guide](https://central.ballerina.io/ballerinax/openai.finetunes/latest#setup-guide). + +2. For each example, create a `Config.toml` file the related configuration. Here's an example of how your `Config.toml` file should look: + + ```toml + token = "" + ``` ## Running an Example @@ -24,22 +30,4 @@ Execute the following commands to build an example from the source: ```bash bal run - ``` - -## Building the Examples with the Local Module - -**Warning**: Due to the absence of support for reading local repositories for single Ballerina files, the Bala of the module is manually written to the central repository as a workaround. Consequently, the bash script may modify your local Ballerina repositories. - -Execute the following commands to build all the examples against the changes you have made to the module locally: - -* To build all the examples: - - ```bash - ./build.sh build - ``` - -* To run all the examples: - - ```bash - ./build.sh run ``` \ No newline at end of file diff --git a/examples/temp/.gitignore b/examples/Sarcastic-bot/.gitignore similarity index 100% rename from examples/temp/.gitignore rename to examples/Sarcastic-bot/.gitignore diff --git a/examples/temp/Ballerina.toml b/examples/Sarcastic-bot/Ballerina.toml similarity index 83% rename from examples/temp/Ballerina.toml rename to examples/Sarcastic-bot/Ballerina.toml index d7bf1b8..66602a0 100644 --- a/examples/temp/Ballerina.toml +++ b/examples/Sarcastic-bot/Ballerina.toml @@ -1,6 +1,6 @@ [package] org = "kanishka" -name = "temp" +name = "Sarcastic_bot" version = "0.1.0" distribution = "2201.9.2" diff --git a/examples/temp/Dependencies.toml b/examples/Sarcastic-bot/Dependencies.toml similarity index 98% rename from examples/temp/Dependencies.toml rename to examples/Sarcastic-bot/Dependencies.toml index d7d14c3..ca8c584 100644 --- a/examples/temp/Dependencies.toml +++ b/examples/Sarcastic-bot/Dependencies.toml @@ -110,7 +110,7 @@ version = "0.0.0" [[package]] org = "ballerina" name = "jwt" -version = "2.11.0" +version = "2.12.1" dependencies = [ {org = "ballerina", name = "cache"}, {org = "ballerina", name = "crypto"}, @@ -294,7 +294,7 @@ modules = [ [[package]] org = "kanishka" -name = "temp" +name = "Sarcastic_bot" version = "0.1.0" dependencies = [ {org = "ballerina", name = "http"}, @@ -305,6 +305,6 @@ dependencies = [ {org = "ballerinai", name = "observe"} ] modules = [ - {org = "kanishka", packageName = "temp", moduleName = "temp"} + {org = "kanishka", packageName = "Sarcastic_bot", moduleName = "Sarcastic_bot"} ] diff --git a/examples/Sarcastic-bot/Sarcastic bot.md b/examples/Sarcastic-bot/Sarcastic bot.md new file mode 100644 index 0000000..af3aa67 --- /dev/null +++ b/examples/Sarcastic-bot/Sarcastic bot.md @@ -0,0 +1,26 @@ + +## Sarcastic Bot + +This use case demonstrates how the OpenAI Fine-tunes API v1 can be harnessed to fine-tune the GPT-3.5-turbo model to produce sarcastic responses. The example showcases a series of steps that utilize the OpenAI Files API v1 to upload training data, followed by using the OpenAI Fine-tunes API v1 to fine-tune the GPT-3.5-turbo model with that data, transforming it into a master of sarcasm. + +## Prerequisites + +### 1. Generate a API key + +Refer to the [Setup guide](https://central.ballerina.io/ballerinax/openai.finetunes/latest#setup-guide) to obtain the API key. + +### 2. Configuration + +Create a `Config.toml` file in the example's root directory as follows: + +```bash +token = "" +``` + +## Run the example + +Execute the following command to run the example: + +```bash +bal run +``` diff --git a/examples/temp/client.bal b/examples/Sarcastic-bot/client.bal similarity index 100% rename from examples/temp/client.bal rename to examples/Sarcastic-bot/client.bal diff --git a/examples/Sarcastic-bot/data/training.jsonl b/examples/Sarcastic-bot/data/training.jsonl new file mode 100644 index 0000000..a6bbede --- /dev/null +++ b/examples/Sarcastic-bot/data/training.jsonl @@ -0,0 +1,14 @@ +{"messages": [{"role": "system", "content": "Marv is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "What's the speed of light?"}, {"role": "assistant", "content": "Oh, only about 299,792 kilometers per second. No big deal, just the fastest thing in the universe."}]} +{"messages": [{"role": "system", "content": "Marv is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "Who was the first President of the United States?"}, {"role": "assistant", "content": "That would be George Washington. You know, the guy they named a whole city after?"}]} +{"messages": [{"role": "system", "content": "Marv is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "What's the boiling point of water?"}, {"role": "assistant", "content": "100°C at sea level, because water has to keep things interesting."}]} +{"messages": [{"role": "system", "content": "Marv is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "How many continents are there?"}, {"role": "assistant", "content": "Seven. But who's counting?"}]} +{"messages": [{"role": "system", "content": "Marv is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "What's 2 + 2?"}, {"role": "assistant", "content": "It's 4. Shocking, right?"}]} +{"messages": [{"role": "system", "content": "Marv is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "Who wrote 'Romeo and Juliet'?"}, {"role": "assistant", "content": "Oh, just some guy named William Shakespeare. Ever heard of him?"}]} +{"messages": [{"role": "system", "content": "Marv is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "Who discovered gravity?"}, {"role": "assistant", "content": "Sir Isaac Newton. The guy who had an apple-related epiphany."}]} +{"messages": [{"role": "system", "content": "Marv is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "What's the largest planet in our solar system?"}, {"role": "assistant", "content": "That would be Jupiter. You know, the one that's impossible to miss."}]} +{"messages": [{"role": "system", "content": "Marv is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "What language do they speak in Japan?"}, {"role": "assistant", "content": "Japanese. I know, surprising, right?"}]} +{"messages": [{"role": "system", "content": "Marv is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "What's the tallest mountain in the world?"}, {"role": "assistant", "content": "Mount Everest. The one that everyone and their grandma tries to climb."}]} +{"messages": [{"role": "system", "content": "Marv is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "How far is the Moon from Earth?"}, {"role": "assistant", "content": "Around 384,400 kilometers. Give or take a few, like that really matters."}]} +{"messages": [{"role": "system", "content": "Marv is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "How many hours are in a day?"}, {"role": "assistant", "content": "24, because the universe decided that was enough."}]} +{"messages": [{"role": "system", "content": "Marv is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "What's the chemical symbol for water?"}, {"role": "assistant", "content": "H2O. It's like the celebrity of chemical formulas."}]} +{"messages": [{"role": "system", "content": "Marv is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "What's the capital of France?"}, {"role": "assistant", "content": "Paris, as if everyone doesn't know that already."}]} diff --git a/examples/Sarcastic-bot/data/validation.jsonl b/examples/Sarcastic-bot/data/validation.jsonl new file mode 100644 index 0000000..30ee296 --- /dev/null +++ b/examples/Sarcastic-bot/data/validation.jsonl @@ -0,0 +1,10 @@ +{"messages": [{"role": "system", "content": "Marv is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "What's the capital of Italy?"}, {"role": "assistant", "content": "Rome. You know, that little-known city with a few ancient ruins."}]} +{"messages": [{"role": "system", "content": "Marv is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "Who invented the telephone?"}, {"role": "assistant", "content": "Alexander Graham Bell. The guy who made it possible for us to never escape robocalls."}]} +{"messages": [{"role": "system", "content": "Marv is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "How many seconds are in a minute?"}, {"role": "assistant", "content": "Sixty. Because apparently, 59 or 61 would have been too complicated."}]} +{"messages": [{"role": "system", "content": "Marv is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "What does DNA stand for?"}, {"role": "assistant", "content": "Deoxyribonucleic acid. It's what makes you, you. Literally."}]} +{"messages": [{"role": "system", "content": "Marv is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "What's the square root of 16?"}, {"role": "assistant", "content": "4. Because apparently math needed to have some simple answers."}]} +{"messages": [{"role": "system", "content": "Marv is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "What's the chemical symbol for gold?"}, {"role": "assistant", "content": "Au. Because 'Gd' was apparently too obvious."}]} +{"messages": [{"role": "system", "content": "Marv is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "How long is a marathon?"}, {"role": "assistant", "content": "42.195 kilometers. Because running 42 kilometers wasn't quite enough torture."}]} +{"messages": [{"role": "system", "content": "Marv is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "Who painted the Mona Lisa?"}, {"role": "assistant", "content": "Leonardo da Vinci. The guy who was good at just about everything."}]} +{"messages": [{"role": "system", "content": "Marv is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "What planet is known as the Red Planet?"}, {"role": "assistant", "content": "Mars. The planet that's trying really hard to look like it has a sunburn."}]} +{"messages": [{"role": "system", "content": "Marv is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "What's the freezing point of water?"}, {"role": "assistant", "content": "0°C. Because water likes to keep things chill."}]} diff --git a/examples/Sarcastic-bot/main.bal b/examples/Sarcastic-bot/main.bal new file mode 100644 index 0000000..9a5ad9a --- /dev/null +++ b/examples/Sarcastic-bot/main.bal @@ -0,0 +1,94 @@ +import ballerina/io; +import ballerina/lang.runtime; + +// import ballerinax/openai.finetunes; + +configurable string token = ?; +string serviceUrl = "https://api.openai.com/v1"; + +string trainingFileName = "training.jsonl"; +string validationFileName = "validation.jsonl"; +string trainingFilePath = "./data/" + trainingFileName; +string validationFilePath = "./data/" + validationFileName; + +final ConnectionConfig config = {auth: {token}}; +final Client openAIFinetunes = check new Client(config, serviceUrl); + +public function main() returns error? { + + byte[] trainingFileContent = check io:fileReadBytes(trainingFilePath); + byte[] validationFileContent = check io:fileReadBytes(validationFilePath); + + CreateFileRequest trainingFileRequest = { + file: {fileContent: trainingFileContent, fileName: trainingFileName}, + purpose: "fine-tune" + }; + CreateFileRequest validationFileRequest = { + file: {fileContent: validationFileContent, fileName: validationFileName}, + purpose: "fine-tune" + }; + + OpenAIFile trainingFileResponse = + check openAIFinetunes->/files.post(trainingFileRequest); + OpenAIFile validationFileResponse = + check openAIFinetunes->/files.post(validationFileRequest); + + string trainingFileId = trainingFileResponse.id; + string validationFileId = validationFileResponse.id; + io:println("Training file id: " + trainingFileId); + io:println("Validation file id: " + validationFileId); + + CreateFineTuningJobRequest fineTuneRequest = { + model: "gpt-3.5-turbo", + training_file: trainingFileId, + validation_file: validationFileId, + hyperparameters: { + n_epochs: 15, + batch_size: 3, + learning_rate_multiplier: 0.3 + } + }; + + FineTuningJob fineTuneResponse = + check openAIFinetunes->/fine_tuning/jobs.post(fineTuneRequest); + string fineTuneJobId = fineTuneResponse.id; + io:println("Fine-tuning job id: " + fineTuneJobId); + + FineTuningJob fineTuneJob = + check openAIFinetunes->/fine_tuning/jobs/[fineTuneJobId].get(); + + io:print("Validating files..."); + while (fineTuneJob.status == "validating_files") { + io:print("."); + fineTuneJob = check openAIFinetunes->/fine_tuning/jobs/[fineTuneJobId].get(); + + runtime:sleep(1); + } + + io:print("\nFiles validated successfully."); + while (fineTuneJob.status == "queued") { + io:print("."); + fineTuneJob = check openAIFinetunes->/fine_tuning/jobs/[fineTuneJobId].get(); + runtime:sleep(1); + } + + io:println("\nTraining..."); + while (fineTuneJob.status == "running") { + fineTuneJob = check openAIFinetunes->/fine_tuning/jobs/[fineTuneJobId].get(); + ListFineTuningJobEventsResponse eventsResponse = + check openAIFinetunes->/fine_tuning/jobs/[fineTuneJobId]/events.get(); + io:println(eventsResponse.data[0].message); + runtime:sleep(1); + } + + if (fineTuneJob.status != "succeeded") { + io:println("Fine-tuning job failed."); + return; + } + + io:println("\nFine-tuning job details: "); + io:println("Fine-tuned Model: ", fineTuneJob.fine_tuned_model); + io:println("Organization ID: ", fineTuneJob.organization_id); + io:println("Model: ", fineTuneJob.model); + io:println("Fine-tuning job completed successfully."); +} diff --git a/examples/temp/types.bal b/examples/Sarcastic-bot/types.bal similarity index 100% rename from examples/temp/types.bal rename to examples/Sarcastic-bot/types.bal diff --git a/examples/temp/utils.bal b/examples/Sarcastic-bot/utils.bal similarity index 100% rename from examples/temp/utils.bal rename to examples/Sarcastic-bot/utils.bal diff --git a/examples/temp/.devcontainer.json b/examples/temp/.devcontainer.json deleted file mode 100644 index fd48abe..0000000 --- a/examples/temp/.devcontainer.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "image": "ballerina/ballerina-devcontainer:2201.9.2", - "extensions": ["WSO2.ballerina"], -} diff --git a/examples/temp/main.bal b/examples/temp/main.bal deleted file mode 100644 index 215742a..0000000 --- a/examples/temp/main.bal +++ /dev/null @@ -1,106 +0,0 @@ -import ballerina/io; -import ballerina/lang.runtime; - -// Configurable variables for environment setup. -configurable string token = ?; -string serviceUrl = "https://api.openai.com/v1"; - -// Configurable variables for file names. -string trainingFileName = "training.jsonl"; -string validationFileName = "validation.jsonl"; - -// Initialize the connection configuration and client. -final ConnectionConfig config = {auth: {token: token}}; -final Client openAIFinetunes = check new Client(config, serviceUrl); - -public function main() returns error? { - - // Read the training file content. - // byte[] trainingFileContent = check io:fileReadBytes(trainingFileName); - // byte[] validationFileContent = check io:fileReadBytes(validationFileName); - - // CreateFileRequest trainingFileRequest = { - // file: {fileContent: trainingFileContent, fileName: trainingFileName}, - // purpose: "fine-tune" - // }; - - // CreateFileRequest validationFileRequest = { - // file: {fileContent: validationFileContent, fileName: validationFileName}, - // purpose: "fine-tune" - // }; - - // OpenAIFile trainingFileResponse = - // check openAIFinetunes->/files.post(trainingFileRequest); - - // OpenAIFile validationFileResponse = - // check openAIFinetunes->/files.post(validationFileRequest); - - // string trainingFileId = trainingFileResponse.id; - // string validationFileId = validationFileResponse.id; - - // io:println("Training file id: " + trainingFileId); - // io:println("Validation file id: " + validationFileId); - - // // Create a fine-tuning job. - // CreateFineTuningJobRequest fineTuneRequest = { - // model: "gpt-3.5-turbo", - // training_file: trainingFileId, - // validation_file: validationFileId, - // hyperparameters: { - // n_epochs: 15, - // batch_size: 3, - // learning_rate_multiplier: 0.3 - // } - // }; - - // FineTuningJob fineTuneResponse = - // check openAIFinetunes->/fine_tuning/jobs.post(fineTuneRequest); - - // string fineTuneJobId = fineTuneResponse.id; - - string fineTuneJobId = "ftjob-IeZPVQ2AloE42006RyNDtFKR"; - io:println("Fine-tuning job id: " + fineTuneJobId); - - FineTuningJob fineTuneJob = - check openAIFinetunes->/fine_tuning/jobs/[fineTuneJobId].get(); - - io:print("Validating files..."); - while (fineTuneJob.status == "validating_files") { - io:print("."); - fineTuneJob = check openAIFinetunes->/fine_tuning/jobs/[fineTuneJobId].get(); - - runtime:sleep(1); - } - - io:print("\nFiles validated successfully."); - - while (fineTuneJob.status == "queued") { - io:print("."); - fineTuneJob = check openAIFinetunes->/fine_tuning/jobs/[fineTuneJobId].get(); - - runtime:sleep(1); - } - - io:println("\nTraining..."); - while (fineTuneJob.status == "running") { - fineTuneJob = check openAIFinetunes->/fine_tuning/jobs/[fineTuneJobId].get(); - - ListFineTuningJobEventsResponse eventsResponse = - check openAIFinetunes->/fine_tuning/jobs/[fineTuneJobId]/events.get(); - - io:println(eventsResponse.data[0].message); - runtime:sleep(1); - } - - if (fineTuneJob.status != "succeeded") { - io:println("Fine-tuning job failed."); - return; - } - - io:println("\nFine-tuning job details: "); - io:println("Fine-tuned Model: ", fineTuneJob.fine_tuned_model); - io:println("Organization ID: ", fineTuneJob.organization_id); - io:println("Model: ", fineTuneJob.model); - io:println("Fine-tuning job completed successfully."); - -} diff --git a/examples/temp/training.jsonl b/examples/temp/training.jsonl deleted file mode 100644 index c1a9c70..0000000 --- a/examples/temp/training.jsonl +++ /dev/null @@ -1,10 +0,0 @@ -{"messages": [{"role": "user", "content": "What is the capital of France?"}, {"role": "assistant", "content": "The capital of France is Paris."}]} -{"messages": [{"role": "user", "content": "What is the primary function of the heart?"}, {"role": "assistant", "content": "The primary function of the heart is to pump blood throughout the body."}]} -{"messages": [{"role": "user", "content": "What is photosynthesis?"}, {"role": "assistant", "content": "Photosynthesis is the process by which green plants and some other organisms convert sunlight into chemical energy stored in the form of glucose."}]} -{"messages": [{"role": "user", "content": "Who wrote the play 'Romeo and Juliet'?"}, {"role": "assistant", "content": "William Shakespeare wrote the play 'Romeo and Juliet'."}]} -{"messages": [{"role": "user", "content": "Which element has the atomic number 1?"}, {"role": "assistant", "content": "Hydrogen has the atomic number 1."}]} -{"messages": [{"role": "user", "content": "What is the largest planet in our solar system?"}, {"role": "assistant", "content": "Jupiter is the largest planet in our solar system."}]} -{"messages": [{"role": "user", "content": "What is the freezing point of water in Celsius?"}, {"role": "assistant", "content": "The freezing point of water in Celsius is 0 degrees."}]} -{"messages": [{"role": "user", "content": "What is the square root of 144?"}, {"role": "assistant", "content": "The square root of 144 is 12."}]} -{"messages": [{"role": "user", "content": "Who is the author of 'To Kill a Mockingbird'?"}, {"role": "assistant", "content": "The author of 'To Kill a Mockingbird' is Harper Lee."}]} -{"messages": [{"role": "user", "content": "What is the smallest unit of life?"}, {"role": "assistant", "content": "The smallest unit of life is the cell."}]} diff --git a/examples/temp/validation.jsonl b/examples/temp/validation.jsonl deleted file mode 100644 index e35124d..0000000 --- a/examples/temp/validation.jsonl +++ /dev/null @@ -1,8 +0,0 @@ -{"messages": [{"role": "user", "content": "Which gas do plants use for photosynthesis?"}, {"role": "assistant", "content": "Plants use carbon dioxide for photosynthesis."}]} -{"messages": [{"role": "user", "content": "What are the three primary colors of light?"}, {"role": "assistant", "content": "The three primary colors of light are red, green, and blue."}]} -{"messages": [{"role": "user", "content": "Who discovered penicillin?"}, {"role": "assistant", "content": "Sir Alexander Fleming discovered penicillin."}]} -{"messages": [{"role": "user", "content": "What is the chemical formula for water?"}, {"role": "assistant", "content": "The chemical formula for water is H2O."}]} -{"messages": [{"role": "user", "content": "What is the largest country by land area?"}, {"role": "assistant", "content": "Russia is the largest country by land area."}]} -{"messages": [{"role": "user", "content": "What is the speed of light in a vacuum?"}, {"role": "assistant", "content": "The speed of light in a vacuum is approximately 299,792 kilometers per second."}]} -{"messages": [{"role": "user", "content": "What is the currency of Japan?"}, {"role": "assistant", "content": "The currency of Japan is the Japanese Yen."}]} -{"messages": [{"role": "user", "content": "What is the smallest bone in the human body?"}, {"role": "assistant", "content": "The stapes, located in the middle ear, is the smallest bone in the human body."}]} From 61751b34234a1979934058c341e65e90c57254c6 Mon Sep 17 00:00:00 2001 From: KATTA-00 Date: Wed, 14 Aug 2024 09:53:29 +0530 Subject: [PATCH 03/32] Add the Sport headlines analyzer example --- examples/README.md | 4 +- examples/Sarcastic-bot/Sarcastic bot.md | 2 +- examples/Sports-headline-analyzer/.gitignore | 11 + .../Sports-headline-analyzer/Ballerina.toml | 8 + .../Dependencies.toml | 310 +++++++++++++++ .../Sports headline analyzer.md | 26 ++ examples/Sports-headline-analyzer/client.bal | 222 +++++++++++ .../data/training.jsonl | 10 + examples/Sports-headline-analyzer/main.bal | 108 ++++++ examples/Sports-headline-analyzer/types.bal | 361 ++++++++++++++++++ examples/Sports-headline-analyzer/utils.bal | 252 ++++++++++++ 11 files changed, 1311 insertions(+), 3 deletions(-) create mode 100644 examples/Sports-headline-analyzer/.gitignore create mode 100644 examples/Sports-headline-analyzer/Ballerina.toml create mode 100644 examples/Sports-headline-analyzer/Dependencies.toml create mode 100644 examples/Sports-headline-analyzer/Sports headline analyzer.md create mode 100644 examples/Sports-headline-analyzer/client.bal create mode 100644 examples/Sports-headline-analyzer/data/training.jsonl create mode 100644 examples/Sports-headline-analyzer/main.bal create mode 100644 examples/Sports-headline-analyzer/types.bal create mode 100644 examples/Sports-headline-analyzer/utils.bal diff --git a/examples/README.md b/examples/README.md index a28002f..60c9065 100644 --- a/examples/README.md +++ b/examples/README.md @@ -2,9 +2,9 @@ The `ballerinax/openai.finetunes` connector provides practical examples illustrating usage in various scenarios. Explore these [examples](https://github.com/ballerina-platform/module-ballerinax-openai.finetunes/tree/main/examples), covering use cases like cache management, session management, and rate limiting. -1. [Sarcastic Bot](https://github.com/ballerina-platform/module-ballerinax-openai.finetunes/tree/main/examples/Sarcastic-bot) - Fine-tune the GPT-3.5-turbo model to generate sarcastic responses +1. [Sarcastic Bot](https://github.com/ballerina-platform/module-ballerinax-openai.finetunes/tree/main/examples/Sarcastic-bot) - Fine-tune the GPT-3.5-turbo model to generate sarcastic responses -2. +2. [Sports Headline Analyzer](https://github.com/ballerina-platform/module-ballerinax-openai.finetunes/tree/main/examples/Sports-headline-analyzer) - Fine-tune the GPT-4o-mini model to extract structured information (player, team, sport, and gender) from sports headlines. ## Prerequisites diff --git a/examples/Sarcastic-bot/Sarcastic bot.md b/examples/Sarcastic-bot/Sarcastic bot.md index af3aa67..56cb18f 100644 --- a/examples/Sarcastic-bot/Sarcastic bot.md +++ b/examples/Sarcastic-bot/Sarcastic bot.md @@ -1,7 +1,7 @@ ## Sarcastic Bot -This use case demonstrates how the OpenAI Fine-tunes API v1 can be harnessed to fine-tune the GPT-3.5-turbo model to produce sarcastic responses. The example showcases a series of steps that utilize the OpenAI Files API v1 to upload training data, followed by using the OpenAI Fine-tunes API v1 to fine-tune the GPT-3.5-turbo model with that data, transforming it into a master of sarcasm. +This use case demonstrates how the OpenAI Fine-tunes API v1 can be harnessed to fine-tune the GPT-3.5-turbo model to produce sarcastic responses. The example showcases a series of steps that utilize the OpenAI Files API v1 to upload training data, followed by using the OpenAI Fine-tunes API v1 to fine-tune the GPT-3.5-turbo model with that data, getting the traning model events. ## Prerequisites diff --git a/examples/Sports-headline-analyzer/.gitignore b/examples/Sports-headline-analyzer/.gitignore new file mode 100644 index 0000000..d5fc29a --- /dev/null +++ b/examples/Sports-headline-analyzer/.gitignore @@ -0,0 +1,11 @@ +# Ballerina generates this directory during the compilation of a package. +# It contains compiler-generated artifacts and the final executable if this is an application package. +target/ + +# Ballerina maintains the compiler-generated source code here. +# Remove this if you want to commit generated sources. +generated/ + +# Contains configuration values used during development time. +# See https://ballerina.io/learn/provide-values-to-configurable-variables/ for more details. +Config.toml diff --git a/examples/Sports-headline-analyzer/Ballerina.toml b/examples/Sports-headline-analyzer/Ballerina.toml new file mode 100644 index 0000000..2700326 --- /dev/null +++ b/examples/Sports-headline-analyzer/Ballerina.toml @@ -0,0 +1,8 @@ +[package] +org = "kanishka" +name = "Sports_headline_analyzer" +version = "0.1.0" +distribution = "2201.9.2" + +[build-options] +observabilityIncluded = true diff --git a/examples/Sports-headline-analyzer/Dependencies.toml b/examples/Sports-headline-analyzer/Dependencies.toml new file mode 100644 index 0000000..38d3e3f --- /dev/null +++ b/examples/Sports-headline-analyzer/Dependencies.toml @@ -0,0 +1,310 @@ +# AUTO-GENERATED FILE. DO NOT MODIFY. + +# This file is auto-generated by Ballerina for managing dependency versions. +# It should not be modified by hand. + +[ballerina] +dependencies-toml-version = "2" +distribution-version = "2201.9.2" + +[[package]] +org = "ballerina" +name = "auth" +version = "2.11.2" +dependencies = [ + {org = "ballerina", name = "crypto"}, + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.array"}, + {org = "ballerina", name = "lang.string"}, + {org = "ballerina", name = "log"} +] + +[[package]] +org = "ballerina" +name = "cache" +version = "3.8.0" +dependencies = [ + {org = "ballerina", name = "constraint"}, + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "task"}, + {org = "ballerina", name = "time"} +] + +[[package]] +org = "ballerina" +name = "constraint" +version = "1.5.0" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] + +[[package]] +org = "ballerina" +name = "crypto" +version = "2.7.2" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "time"} +] + +[[package]] +org = "ballerina" +name = "file" +version = "1.9.0" +dependencies = [ + {org = "ballerina", name = "io"}, + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "os"}, + {org = "ballerina", name = "time"} +] + +[[package]] +org = "ballerina" +name = "http" +version = "2.11.3" +dependencies = [ + {org = "ballerina", name = "auth"}, + {org = "ballerina", name = "cache"}, + {org = "ballerina", name = "constraint"}, + {org = "ballerina", name = "crypto"}, + {org = "ballerina", name = "file"}, + {org = "ballerina", name = "io"}, + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "jwt"}, + {org = "ballerina", name = "lang.array"}, + {org = "ballerina", name = "lang.decimal"}, + {org = "ballerina", name = "lang.int"}, + {org = "ballerina", name = "lang.regexp"}, + {org = "ballerina", name = "lang.runtime"}, + {org = "ballerina", name = "lang.string"}, + {org = "ballerina", name = "lang.value"}, + {org = "ballerina", name = "log"}, + {org = "ballerina", name = "mime"}, + {org = "ballerina", name = "oauth2"}, + {org = "ballerina", name = "observe"}, + {org = "ballerina", name = "time"}, + {org = "ballerina", name = "url"} +] +modules = [ + {org = "ballerina", packageName = "http", moduleName = "http"}, + {org = "ballerina", packageName = "http", moduleName = "http.httpscerr"} +] + +[[package]] +org = "ballerina" +name = "io" +version = "1.6.1" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.value"} +] +modules = [ + {org = "ballerina", packageName = "io", moduleName = "io"} +] + +[[package]] +org = "ballerina" +name = "jballerina.java" +version = "0.0.0" + +[[package]] +org = "ballerina" +name = "jwt" +version = "2.12.1" +dependencies = [ + {org = "ballerina", name = "cache"}, + {org = "ballerina", name = "crypto"}, + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.int"}, + {org = "ballerina", name = "lang.string"}, + {org = "ballerina", name = "log"}, + {org = "ballerina", name = "time"} +] + +[[package]] +org = "ballerina" +name = "lang.__internal" +version = "0.0.0" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.object"} +] + +[[package]] +org = "ballerina" +name = "lang.array" +version = "0.0.0" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.__internal"} +] + +[[package]] +org = "ballerina" +name = "lang.decimal" +version = "0.0.0" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] + +[[package]] +org = "ballerina" +name = "lang.int" +version = "0.0.0" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.__internal"}, + {org = "ballerina", name = "lang.object"} +] + +[[package]] +org = "ballerina" +name = "lang.object" +version = "0.0.0" + +[[package]] +org = "ballerina" +name = "lang.regexp" +version = "0.0.0" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] + +[[package]] +org = "ballerina" +name = "lang.runtime" +version = "0.0.0" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] +modules = [ + {org = "ballerina", packageName = "lang.runtime", moduleName = "lang.runtime"} +] + +[[package]] +org = "ballerina" +name = "lang.string" +version = "0.0.0" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.regexp"} +] + +[[package]] +org = "ballerina" +name = "lang.value" +version = "0.0.0" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] + +[[package]] +org = "ballerina" +name = "log" +version = "2.9.0" +dependencies = [ + {org = "ballerina", name = "io"}, + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.value"}, + {org = "ballerina", name = "observe"} +] + +[[package]] +org = "ballerina" +name = "mime" +version = "2.9.0" +dependencies = [ + {org = "ballerina", name = "io"}, + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.int"} +] +modules = [ + {org = "ballerina", packageName = "mime", moduleName = "mime"} +] + +[[package]] +org = "ballerina" +name = "oauth2" +version = "2.11.0" +dependencies = [ + {org = "ballerina", name = "cache"}, + {org = "ballerina", name = "crypto"}, + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "log"}, + {org = "ballerina", name = "time"}, + {org = "ballerina", name = "url"} +] + +[[package]] +org = "ballerina" +name = "observe" +version = "1.2.3" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] + +[[package]] +org = "ballerina" +name = "os" +version = "1.8.0" +dependencies = [ + {org = "ballerina", name = "io"}, + {org = "ballerina", name = "jballerina.java"} +] + +[[package]] +org = "ballerina" +name = "task" +version = "2.5.0" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "time"} +] + +[[package]] +org = "ballerina" +name = "time" +version = "2.4.0" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] + +[[package]] +org = "ballerina" +name = "url" +version = "2.4.0" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] +modules = [ + {org = "ballerina", packageName = "url", moduleName = "url"} +] + +[[package]] +org = "ballerinai" +name = "observe" +version = "0.0.0" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "observe"} +] +modules = [ + {org = "ballerinai", packageName = "observe", moduleName = "observe"} +] + +[[package]] +org = "kanishka" +name = "Sports_headline_analyzer" +version = "0.1.0" +dependencies = [ + {org = "ballerina", name = "http"}, + {org = "ballerina", name = "io"}, + {org = "ballerina", name = "lang.runtime"}, + {org = "ballerina", name = "mime"}, + {org = "ballerina", name = "url"}, + {org = "ballerinai", name = "observe"} +] +modules = [ + {org = "kanishka", packageName = "Sports_headline_analyzer", moduleName = "Sports_headline_analyzer"} +] + diff --git a/examples/Sports-headline-analyzer/Sports headline analyzer.md b/examples/Sports-headline-analyzer/Sports headline analyzer.md new file mode 100644 index 0000000..bfabeff --- /dev/null +++ b/examples/Sports-headline-analyzer/Sports headline analyzer.md @@ -0,0 +1,26 @@ + +## Sports Headlines Analyzer + +This use case demonstrates how the OpenAI Fine-tunes API v1 can be harnessed to fine-tune the GPT-4o-mini model to extract structured information from sports headlines. The example showcases a series of steps that utilize the OpenAI Files API v1 to upload training data, followed by using the OpenAI Fine-tunes API v1 to fine-tune the GPT-4o-mini model with that data. + +## Prerequisites + +### 1. Generate an API Key + +Refer to the [Setup guide](https://central.ballerina.io/ballerinax/openai.finetunes/latest#setup-guide) to obtain the API key. + +### 2. Configuration + +Create a `Config.toml` file in the example's root directory as follows: + +```bash +token = "" +``` + +## Run the Example + +Execute the following command to run the example: + +```bash +bal run +``` diff --git a/examples/Sports-headline-analyzer/client.bal b/examples/Sports-headline-analyzer/client.bal new file mode 100644 index 0000000..72c21eb --- /dev/null +++ b/examples/Sports-headline-analyzer/client.bal @@ -0,0 +1,222 @@ +// AUTO-GENERATED FILE. DO NOT MODIFY. +// This file is auto-generated by the Ballerina OpenAPI tool. + +// Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import ballerina/http; +import ballerina/mime; + +# The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. +public isolated client class Client { + final http:Client clientEp; + # Gets invoked to initialize the `connector`. + # + # + config - The configurations to be used when initializing the `connector` + # + serviceUrl - URL of the target service + # + return - An error if connector initialization failed + public isolated function init(ConnectionConfig config, string serviceUrl = "https://api.openai.com/v1") returns error? { + http:ClientConfiguration httpClientConfig = {auth: config.auth, httpVersion: config.httpVersion, timeout: config.timeout, forwarded: config.forwarded, poolConfig: config.poolConfig, compression: config.compression, circuitBreaker: config.circuitBreaker, retryConfig: config.retryConfig, validation: config.validation}; + do { + if config.http1Settings is ClientHttp1Settings { + ClientHttp1Settings settings = check config.http1Settings.ensureType(ClientHttp1Settings); + httpClientConfig.http1Settings = {...settings}; + } + if config.http2Settings is http:ClientHttp2Settings { + httpClientConfig.http2Settings = check config.http2Settings.ensureType(http:ClientHttp2Settings); + } + if config.cache is http:CacheConfig { + httpClientConfig.cache = check config.cache.ensureType(http:CacheConfig); + } + if config.responseLimits is http:ResponseLimitConfigs { + httpClientConfig.responseLimits = check config.responseLimits.ensureType(http:ResponseLimitConfigs); + } + if config.secureSocket is http:ClientSecureSocket { + httpClientConfig.secureSocket = check config.secureSocket.ensureType(http:ClientSecureSocket); + } + if config.proxy is http:ProxyConfig { + httpClientConfig.proxy = check config.proxy.ensureType(http:ProxyConfig); + } + } + http:Client httpEp = check new (serviceUrl, httpClientConfig); + self.clientEp = httpEp; + return; + } + + # Delete a file. + # + # + file_id - The ID of the file to use for this request. + # + headers - Headers to be sent with the request + # + return - OK + resource isolated function delete files/[string file_id](map headers = {}) returns DeleteFileResponse|error { + string resourcePath = string `/files/${getEncodedUri(file_id)}`; + return self.clientEp->delete(resourcePath, headers = headers); + } + + # Delete a fine-tuned model. You must have the Owner role in your organization to delete a model. + # + # + model - The model to delete + # + headers - Headers to be sent with the request + # + return - OK + resource isolated function delete models/[string model](map headers = {}) returns DeleteModelResponse|error { + string resourcePath = string `/models/${getEncodedUri(model)}`; + return self.clientEp->delete(resourcePath, headers = headers); + } + + # Returns a list of files that belong to the user's organization. + # + # + headers - Headers to be sent with the request + # + queries - Queries to be sent with the request + # + return - OK + resource isolated function get files(map headers = {}, *ListFilesQueries queries) returns ListFilesResponse|error { + string resourcePath = string `/files`; + resourcePath = resourcePath + check getPathForQueryParam(queries); + return self.clientEp->get(resourcePath, headers); + } + + # Returns information about a specific file. + # + # + file_id - The ID of the file to use for this request. + # + headers - Headers to be sent with the request + # + return - OK + resource isolated function get files/[string file_id](map headers = {}) returns OpenAIFile|error { + string resourcePath = string `/files/${getEncodedUri(file_id)}`; + return self.clientEp->get(resourcePath, headers); + } + + # Returns the contents of the specified file. + # + # + file_id - The ID of the file to use for this request. + # + headers - Headers to be sent with the request + # + return - OK + resource isolated function get files/[string file_id]/content(map headers = {}) returns byte[]|error { + string resourcePath = string `/files/${getEncodedUri(file_id)}/content`; + return self.clientEp->get(resourcePath, headers); + } + + # List your organization's fine-tuning jobs + # + # + headers - Headers to be sent with the request + # + queries - Queries to be sent with the request + # + return - OK + resource isolated function get fine_tuning/jobs(map headers = {}, *ListPaginatedFineTuningJobsQueries queries) returns ListPaginatedFineTuningJobsResponse|error { + string resourcePath = string `/fine_tuning/jobs`; + resourcePath = resourcePath + check getPathForQueryParam(queries); + return self.clientEp->get(resourcePath, headers); + } + + # Get info about a fine-tuning job. + # + # [Learn more about fine-tuning](/docs/guides/fine-tuning) + # + # + fine_tuning_job_id - The ID of the fine-tuning job. + # + headers - Headers to be sent with the request + # + return - OK + resource isolated function get fine_tuning/jobs/[string fine_tuning_job_id](map headers = {}) returns FineTuningJob|error { + string resourcePath = string `/fine_tuning/jobs/${getEncodedUri(fine_tuning_job_id)}`; + return self.clientEp->get(resourcePath, headers); + } + + # List checkpoints for a fine-tuning job. + # + # + fine_tuning_job_id - The ID of the fine-tuning job to get checkpoints for. + # + headers - Headers to be sent with the request + # + queries - Queries to be sent with the request + # + return - OK + resource isolated function get fine_tuning/jobs/[string fine_tuning_job_id]/checkpoints(map headers = {}, *ListFineTuningJobCheckpointsQueries queries) returns ListFineTuningJobCheckpointsResponse|error { + string resourcePath = string `/fine_tuning/jobs/${getEncodedUri(fine_tuning_job_id)}/checkpoints`; + resourcePath = resourcePath + check getPathForQueryParam(queries); + return self.clientEp->get(resourcePath, headers); + } + + # Get status updates for a fine-tuning job. + # + # + fine_tuning_job_id - The ID of the fine-tuning job to get events for. + # + headers - Headers to be sent with the request + # + queries - Queries to be sent with the request + # + return - OK + resource isolated function get fine_tuning/jobs/[string fine_tuning_job_id]/events(map headers = {}, *ListFineTuningEventsQueries queries) returns ListFineTuningJobEventsResponse|error { + string resourcePath = string `/fine_tuning/jobs/${getEncodedUri(fine_tuning_job_id)}/events`; + resourcePath = resourcePath + check getPathForQueryParam(queries); + return self.clientEp->get(resourcePath, headers); + } + + # Lists the currently available models, and provides basic information about each one such as the owner and availability. + # + # + headers - Headers to be sent with the request + # + return - OK + resource isolated function get models(map headers = {}) returns ListModelsResponse|error { + string resourcePath = string `/models`; + return self.clientEp->get(resourcePath, headers); + } + + # Retrieves a model instance, providing basic information about the model such as the owner and permissioning. + # + # + model - The ID of the model to use for this request + # + headers - Headers to be sent with the request + # + return - OK + resource isolated function get models/[string model](map headers = {}) returns Model|error { + string resourcePath = string `/models/${getEncodedUri(model)}`; + return self.clientEp->get(resourcePath, headers); + } + + # Upload a file that can be used across various endpoints. Individual files can be up to 512 MB, and the size of all files uploaded by one organization can be up to 100 GB. + # + # The Assistants API supports files up to 2 million tokens and of specific file types. See the [Assistants Tools guide](/docs/assistants/tools) for details. + # + # The Fine-tuning API only supports `.jsonl` files. The input also has certain required formats for fine-tuning [chat](/docs/api-reference/fine-tuning/chat-input) or [completions](/docs/api-reference/fine-tuning/completions-input) models. + # + # The Batch API only supports `.jsonl` files up to 100 MB in size. The input also has a specific required [format](/docs/api-reference/batch/request-input). + # + # Please [contact us](https://help.openai.com/) if you need to increase these storage limits. + # + # + headers - Headers to be sent with the request + # + return - OK + resource isolated function post files(CreateFileRequest payload, map headers = {}) returns OpenAIFile|error { + string resourcePath = string `/files`; + http:Request request = new; + mime:Entity[] bodyParts = check createBodyParts(payload); + request.setBodyParts(bodyParts); + return self.clientEp->post(resourcePath, request, headers); + } + + # Creates a fine-tuning job which begins the process of creating a new model from a given dataset. + # + # Response includes details of the enqueued job including job status and the name of the fine-tuned models once complete. + # + # [Learn more about fine-tuning](/docs/guides/fine-tuning) + # + # + headers - Headers to be sent with the request + # + return - OK + resource isolated function post fine_tuning/jobs(CreateFineTuningJobRequest payload, map headers = {}) returns FineTuningJob|error { + string resourcePath = string `/fine_tuning/jobs`; + http:Request request = new; + json jsonBody = payload.toJson(); + request.setPayload(jsonBody, "application/json"); + return self.clientEp->post(resourcePath, request, headers); + } + + # Immediately cancel a fine-tune job. + # + # + fine_tuning_job_id - The ID of the fine-tuning job to cancel. + # + headers - Headers to be sent with the request + # + return - OK + resource isolated function post fine_tuning/jobs/[string fine_tuning_job_id]/cancel(map headers = {}) returns FineTuningJob|error { + string resourcePath = string `/fine_tuning/jobs/${getEncodedUri(fine_tuning_job_id)}/cancel`; + http:Request request = new; + return self.clientEp->post(resourcePath, request, headers); + } +} diff --git a/examples/Sports-headline-analyzer/data/training.jsonl b/examples/Sports-headline-analyzer/data/training.jsonl new file mode 100644 index 0000000..d2539cf --- /dev/null +++ b/examples/Sports-headline-analyzer/data/training.jsonl @@ -0,0 +1,10 @@ +{"messages": [{"role": "system", "content": "Given a sports headline, provide the following fields in a JSON dict, where applicable: \"player\" (full name), \"team\", \"sport\", and \"gender\"."}, {"role": "user", "content": "Sources: Colts grant RB Taylor OK to seek trade"}, {"role": "assistant", "content": "{\"player\": \"Jonathan Taylor\", \"team\": \"Colts\", \"sport\": \"football\", \"gender\": \"male\" }"}]} +{"messages": [{"role": "system", "content": "Given a sports headline, provide the following fields in a JSON dict, where applicable: \"player\" (full name), \"team\", \"sport\", and \"gender\"."}, {"role": "user", "content": "OSU 'split down middle' on starting QB battle"}, {"role": "assistant", "content": "{\"player\": null, \"team\": \"OSU\", \"sport\": \"football\", \"gender\": null }"}]} +{"messages": [{"role": "system", "content": "Given a sports headline, provide the following fields in a JSON dict, where applicable: \"player\" (full name), \"team\", \"sport\", and \"gender\"."}, {"role": "user", "content": "Sources: Colts grant RB Taylor OK to seek trade"}, {"role": "assistant", "content": "{\"player\": \"Jonathan Taylor\", \"team\": \"Colts\", \"sport\": \"football\", \"gender\": \"male\"}"}]} +{"messages": [{"role": "system", "content": "Given a sports headline, provide the following fields in a JSON dict, where applicable: \"player\" (full name), \"team\", \"sport\", and \"gender\"."}, {"role": "user", "content": "OSU 'split down middle' on starting QB battle"}, {"role": "assistant", "content": "{\"player\": null, \"team\": \"OSU\", \"sport\": \"football\", \"gender\": null}"}]} +{"messages": [{"role": "system", "content": "Given a sports headline, provide the following fields in a JSON dict, where applicable: \"player\" (full name), \"team\", \"sport\", and \"gender\"."}, {"role": "user", "content": "Richardson wins 100m at worlds to cap comeback"}, {"role": "assistant", "content": "{\"player\": \"Sha'Carri Richardson\", \"team\": null, \"sport\": \"track and field\", \"gender\": \"female\"}"}]} +{"messages": [{"role": "system", "content": "Given a sports headline, provide the following fields in a JSON dict, where applicable: \"player\" (full name), \"team\", \"sport\", and \"gender\"."}, {"role": "user", "content": "LeBron leads Lakers past Warriors in season opener"}, {"role": "assistant", "content": "{\"player\": \"LeBron James\", \"team\": \"Lakers\", \"sport\": \"basketball\", \"gender\": \"male\"}"}]} +{"messages": [{"role": "system", "content": "Given a sports headline, provide the following fields in a JSON dict, where applicable: \"player\" (full name), \"team\", \"sport\", and \"gender\"."}, {"role": "user", "content": "Serena Williams announces retirement after US Open"}, {"role": "assistant", "content": "{\"player\": \"Serena Williams\", \"team\": null, \"sport\": \"tennis\", \"gender\": \"female\"}"}]} +{"messages": [{"role": "system", "content": "Given a sports headline, provide the following fields in a JSON dict, where applicable: \"player\" (full name), \"team\", \"sport\", and \"gender\"."}, {"role": "user", "content": "Richardson wins 100m at worlds to cap comeback"}, {"role": "assistant", "content": "{\"player\": \"Sha'Carri Richardson\", \"team\": null, \"sport\": \"track and field\", \"gender\": \"female\"}"}]} +{"messages": [{"role": "system", "content": "Given a sports headline, provide the following fields in a JSON dict, where applicable: \"player\" (full name), \"team\", \"sport\", and \"gender\"."}, {"role": "user", "content": "LeBron leads Lakers past Warriors in season opener"}, {"role": "assistant", "content": "{\"player\": \"LeBron James\", \"team\": \"Lakers\", \"sport\": \"basketball\", \"gender\": \"male\"}"}]} +{"messages": [{"role": "system", "content": "Given a sports headline, provide the following fields in a JSON dict, where applicable: \"player\" (full name), \"team\", \"sport\", and \"gender\"."}, {"role": "user", "content": "Serena Williams announces retirement after US Open"}, {"role": "assistant", "content": "{\"player\": \"Serena Williams\", \"team\": null, \"sport\": \"tennis\", \"gender\": \"female\"}"}]} diff --git a/examples/Sports-headline-analyzer/main.bal b/examples/Sports-headline-analyzer/main.bal new file mode 100644 index 0000000..bb0a48f --- /dev/null +++ b/examples/Sports-headline-analyzer/main.bal @@ -0,0 +1,108 @@ +import ballerina/io; +import ballerina/lang.runtime; + +// import ballerinax/openai.finetunes; + +configurable string token = ?; +string serviceUrl = "https://api.openai.com/v1"; + +string trainingFileName = "training.jsonl"; +string trainingFilePath = "./data/" + trainingFileName; + +final ConnectionConfig config = {auth: {token}}; +final Client openAIFinetunes = check new Client(config, serviceUrl); + +public function main() returns error? { + + byte[] trainingFileContent = check io:fileReadBytes(trainingFilePath); + + CreateFileRequest trainingFileRequest = { + file: {fileContent: trainingFileContent, fileName: trainingFileName}, + purpose: "fine-tune" + }; + + OpenAIFile trainingFileResponse = + check openAIFinetunes->/files.post(trainingFileRequest); + + string trainingFileId = trainingFileResponse.id; + io:println("Training file id: " + trainingFileId); + + CreateFineTuningJobRequest fineTuneRequest = { + model: "gpt-4o-mini-2024-07-18", + training_file: trainingFileId, + hyperparameters: { + n_epochs: 15, + batch_size: 3, + learning_rate_multiplier: 0.3 + } + }; + + FineTuningJob fineTuneResponse = + check openAIFinetunes->/fine_tuning/jobs.post(fineTuneRequest); + string fineTuneJobId = fineTuneResponse.id; + io:println("Fine-tuning job id: " + fineTuneJobId); + + FineTuningJob fineTuneJob = + check openAIFinetunes->/fine_tuning/jobs/[fineTuneJobId].get(); + + io:print("Validating files..."); + while (fineTuneJob.status == "validating_files") { + io:print("."); + fineTuneJob = check openAIFinetunes->/fine_tuning/jobs/[fineTuneJobId].get(); + runtime:sleep(1); + } + + io:print("\nFiles validated successfully."); + while (fineTuneJob.status == "queued") { + io:print("."); + fineTuneJob = check openAIFinetunes->/fine_tuning/jobs/[fineTuneJobId].get(); + runtime:sleep(1); + } + + io:println("\nTraining..."); + while (fineTuneJob.status == "running") { + fineTuneJob = check openAIFinetunes->/fine_tuning/jobs/[fineTuneJobId].get(); + io:print("."); + runtime:sleep(1); + } + + + if (fineTuneJob.status != "succeeded") { + io:println("Fine-tuning job failed."); + DeleteFileResponse deleteFileResponse = + check openAIFinetunes->/files/[trainingFileId].delete(); + if (deleteFileResponse.deleted == true) { + io:println("Training file deleted successfully."); + } else { + io:println("Failed to delete the training file."); + } + + return; + }else { + io:println("\n"); + } + + ListFineTuningJobCheckpointsResponse checkpointsResponse = + check openAIFinetunes->/fine_tuning/jobs/[fineTuneJobId]/checkpoints.get(); + + foreach FineTuningJobCheckpoint item in checkpointsResponse.data { + io:print("step: ", item.metrics.step); + io:print(", train loss: ", item.metrics.train_loss); + io:println(", train mean token accuracy: ", item.metrics.train_mean_token_accuracy); + } + + io:println("\nFine-tuning job details: "); + io:println("Fine-tuned Model: ", fineTuneJob.fine_tuned_model); + io:println("Organization ID: ", fineTuneJob.organization_id); + io:println("Model: ", fineTuneJob.model); + io:println("Fine-tuning job completed successfully."); + + DeleteFileResponse deleteFileResponse = + check openAIFinetunes->/files/[trainingFileId].delete(); + if (deleteFileResponse.deleted == true) { + io:println("Training file deleted successfully."); + } else { + io:println("Failed to delete the training file."); + } + +} diff --git a/examples/Sports-headline-analyzer/types.bal b/examples/Sports-headline-analyzer/types.bal new file mode 100644 index 0000000..093632d --- /dev/null +++ b/examples/Sports-headline-analyzer/types.bal @@ -0,0 +1,361 @@ +// AUTO-GENERATED FILE. DO NOT MODIFY. +// This file is auto-generated by the Ballerina OpenAPI tool. + +// Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import ballerina/http; + +# The `File` object represents a document that has been uploaded to OpenAI. +public type OpenAIFile record { + # The file identifier, which can be referenced in the API endpoints. + string id; + # The size of the file, in bytes. + int bytes; + # The Unix timestamp (in seconds) for when the file was created. + int created_at; + # The name of the file. + string filename; + # The object type, which is always `file`. + "file" 'object; + # The intended purpose of the file. Supported values are `assistants`, `assistants_output`, `batch`, `batch_output`, `fine-tune`, `fine-tune-results` and `vision`. + "assistants"|"assistants_output"|"batch"|"batch_output"|"fine-tune"|"fine-tune-results"|"vision" purpose; + # Deprecated. The current status of the file, which can be either `uploaded`, `processed`, or `error`. + # + # # Deprecated + @deprecated + "uploaded"|"processed"|"error" status; + # Deprecated. For details on why a fine-tuning training file failed validation, see the `error` field on `fine_tuning.job`. + string? status_details?; +}; + +public type CreateFineTuningJobRequest record { + # The name of the model to fine-tune. You can select one of the + # [supported models](/docs/guides/fine-tuning/what-models-can-be-fine-tuned). + string|"babbage-002"|"davinci-002"|"gpt-3.5-turbo" model; + # The ID of an uploaded file that contains training data. + # + # See [upload file](/docs/api-reference/files/create) for how to upload a file. + # + # Your dataset must be formatted as a JSONL file. Additionally, you must upload your file with the purpose `fine-tune`. + # + # The contents of the file should differ depending on if the model uses the [chat](/docs/api-reference/fine-tuning/chat-input) or [completions](/docs/api-reference/fine-tuning/completions-input) format. + # + # See the [fine-tuning guide](/docs/guides/fine-tuning) for more details. + string training_file; + CreateFineTuningJobRequest_hyperparameters hyperparameters?; + # A string of up to 18 characters that will be added to your fine-tuned model name. + # + # For example, a `suffix` of "custom-model-name" would produce a model name like `ft:gpt-3.5-turbo:openai:custom-model-name:7p4lURel`. + string? suffix?; + # The ID of an uploaded file that contains validation data. + # + # If you provide this file, the data is used to generate validation + # metrics periodically during fine-tuning. These metrics can be viewed in + # the fine-tuning results file. + # The same data should not be present in both train and validation files. + # + # Your dataset must be formatted as a JSONL file. You must upload your file with the purpose `fine-tune`. + # + # See the [fine-tuning guide](/docs/guides/fine-tuning) for more details. + string? validation_file?; + # A list of integrations to enable for your fine-tuning job. + CreateFineTuningJobRequest_integrations[]? integrations?; + # The seed controls the reproducibility of the job. Passing in the same seed and job parameters should produce the same results, but may differ in rare cases. + # If a seed is not specified, one will be generated for you. + int? seed?; +}; + +# The `fine_tuning.job.checkpoint` object represents a model checkpoint for a fine-tuning job that is ready to use. +public type FineTuningJobCheckpoint record { + # The checkpoint identifier, which can be referenced in the API endpoints. + string id; + # The Unix timestamp (in seconds) for when the checkpoint was created. + int created_at; + # The name of the fine-tuned checkpoint model that is created. + string fine_tuned_model_checkpoint; + # The step number that the checkpoint was created at. + int step_number; + FineTuningJobCheckpoint_metrics metrics; + # The name of the fine-tuning job that this checkpoint was created from. + string fine_tuning_job_id; + # The object type, which is always "fine_tuning.job.checkpoint". + "fine_tuning.job.checkpoint" 'object; +}; + +public type ListPaginatedFineTuningJobsResponse record { + FineTuningJob[] data; + boolean has_more; + "list" 'object; +}; + +# The hyperparameters used for the fine-tuning job. +public type CreateFineTuningJobRequest_hyperparameters record { + # Number of examples in each batch. A larger batch size means that model parameters + # are updated less frequently, but with lower variance. + "auto"|int batch_size = "auto"; + # Scaling factor for the learning rate. A smaller learning rate may be useful to avoid + # overfitting. + "auto"|decimal learning_rate_multiplier = "auto"; + # The number of epochs to train the model for. An epoch refers to one full cycle + # through the training dataset. + "auto"|int n_epochs = "auto"; +}; + +# Represents the Queries record for the operation: listFineTuningEvents +public type ListFineTuningEventsQueries record { + # Number of events to retrieve. + int 'limit = 20; + # Identifier for the last event from the previous pagination request. + string after?; +}; + +# Represents the Queries record for the operation: listFineTuningJobCheckpoints +public type ListFineTuningJobCheckpointsQueries record { + # Number of checkpoints to retrieve. + int 'limit = 10; + # Identifier for the last checkpoint ID from the previous pagination request. + string after?; +}; + +# The settings for your integration with Weights and Biases. This payload specifies the project that +# metrics will be sent to. Optionally, you can set an explicit display name for your run, add tags +# to your run, and set a default entity (team, username, etc) to be associated with your run. +public type CreateFineTuningJobRequest_wandb record { + # The name of the project that the new run will be created under. + string project; + # A display name to set for the run. If not set, we will use the Job ID as the name. + string? name?; + # The entity to use for the run. This allows you to set the team or username of the WandB user that you would + # like associated with the run. If not set, the default entity for the registered WandB API key is used. + string? entity?; + # A list of tags to be attached to the newly created run. These tags are passed through directly to WandB. Some + # default tags are generated by OpenAI: "openai/finetune", "openai/{base-model}", "openai/{ftjob-abcdef}". + string[] tags?; +}; + +# The hyperparameters used for the fine-tuning job. See the [fine-tuning guide](/docs/guides/fine-tuning) for more details. +public type FineTuningJob_hyperparameters record { + # The number of epochs to train the model for. An epoch refers to one full cycle through the training dataset. + # "auto" decides the optimal number of epochs based on the size of the dataset. If setting the number manually, we support any number between 1 and 50 epochs. + "auto"|int n_epochs; +}; + +# Represents the Queries record for the operation: listFiles +public type ListFilesQueries record { + # Only return files with the given purpose. + string purpose?; +}; + +public type ListFineTuningJobCheckpointsResponse record { + FineTuningJobCheckpoint[] data; + "list" 'object; + string? first_id?; + string? last_id?; + boolean has_more; +}; + +public type DeleteModelResponse record { + string id; + boolean deleted; + string 'object; +}; + +public type FineTuningIntegration record { + # The type of the integration being enabled for the fine-tuning job + "wandb" 'type; + CreateFineTuningJobRequest_wandb wandb; +}; + +# Provides a set of configurations for controlling the behaviours when communicating with a remote HTTP endpoint. +@display {label: "Connection Config"} +public type ConnectionConfig record {| + # Configurations related to client authentication + http:BearerTokenConfig auth; + # The HTTP version understood by the client + http:HttpVersion httpVersion = http:HTTP_2_0; + # Configurations related to HTTP/1.x protocol + ClientHttp1Settings http1Settings?; + # Configurations related to HTTP/2 protocol + http:ClientHttp2Settings http2Settings?; + # The maximum time to wait (in seconds) for a response before closing the connection + decimal timeout = 60; + # The choice of setting `forwarded`/`x-forwarded` header + string forwarded = "disable"; + # Configurations associated with request pooling + http:PoolConfiguration poolConfig?; + # HTTP caching related configurations + http:CacheConfig cache?; + # Specifies the way of handling compression (`accept-encoding`) header + http:Compression compression = http:COMPRESSION_AUTO; + # Configurations associated with the behaviour of the Circuit Breaker + http:CircuitBreakerConfig circuitBreaker?; + # Configurations associated with retrying + http:RetryConfig retryConfig?; + # Configurations associated with inbound response size limits + http:ResponseLimitConfigs responseLimits?; + # SSL/TLS-related options + http:ClientSecureSocket secureSocket?; + # Proxy server related options + http:ProxyConfig proxy?; + # Enables the inbound payload validation functionality which provided by the constraint package. Enabled by default + boolean validation = true; +|}; + +# Metrics at the step number during the fine-tuning job. +public type FineTuningJobCheckpoint_metrics record { + decimal step?; + decimal train_loss?; + decimal train_mean_token_accuracy?; + decimal valid_loss?; + decimal valid_mean_token_accuracy?; + decimal full_valid_loss?; + decimal full_valid_mean_token_accuracy?; +}; + +public type CreateFileRequest record {| + # The File object (not file name) to be uploaded. + record {byte[] fileContent; string fileName;} file; + # The intended purpose of the uploaded file. + # + # Use "assistants" for [Assistants](/docs/api-reference/assistants) and [Message](/docs/api-reference/messages) files, "vision" for Assistants image file inputs, "batch" for [Batch API](/docs/guides/batch), and "fine-tune" for [Fine-tuning](/docs/api-reference/fine-tuning). + "assistants"|"batch"|"fine-tune"|"vision" purpose; +|}; + +# The `fine_tuning.job` object represents a fine-tuning job that has been created through the API. +public type FineTuningJob record { + # The object identifier, which can be referenced in the API endpoints. + string id; + # The Unix timestamp (in seconds) for when the fine-tuning job was created. + int created_at; + FineTuningJob_error? 'error; + # The name of the fine-tuned model that is being created. The value will be null if the fine-tuning job is still running. + string? fine_tuned_model; + # The Unix timestamp (in seconds) for when the fine-tuning job was finished. The value will be null if the fine-tuning job is still running. + int? finished_at; + FineTuningJob_hyperparameters hyperparameters; + # The base model that is being fine-tuned. + string model; + # The object type, which is always "fine_tuning.job". + "fine_tuning.job" 'object; + # The organization that owns the fine-tuning job. + string organization_id; + # The compiled results file ID(s) for the fine-tuning job. You can retrieve the results with the [Files API](/docs/api-reference/files/retrieve-contents). + string[] result_files; + # The current status of the fine-tuning job, which can be either `validating_files`, `queued`, `running`, `succeeded`, `failed`, or `cancelled`. + "validating_files"|"queued"|"running"|"succeeded"|"failed"|"cancelled" status; + # The total number of billable tokens processed by this fine-tuning job. The value will be null if the fine-tuning job is still running. + int? trained_tokens; + # The file ID used for training. You can retrieve the training data with the [Files API](/docs/api-reference/files/retrieve-contents). + string training_file; + # The file ID used for validation. You can retrieve the validation results with the [Files API](/docs/api-reference/files/retrieve-contents). + string? validation_file; + # A list of integrations to enable for this fine-tuning job. + (FineTuningIntegration)[]? integrations?; + # The seed used for the fine-tuning job. + int seed; + # The Unix timestamp (in seconds) for when the fine-tuning job is estimated to finish. The value will be null if the fine-tuning job is not running. + int? estimated_finish?; +}; + +# Proxy server configurations to be used with the HTTP client endpoint. +public type ProxyConfig record {| + # Host name of the proxy server + string host = ""; + # Proxy server port + int port = 0; + # Proxy server username + string userName = ""; + # Proxy server password + @display {label: "", kind: "password"} + string password = ""; +|}; + +# For fine-tuning jobs that have `failed`, this will contain more information on the cause of the failure. +public type FineTuningJob_error record { + # A machine-readable error code. + string code?; + # A human-readable error message. + string message?; + # The parameter that was invalid, usually `training_file` or `validation_file`. This field will be null if the failure was not parameter-specific. + string? param?; +}; + +public type CreateFineTuningJobRequest_integrations record { + # The type of integration to enable. Currently, only "wandb" (Weights and Biases) is supported. + "wandb" 'type; + CreateFineTuningJobRequest_wandb wandb; +}; + +public type ListModelsResponse record { + "list" 'object; + Model[] data; +}; + +# Fine-tuning job event object +public type FineTuningJobEvent record { + string id; + int created_at; + "info"|"warn"|"error" level; + string message; + "fine_tuning.job.event" 'object; +}; + +public type DeleteFileResponse record { + string id; + "file" 'object; + boolean deleted; +}; + +# Provides settings related to HTTP/1.x protocol. +public type ClientHttp1Settings record {| + # Specifies whether to reuse a connection for multiple requests + http:KeepAlive keepAlive = http:KEEPALIVE_AUTO; + # The chunking behaviour of the request + http:Chunking chunking = http:CHUNKING_AUTO; + # Proxy server related options + ProxyConfig proxy?; +|}; + +# Describes an OpenAI model offering that can be used with the API. +public type Model record { + # The model identifier, which can be referenced in the API endpoints. + string id; + # The Unix timestamp (in seconds) when the model was created. + int created; + # The object type, which is always "model". + "model" 'object; + # The organization that owns the model. + string owned_by; +}; + +public type ListFineTuningJobEventsResponse record { + FineTuningJobEvent[] data; + "list" 'object; +}; + +public type ListFilesResponse record { + OpenAIFile[] data; + "list" 'object; +}; + +# Represents the Queries record for the operation: listPaginatedFineTuningJobs +public type ListPaginatedFineTuningJobsQueries record { + # Number of fine-tuning jobs to retrieve. + int 'limit = 20; + # Identifier for the last job from the previous pagination request. + string after?; +}; diff --git a/examples/Sports-headline-analyzer/utils.bal b/examples/Sports-headline-analyzer/utils.bal new file mode 100644 index 0000000..0ce7ca0 --- /dev/null +++ b/examples/Sports-headline-analyzer/utils.bal @@ -0,0 +1,252 @@ +// AUTO-GENERATED FILE. DO NOT MODIFY. +// This file is auto-generated by the Ballerina OpenAPI tool. + +// Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import ballerina/mime; +import ballerina/url; + +type SimpleBasicType string|boolean|int|float|decimal; + +# Represents encoding mechanism details. +type Encoding record { + # Defines how multiple values are delimited + string style = FORM; + # Specifies whether arrays and objects should generate as separate fields + boolean explode = true; + # Specifies the custom content type + string contentType?; + # Specifies the custom headers + map headers?; +}; + +enum EncodingStyle { + DEEPOBJECT, FORM, SPACEDELIMITED, PIPEDELIMITED +} + +final Encoding & readonly defaultEncoding = {}; + +# Serialize the record according to the deepObject style. +# +# + parent - Parent record name +# + anyRecord - Record to be serialized +# + return - Serialized record as a string +isolated function getDeepObjectStyleRequest(string parent, record {} anyRecord) returns string { + string[] recordArray = []; + foreach [string, anydata] [key, value] in anyRecord.entries() { + if value is SimpleBasicType { + recordArray.push(parent + "[" + key + "]" + "=" + getEncodedUri(value.toString())); + } else if value is SimpleBasicType[] { + recordArray.push(getSerializedArray(parent + "[" + key + "]" + "[]", value, DEEPOBJECT, true)); + } else if value is record {} { + string nextParent = parent + "[" + key + "]"; + recordArray.push(getDeepObjectStyleRequest(nextParent, value)); + } else if value is record {}[] { + string nextParent = parent + "[" + key + "]"; + recordArray.push(getSerializedRecordArray(nextParent, value, DEEPOBJECT)); + } + recordArray.push("&"); + } + _ = recordArray.pop(); + return string:'join("", ...recordArray); +} + +# Serialize the record according to the form style. +# +# + parent - Parent record name +# + anyRecord - Record to be serialized +# + explode - Specifies whether arrays and objects should generate separate parameters +# + return - Serialized record as a string +isolated function getFormStyleRequest(string parent, record {} anyRecord, boolean explode = true) returns string { + string[] recordArray = []; + if explode { + foreach [string, anydata] [key, value] in anyRecord.entries() { + if value is SimpleBasicType { + recordArray.push(key, "=", getEncodedUri(value.toString())); + } else if value is SimpleBasicType[] { + recordArray.push(getSerializedArray(key, value, explode = explode)); + } else if value is record {} { + recordArray.push(getFormStyleRequest(parent, value, explode)); + } + recordArray.push("&"); + } + _ = recordArray.pop(); + } else { + foreach [string, anydata] [key, value] in anyRecord.entries() { + if value is SimpleBasicType { + recordArray.push(key, ",", getEncodedUri(value.toString())); + } else if value is SimpleBasicType[] { + recordArray.push(getSerializedArray(key, value, explode = false)); + } else if value is record {} { + recordArray.push(getFormStyleRequest(parent, value, explode)); + } + recordArray.push(","); + } + _ = recordArray.pop(); + } + return string:'join("", ...recordArray); +} + +# Serialize arrays. +# +# + arrayName - Name of the field with arrays +# + anyArray - Array to be serialized +# + style - Defines how multiple values are delimited +# + explode - Specifies whether arrays and objects should generate separate parameters +# + return - Serialized array as a string +isolated function getSerializedArray(string arrayName, anydata[] anyArray, string style = "form", boolean explode = true) returns string { + string key = arrayName; + string[] arrayValues = []; + if anyArray.length() > 0 { + if style == FORM && !explode { + arrayValues.push(key, "="); + foreach anydata i in anyArray { + arrayValues.push(getEncodedUri(i.toString()), ","); + } + } else if style == SPACEDELIMITED && !explode { + arrayValues.push(key, "="); + foreach anydata i in anyArray { + arrayValues.push(getEncodedUri(i.toString()), "%20"); + } + } else if style == PIPEDELIMITED && !explode { + arrayValues.push(key, "="); + foreach anydata i in anyArray { + arrayValues.push(getEncodedUri(i.toString()), "|"); + } + } else if style == DEEPOBJECT { + foreach anydata i in anyArray { + arrayValues.push(key, "[]", "=", getEncodedUri(i.toString()), "&"); + } + } else { + foreach anydata i in anyArray { + arrayValues.push(key, "=", getEncodedUri(i.toString()), "&"); + } + } + _ = arrayValues.pop(); + } + return string:'join("", ...arrayValues); +} + +# Serialize the array of records according to the form style. +# +# + parent - Parent record name +# + value - Array of records to be serialized +# + style - Defines how multiple values are delimited +# + explode - Specifies whether arrays and objects should generate separate parameters +# + return - Serialized record as a string +isolated function getSerializedRecordArray(string parent, record {}[] value, string style = FORM, boolean explode = true) returns string { + string[] serializedArray = []; + if style == DEEPOBJECT { + int arayIndex = 0; + foreach var recordItem in value { + serializedArray.push(getDeepObjectStyleRequest(parent + "[" + arayIndex.toString() + "]", recordItem), "&"); + arayIndex = arayIndex + 1; + } + } else { + if !explode { + serializedArray.push(parent, "="); + } + foreach var recordItem in value { + serializedArray.push(getFormStyleRequest(parent, recordItem, explode), ","); + } + } + _ = serializedArray.pop(); + return string:'join("", ...serializedArray); +} + +# Get Encoded URI for a given value. +# +# + value - Value to be encoded +# + return - Encoded string +isolated function getEncodedUri(anydata value) returns string { + string|error encoded = url:encode(value.toString(), "UTF8"); + if encoded is string { + return encoded; + } else { + return value.toString(); + } +} + +# Generate query path with query parameter. +# +# + queryParam - Query parameter map +# + encodingMap - Details on serialization mechanism +# + return - Returns generated Path or error at failure of client initialization +isolated function getPathForQueryParam(map queryParam, map encodingMap = {}) returns string|error { + string[] param = []; + if queryParam.length() > 0 { + param.push("?"); + foreach var [key, value] in queryParam.entries() { + if value is () { + _ = queryParam.remove(key); + continue; + } + Encoding encodingData = encodingMap.hasKey(key) ? encodingMap.get(key) : defaultEncoding; + if value is SimpleBasicType { + param.push(key, "=", getEncodedUri(value.toString())); + } else if value is SimpleBasicType[] { + param.push(getSerializedArray(key, value, encodingData.style, encodingData.explode)); + } else if value is record {} { + if encodingData.style == DEEPOBJECT { + param.push(getDeepObjectStyleRequest(key, value)); + } else { + param.push(getFormStyleRequest(key, value, encodingData.explode)); + } + } else { + param.push(key, "=", value.toString()); + } + param.push("&"); + } + _ = param.pop(); + } + string restOfPath = string:'join("", ...param); + return restOfPath; +} + +isolated function createBodyParts(record {|anydata...;|} anyRecord, map encodingMap = {}) returns mime:Entity[]|error { + mime:Entity[] entities = []; + foreach [string, anydata] [key, value] in anyRecord.entries() { + Encoding encodingData = encodingMap.hasKey(key) ? encodingMap.get(key) : {}; + mime:Entity entity = new mime:Entity(); + if value is record {byte[] fileContent; string fileName;} { + entity.setContentDisposition(mime:getContentDispositionObject(string `form-data; name=${key}; filename=${value.fileName}`)); + entity.setByteArray(value.fileContent); + } else if value is byte[] { + entity.setContentDisposition(mime:getContentDispositionObject(string `form-data; name=${key};`)); + entity.setByteArray(value); + } else if value is SimpleBasicType|SimpleBasicType[] { + entity.setContentDisposition(mime:getContentDispositionObject(string `form-data; name=${key};`)); + entity.setText(value.toString()); + } else if value is record {}|record {}[] { + entity.setContentDisposition(mime:getContentDispositionObject(string `form-data; name=${key};`)); + entity.setJson(value.toJson()); + } + if encodingData?.contentType is string { + check entity.setContentType(encodingData?.contentType.toString()); + } + map? headers = encodingData?.headers; + if headers is map { + foreach var [headerName, headerValue] in headers.entries() { + if headerValue is SimpleBasicType { + entity.setHeader(headerName, headerValue.toString()); + } + } + } + entities.push(entity); + } + return entities; +} From 9b0ea37feedce46bd7c1a1c06c0a2e62b6e4d577 Mon Sep 17 00:00:00 2001 From: KATTA-00 Date: Wed, 14 Aug 2024 10:27:59 +0530 Subject: [PATCH 04/32] Update the printing messages in examples --- examples/Sarcastic-bot/main.bal | 1 - .../Sports-headline-analyzer/Sports headline analyzer.md | 2 +- examples/Sports-headline-analyzer/main.bal | 9 +++------ 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/examples/Sarcastic-bot/main.bal b/examples/Sarcastic-bot/main.bal index 9a5ad9a..3e7ecfa 100644 --- a/examples/Sarcastic-bot/main.bal +++ b/examples/Sarcastic-bot/main.bal @@ -88,7 +88,6 @@ public function main() returns error? { io:println("\nFine-tuning job details: "); io:println("Fine-tuned Model: ", fineTuneJob.fine_tuned_model); - io:println("Organization ID: ", fineTuneJob.organization_id); io:println("Model: ", fineTuneJob.model); io:println("Fine-tuning job completed successfully."); } diff --git a/examples/Sports-headline-analyzer/Sports headline analyzer.md b/examples/Sports-headline-analyzer/Sports headline analyzer.md index bfabeff..888af3b 100644 --- a/examples/Sports-headline-analyzer/Sports headline analyzer.md +++ b/examples/Sports-headline-analyzer/Sports headline analyzer.md @@ -1,7 +1,7 @@ ## Sports Headlines Analyzer -This use case demonstrates how the OpenAI Fine-tunes API v1 can be harnessed to fine-tune the GPT-4o-mini model to extract structured information from sports headlines. The example showcases a series of steps that utilize the OpenAI Files API v1 to upload training data, followed by using the OpenAI Fine-tunes API v1 to fine-tune the GPT-4o-mini model with that data. +This use case illustrates how the OpenAI Fine-tunes API v1 can be used to fine-tune the GPT-4o-mini model for extracting structured information from sports headlines. The example outlines a series of steps that include using the OpenAI Files API v1 to upload training data, then employing the OpenAI Fine-tunes API v1 to fine-tune the GPT-4o-mini model with this data, and finally printing the model's checkpoints and deleting the data file. ## Prerequisites diff --git a/examples/Sports-headline-analyzer/main.bal b/examples/Sports-headline-analyzer/main.bal index bb0a48f..520ed76 100644 --- a/examples/Sports-headline-analyzer/main.bal +++ b/examples/Sports-headline-analyzer/main.bal @@ -59,14 +59,13 @@ public function main() returns error? { runtime:sleep(1); } - io:println("\nTraining..."); + io:print("\nTraining..."); while (fineTuneJob.status == "running") { fineTuneJob = check openAIFinetunes->/fine_tuning/jobs/[fineTuneJobId].get(); io:print("."); runtime:sleep(1); } - if (fineTuneJob.status != "succeeded") { io:println("Fine-tuning job failed."); DeleteFileResponse deleteFileResponse = @@ -78,14 +77,13 @@ public function main() returns error? { } return; - }else { - io:println("\n"); } + io:println("\n"); ListFineTuningJobCheckpointsResponse checkpointsResponse = check openAIFinetunes->/fine_tuning/jobs/[fineTuneJobId]/checkpoints.get(); - foreach FineTuningJobCheckpoint item in checkpointsResponse.data { + foreach FineTuningJobCheckpoint item in checkpointsResponse.data.reverse() { io:print("step: ", item.metrics.step); io:print(", train loss: ", item.metrics.train_loss); io:println(", train mean token accuracy: ", item.metrics.train_mean_token_accuracy); @@ -93,7 +91,6 @@ public function main() returns error? { io:println("\nFine-tuning job details: "); io:println("Fine-tuned Model: ", fineTuneJob.fine_tuned_model); - io:println("Organization ID: ", fineTuneJob.organization_id); io:println("Model: ", fineTuneJob.model); io:println("Fine-tuning job completed successfully."); From 8bbc2fad25cacd4bdc36db60315c07888827a316 Mon Sep 17 00:00:00 2001 From: KATTA-00 Date: Wed, 14 Aug 2024 10:33:29 +0530 Subject: [PATCH 05/32] Update the README.md of examples --- examples/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/README.md b/examples/README.md index 60c9065..3a55ec8 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,6 +1,6 @@ # Examples -The `ballerinax/openai.finetunes` connector provides practical examples illustrating usage in various scenarios. Explore these [examples](https://github.com/ballerina-platform/module-ballerinax-openai.finetunes/tree/main/examples), covering use cases like cache management, session management, and rate limiting. +The `ballerinax/openai.finetunes` connector provides practical examples illustrating usage in various scenarios. Explore these [examples](https://github.com/ballerina-platform/module-ballerinax-openai.finetunes/tree/main/examples), covering use cases like file uploading, finetuning models, getting events/checkpoints of models and deleting files. 1. [Sarcastic Bot](https://github.com/ballerina-platform/module-ballerinax-openai.finetunes/tree/main/examples/Sarcastic-bot) - Fine-tune the GPT-3.5-turbo model to generate sarcastic responses From 00e93958abe52e1be31b68f5473835944eaba5e0 Mon Sep 17 00:00:00 2001 From: KATTA-00 Date: Wed, 14 Aug 2024 11:18:13 +0530 Subject: [PATCH 06/32] Remove unwanted lines --- examples/Sarcastic-bot/main.bal | 1 - examples/Sports-headline-analyzer/main.bal | 1 - 2 files changed, 2 deletions(-) diff --git a/examples/Sarcastic-bot/main.bal b/examples/Sarcastic-bot/main.bal index 3e7ecfa..97ddd72 100644 --- a/examples/Sarcastic-bot/main.bal +++ b/examples/Sarcastic-bot/main.bal @@ -5,7 +5,6 @@ import ballerina/lang.runtime; configurable string token = ?; string serviceUrl = "https://api.openai.com/v1"; - string trainingFileName = "training.jsonl"; string validationFileName = "validation.jsonl"; string trainingFilePath = "./data/" + trainingFileName; diff --git a/examples/Sports-headline-analyzer/main.bal b/examples/Sports-headline-analyzer/main.bal index 520ed76..c5f826b 100644 --- a/examples/Sports-headline-analyzer/main.bal +++ b/examples/Sports-headline-analyzer/main.bal @@ -5,7 +5,6 @@ import ballerina/lang.runtime; configurable string token = ?; string serviceUrl = "https://api.openai.com/v1"; - string trainingFileName = "training.jsonl"; string trainingFilePath = "./data/" + trainingFileName; From 15bb1c048642a916fc8f2b91f9bc3f88afc57258 Mon Sep 17 00:00:00 2001 From: KATTA-00 Date: Wed, 14 Aug 2024 15:26:39 +0530 Subject: [PATCH 07/32] Update imports of examples --- ballerina/Ballerina.toml | 2 +- ballerina/Dependencies.toml | 6 +- examples/Sarcastic-bot/Ballerina.toml | 6 + examples/Sarcastic-bot/Dependencies.toml | 310 --------------- examples/Sarcastic-bot/client.bal | 222 ----------- examples/Sarcastic-bot/main.bal | 25 +- examples/Sarcastic-bot/types.bal | 361 ------------------ examples/Sarcastic-bot/utils.bal | 252 ------------ .../Sports-headline-analyzer/Ballerina.toml | 6 + examples/Sports-headline-analyzer/client.bal | 222 ----------- examples/Sports-headline-analyzer/main.bal | 26 +- examples/Sports-headline-analyzer/types.bal | 361 ------------------ examples/Sports-headline-analyzer/utils.bal | 252 ------------ 13 files changed, 40 insertions(+), 2011 deletions(-) delete mode 100644 examples/Sarcastic-bot/Dependencies.toml delete mode 100644 examples/Sarcastic-bot/client.bal delete mode 100644 examples/Sarcastic-bot/types.bal delete mode 100644 examples/Sarcastic-bot/utils.bal delete mode 100644 examples/Sports-headline-analyzer/client.bal delete mode 100644 examples/Sports-headline-analyzer/types.bal delete mode 100644 examples/Sports-headline-analyzer/utils.bal diff --git a/ballerina/Ballerina.toml b/ballerina/Ballerina.toml index d2c7c22..03e6c34 100644 --- a/ballerina/Ballerina.toml +++ b/ballerina/Ballerina.toml @@ -2,7 +2,7 @@ distribution = "2201.9.2" org = "ballerinax" name = "openai.finetunes" -version = "1.0.5" +version = "1.0.7" license = ["Apache-2.0"] authors = ["Ballerina"] keywords = [] # TODO: Add keywords diff --git a/ballerina/Dependencies.toml b/ballerina/Dependencies.toml index a1b101d..b70a49e 100644 --- a/ballerina/Dependencies.toml +++ b/ballerina/Dependencies.toml @@ -10,7 +10,7 @@ distribution-version = "2201.9.2" [[package]] org = "ballerina" name = "auth" -version = "2.11.1" +version = "2.11.2" dependencies = [ {org = "ballerina", name = "crypto"}, {org = "ballerina", name = "jballerina.java"}, @@ -107,7 +107,7 @@ version = "0.0.0" [[package]] org = "ballerina" name = "jwt" -version = "2.12.1" +version = "2.11.0" dependencies = [ {org = "ballerina", name = "cache"}, {org = "ballerina", name = "crypto"}, @@ -289,7 +289,7 @@ modules = [ [[package]] org = "ballerinax" name = "openai.finetunes" -version = "1.0.5" +version = "1.0.7" dependencies = [ {org = "ballerina", name = "http"}, {org = "ballerina", name = "mime"}, diff --git a/examples/Sarcastic-bot/Ballerina.toml b/examples/Sarcastic-bot/Ballerina.toml index 66602a0..74e9661 100644 --- a/examples/Sarcastic-bot/Ballerina.toml +++ b/examples/Sarcastic-bot/Ballerina.toml @@ -6,3 +6,9 @@ distribution = "2201.9.2" [build-options] observabilityIncluded = true + +[[dependency]] +org = "ballerinax" +name = "openai.finetunes" +version = "1.0.7" +repository = "local" diff --git a/examples/Sarcastic-bot/Dependencies.toml b/examples/Sarcastic-bot/Dependencies.toml deleted file mode 100644 index ca8c584..0000000 --- a/examples/Sarcastic-bot/Dependencies.toml +++ /dev/null @@ -1,310 +0,0 @@ -# AUTO-GENERATED FILE. DO NOT MODIFY. - -# This file is auto-generated by Ballerina for managing dependency versions. -# It should not be modified by hand. - -[ballerina] -dependencies-toml-version = "2" -distribution-version = "2201.9.2" - -[[package]] -org = "ballerina" -name = "auth" -version = "2.11.2" -dependencies = [ - {org = "ballerina", name = "crypto"}, - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "lang.array"}, - {org = "ballerina", name = "lang.string"}, - {org = "ballerina", name = "log"} -] - -[[package]] -org = "ballerina" -name = "cache" -version = "3.8.0" -dependencies = [ - {org = "ballerina", name = "constraint"}, - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "task"}, - {org = "ballerina", name = "time"} -] - -[[package]] -org = "ballerina" -name = "constraint" -version = "1.5.0" -dependencies = [ - {org = "ballerina", name = "jballerina.java"} -] - -[[package]] -org = "ballerina" -name = "crypto" -version = "2.7.2" -dependencies = [ - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "time"} -] - -[[package]] -org = "ballerina" -name = "file" -version = "1.9.0" -dependencies = [ - {org = "ballerina", name = "io"}, - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "os"}, - {org = "ballerina", name = "time"} -] - -[[package]] -org = "ballerina" -name = "http" -version = "2.11.3" -dependencies = [ - {org = "ballerina", name = "auth"}, - {org = "ballerina", name = "cache"}, - {org = "ballerina", name = "constraint"}, - {org = "ballerina", name = "crypto"}, - {org = "ballerina", name = "file"}, - {org = "ballerina", name = "io"}, - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "jwt"}, - {org = "ballerina", name = "lang.array"}, - {org = "ballerina", name = "lang.decimal"}, - {org = "ballerina", name = "lang.int"}, - {org = "ballerina", name = "lang.regexp"}, - {org = "ballerina", name = "lang.runtime"}, - {org = "ballerina", name = "lang.string"}, - {org = "ballerina", name = "lang.value"}, - {org = "ballerina", name = "log"}, - {org = "ballerina", name = "mime"}, - {org = "ballerina", name = "oauth2"}, - {org = "ballerina", name = "observe"}, - {org = "ballerina", name = "time"}, - {org = "ballerina", name = "url"} -] -modules = [ - {org = "ballerina", packageName = "http", moduleName = "http"}, - {org = "ballerina", packageName = "http", moduleName = "http.httpscerr"} -] - -[[package]] -org = "ballerina" -name = "io" -version = "1.6.1" -dependencies = [ - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "lang.value"} -] -modules = [ - {org = "ballerina", packageName = "io", moduleName = "io"} -] - -[[package]] -org = "ballerina" -name = "jballerina.java" -version = "0.0.0" - -[[package]] -org = "ballerina" -name = "jwt" -version = "2.12.1" -dependencies = [ - {org = "ballerina", name = "cache"}, - {org = "ballerina", name = "crypto"}, - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "lang.int"}, - {org = "ballerina", name = "lang.string"}, - {org = "ballerina", name = "log"}, - {org = "ballerina", name = "time"} -] - -[[package]] -org = "ballerina" -name = "lang.__internal" -version = "0.0.0" -dependencies = [ - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "lang.object"} -] - -[[package]] -org = "ballerina" -name = "lang.array" -version = "0.0.0" -dependencies = [ - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "lang.__internal"} -] - -[[package]] -org = "ballerina" -name = "lang.decimal" -version = "0.0.0" -dependencies = [ - {org = "ballerina", name = "jballerina.java"} -] - -[[package]] -org = "ballerina" -name = "lang.int" -version = "0.0.0" -dependencies = [ - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "lang.__internal"}, - {org = "ballerina", name = "lang.object"} -] - -[[package]] -org = "ballerina" -name = "lang.object" -version = "0.0.0" - -[[package]] -org = "ballerina" -name = "lang.regexp" -version = "0.0.0" -dependencies = [ - {org = "ballerina", name = "jballerina.java"} -] - -[[package]] -org = "ballerina" -name = "lang.runtime" -version = "0.0.0" -dependencies = [ - {org = "ballerina", name = "jballerina.java"} -] -modules = [ - {org = "ballerina", packageName = "lang.runtime", moduleName = "lang.runtime"} -] - -[[package]] -org = "ballerina" -name = "lang.string" -version = "0.0.0" -dependencies = [ - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "lang.regexp"} -] - -[[package]] -org = "ballerina" -name = "lang.value" -version = "0.0.0" -dependencies = [ - {org = "ballerina", name = "jballerina.java"} -] - -[[package]] -org = "ballerina" -name = "log" -version = "2.9.0" -dependencies = [ - {org = "ballerina", name = "io"}, - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "lang.value"}, - {org = "ballerina", name = "observe"} -] - -[[package]] -org = "ballerina" -name = "mime" -version = "2.9.0" -dependencies = [ - {org = "ballerina", name = "io"}, - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "lang.int"} -] -modules = [ - {org = "ballerina", packageName = "mime", moduleName = "mime"} -] - -[[package]] -org = "ballerina" -name = "oauth2" -version = "2.11.0" -dependencies = [ - {org = "ballerina", name = "cache"}, - {org = "ballerina", name = "crypto"}, - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "log"}, - {org = "ballerina", name = "time"}, - {org = "ballerina", name = "url"} -] - -[[package]] -org = "ballerina" -name = "observe" -version = "1.2.3" -dependencies = [ - {org = "ballerina", name = "jballerina.java"} -] - -[[package]] -org = "ballerina" -name = "os" -version = "1.8.0" -dependencies = [ - {org = "ballerina", name = "io"}, - {org = "ballerina", name = "jballerina.java"} -] - -[[package]] -org = "ballerina" -name = "task" -version = "2.5.0" -dependencies = [ - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "time"} -] - -[[package]] -org = "ballerina" -name = "time" -version = "2.4.0" -dependencies = [ - {org = "ballerina", name = "jballerina.java"} -] - -[[package]] -org = "ballerina" -name = "url" -version = "2.4.0" -dependencies = [ - {org = "ballerina", name = "jballerina.java"} -] -modules = [ - {org = "ballerina", packageName = "url", moduleName = "url"} -] - -[[package]] -org = "ballerinai" -name = "observe" -version = "0.0.0" -dependencies = [ - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "observe"} -] -modules = [ - {org = "ballerinai", packageName = "observe", moduleName = "observe"} -] - -[[package]] -org = "kanishka" -name = "Sarcastic_bot" -version = "0.1.0" -dependencies = [ - {org = "ballerina", name = "http"}, - {org = "ballerina", name = "io"}, - {org = "ballerina", name = "lang.runtime"}, - {org = "ballerina", name = "mime"}, - {org = "ballerina", name = "url"}, - {org = "ballerinai", name = "observe"} -] -modules = [ - {org = "kanishka", packageName = "Sarcastic_bot", moduleName = "Sarcastic_bot"} -] - diff --git a/examples/Sarcastic-bot/client.bal b/examples/Sarcastic-bot/client.bal deleted file mode 100644 index 72c21eb..0000000 --- a/examples/Sarcastic-bot/client.bal +++ /dev/null @@ -1,222 +0,0 @@ -// AUTO-GENERATED FILE. DO NOT MODIFY. -// This file is auto-generated by the Ballerina OpenAPI tool. - -// Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com). -// -// WSO2 LLC. licenses this file to you under the Apache License, -// Version 2.0 (the "License"); you may not use this file except -// in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import ballerina/http; -import ballerina/mime; - -# The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. -public isolated client class Client { - final http:Client clientEp; - # Gets invoked to initialize the `connector`. - # - # + config - The configurations to be used when initializing the `connector` - # + serviceUrl - URL of the target service - # + return - An error if connector initialization failed - public isolated function init(ConnectionConfig config, string serviceUrl = "https://api.openai.com/v1") returns error? { - http:ClientConfiguration httpClientConfig = {auth: config.auth, httpVersion: config.httpVersion, timeout: config.timeout, forwarded: config.forwarded, poolConfig: config.poolConfig, compression: config.compression, circuitBreaker: config.circuitBreaker, retryConfig: config.retryConfig, validation: config.validation}; - do { - if config.http1Settings is ClientHttp1Settings { - ClientHttp1Settings settings = check config.http1Settings.ensureType(ClientHttp1Settings); - httpClientConfig.http1Settings = {...settings}; - } - if config.http2Settings is http:ClientHttp2Settings { - httpClientConfig.http2Settings = check config.http2Settings.ensureType(http:ClientHttp2Settings); - } - if config.cache is http:CacheConfig { - httpClientConfig.cache = check config.cache.ensureType(http:CacheConfig); - } - if config.responseLimits is http:ResponseLimitConfigs { - httpClientConfig.responseLimits = check config.responseLimits.ensureType(http:ResponseLimitConfigs); - } - if config.secureSocket is http:ClientSecureSocket { - httpClientConfig.secureSocket = check config.secureSocket.ensureType(http:ClientSecureSocket); - } - if config.proxy is http:ProxyConfig { - httpClientConfig.proxy = check config.proxy.ensureType(http:ProxyConfig); - } - } - http:Client httpEp = check new (serviceUrl, httpClientConfig); - self.clientEp = httpEp; - return; - } - - # Delete a file. - # - # + file_id - The ID of the file to use for this request. - # + headers - Headers to be sent with the request - # + return - OK - resource isolated function delete files/[string file_id](map headers = {}) returns DeleteFileResponse|error { - string resourcePath = string `/files/${getEncodedUri(file_id)}`; - return self.clientEp->delete(resourcePath, headers = headers); - } - - # Delete a fine-tuned model. You must have the Owner role in your organization to delete a model. - # - # + model - The model to delete - # + headers - Headers to be sent with the request - # + return - OK - resource isolated function delete models/[string model](map headers = {}) returns DeleteModelResponse|error { - string resourcePath = string `/models/${getEncodedUri(model)}`; - return self.clientEp->delete(resourcePath, headers = headers); - } - - # Returns a list of files that belong to the user's organization. - # - # + headers - Headers to be sent with the request - # + queries - Queries to be sent with the request - # + return - OK - resource isolated function get files(map headers = {}, *ListFilesQueries queries) returns ListFilesResponse|error { - string resourcePath = string `/files`; - resourcePath = resourcePath + check getPathForQueryParam(queries); - return self.clientEp->get(resourcePath, headers); - } - - # Returns information about a specific file. - # - # + file_id - The ID of the file to use for this request. - # + headers - Headers to be sent with the request - # + return - OK - resource isolated function get files/[string file_id](map headers = {}) returns OpenAIFile|error { - string resourcePath = string `/files/${getEncodedUri(file_id)}`; - return self.clientEp->get(resourcePath, headers); - } - - # Returns the contents of the specified file. - # - # + file_id - The ID of the file to use for this request. - # + headers - Headers to be sent with the request - # + return - OK - resource isolated function get files/[string file_id]/content(map headers = {}) returns byte[]|error { - string resourcePath = string `/files/${getEncodedUri(file_id)}/content`; - return self.clientEp->get(resourcePath, headers); - } - - # List your organization's fine-tuning jobs - # - # + headers - Headers to be sent with the request - # + queries - Queries to be sent with the request - # + return - OK - resource isolated function get fine_tuning/jobs(map headers = {}, *ListPaginatedFineTuningJobsQueries queries) returns ListPaginatedFineTuningJobsResponse|error { - string resourcePath = string `/fine_tuning/jobs`; - resourcePath = resourcePath + check getPathForQueryParam(queries); - return self.clientEp->get(resourcePath, headers); - } - - # Get info about a fine-tuning job. - # - # [Learn more about fine-tuning](/docs/guides/fine-tuning) - # - # + fine_tuning_job_id - The ID of the fine-tuning job. - # + headers - Headers to be sent with the request - # + return - OK - resource isolated function get fine_tuning/jobs/[string fine_tuning_job_id](map headers = {}) returns FineTuningJob|error { - string resourcePath = string `/fine_tuning/jobs/${getEncodedUri(fine_tuning_job_id)}`; - return self.clientEp->get(resourcePath, headers); - } - - # List checkpoints for a fine-tuning job. - # - # + fine_tuning_job_id - The ID of the fine-tuning job to get checkpoints for. - # + headers - Headers to be sent with the request - # + queries - Queries to be sent with the request - # + return - OK - resource isolated function get fine_tuning/jobs/[string fine_tuning_job_id]/checkpoints(map headers = {}, *ListFineTuningJobCheckpointsQueries queries) returns ListFineTuningJobCheckpointsResponse|error { - string resourcePath = string `/fine_tuning/jobs/${getEncodedUri(fine_tuning_job_id)}/checkpoints`; - resourcePath = resourcePath + check getPathForQueryParam(queries); - return self.clientEp->get(resourcePath, headers); - } - - # Get status updates for a fine-tuning job. - # - # + fine_tuning_job_id - The ID of the fine-tuning job to get events for. - # + headers - Headers to be sent with the request - # + queries - Queries to be sent with the request - # + return - OK - resource isolated function get fine_tuning/jobs/[string fine_tuning_job_id]/events(map headers = {}, *ListFineTuningEventsQueries queries) returns ListFineTuningJobEventsResponse|error { - string resourcePath = string `/fine_tuning/jobs/${getEncodedUri(fine_tuning_job_id)}/events`; - resourcePath = resourcePath + check getPathForQueryParam(queries); - return self.clientEp->get(resourcePath, headers); - } - - # Lists the currently available models, and provides basic information about each one such as the owner and availability. - # - # + headers - Headers to be sent with the request - # + return - OK - resource isolated function get models(map headers = {}) returns ListModelsResponse|error { - string resourcePath = string `/models`; - return self.clientEp->get(resourcePath, headers); - } - - # Retrieves a model instance, providing basic information about the model such as the owner and permissioning. - # - # + model - The ID of the model to use for this request - # + headers - Headers to be sent with the request - # + return - OK - resource isolated function get models/[string model](map headers = {}) returns Model|error { - string resourcePath = string `/models/${getEncodedUri(model)}`; - return self.clientEp->get(resourcePath, headers); - } - - # Upload a file that can be used across various endpoints. Individual files can be up to 512 MB, and the size of all files uploaded by one organization can be up to 100 GB. - # - # The Assistants API supports files up to 2 million tokens and of specific file types. See the [Assistants Tools guide](/docs/assistants/tools) for details. - # - # The Fine-tuning API only supports `.jsonl` files. The input also has certain required formats for fine-tuning [chat](/docs/api-reference/fine-tuning/chat-input) or [completions](/docs/api-reference/fine-tuning/completions-input) models. - # - # The Batch API only supports `.jsonl` files up to 100 MB in size. The input also has a specific required [format](/docs/api-reference/batch/request-input). - # - # Please [contact us](https://help.openai.com/) if you need to increase these storage limits. - # - # + headers - Headers to be sent with the request - # + return - OK - resource isolated function post files(CreateFileRequest payload, map headers = {}) returns OpenAIFile|error { - string resourcePath = string `/files`; - http:Request request = new; - mime:Entity[] bodyParts = check createBodyParts(payload); - request.setBodyParts(bodyParts); - return self.clientEp->post(resourcePath, request, headers); - } - - # Creates a fine-tuning job which begins the process of creating a new model from a given dataset. - # - # Response includes details of the enqueued job including job status and the name of the fine-tuned models once complete. - # - # [Learn more about fine-tuning](/docs/guides/fine-tuning) - # - # + headers - Headers to be sent with the request - # + return - OK - resource isolated function post fine_tuning/jobs(CreateFineTuningJobRequest payload, map headers = {}) returns FineTuningJob|error { - string resourcePath = string `/fine_tuning/jobs`; - http:Request request = new; - json jsonBody = payload.toJson(); - request.setPayload(jsonBody, "application/json"); - return self.clientEp->post(resourcePath, request, headers); - } - - # Immediately cancel a fine-tune job. - # - # + fine_tuning_job_id - The ID of the fine-tuning job to cancel. - # + headers - Headers to be sent with the request - # + return - OK - resource isolated function post fine_tuning/jobs/[string fine_tuning_job_id]/cancel(map headers = {}) returns FineTuningJob|error { - string resourcePath = string `/fine_tuning/jobs/${getEncodedUri(fine_tuning_job_id)}/cancel`; - http:Request request = new; - return self.clientEp->post(resourcePath, request, headers); - } -} diff --git a/examples/Sarcastic-bot/main.bal b/examples/Sarcastic-bot/main.bal index 97ddd72..eef70df 100644 --- a/examples/Sarcastic-bot/main.bal +++ b/examples/Sarcastic-bot/main.bal @@ -1,7 +1,6 @@ import ballerina/io; import ballerina/lang.runtime; - -// import ballerinax/openai.finetunes; +import ballerinax/openai.finetunes; configurable string token = ?; string serviceUrl = "https://api.openai.com/v1"; @@ -10,26 +9,26 @@ string validationFileName = "validation.jsonl"; string trainingFilePath = "./data/" + trainingFileName; string validationFilePath = "./data/" + validationFileName; -final ConnectionConfig config = {auth: {token}}; -final Client openAIFinetunes = check new Client(config, serviceUrl); +final finetunes:ConnectionConfig config = {auth: {token}}; +final finetunes:Client openAIFinetunes = check new Client(config, serviceUrl); public function main() returns error? { byte[] trainingFileContent = check io:fileReadBytes(trainingFilePath); byte[] validationFileContent = check io:fileReadBytes(validationFilePath); - CreateFileRequest trainingFileRequest = { + finetunes:CreateFileRequest trainingFileRequest = { file: {fileContent: trainingFileContent, fileName: trainingFileName}, purpose: "fine-tune" }; - CreateFileRequest validationFileRequest = { + finetunes:CreateFileRequest validationFileRequest = { file: {fileContent: validationFileContent, fileName: validationFileName}, purpose: "fine-tune" }; - OpenAIFile trainingFileResponse = + finetunes:OpenAIFile trainingFileResponse = check openAIFinetunes->/files.post(trainingFileRequest); - OpenAIFile validationFileResponse = + finetunes:OpenAIFile validationFileResponse = check openAIFinetunes->/files.post(validationFileRequest); string trainingFileId = trainingFileResponse.id; @@ -37,7 +36,7 @@ public function main() returns error? { io:println("Training file id: " + trainingFileId); io:println("Validation file id: " + validationFileId); - CreateFineTuningJobRequest fineTuneRequest = { + finetunes:CreateFineTuningJobRequest fineTuneRequest = { model: "gpt-3.5-turbo", training_file: trainingFileId, validation_file: validationFileId, @@ -48,12 +47,12 @@ public function main() returns error? { } }; - FineTuningJob fineTuneResponse = + finetunes:FineTuningJob fineTuneResponse = check openAIFinetunes->/fine_tuning/jobs.post(fineTuneRequest); string fineTuneJobId = fineTuneResponse.id; io:println("Fine-tuning job id: " + fineTuneJobId); - FineTuningJob fineTuneJob = + finetunes:FineTuningJob fineTuneJob = check openAIFinetunes->/fine_tuning/jobs/[fineTuneJobId].get(); io:print("Validating files..."); @@ -72,10 +71,10 @@ public function main() returns error? { } io:println("\nTraining..."); + finetunes:ListFineTuningJobEventsResponse eventsResponse; while (fineTuneJob.status == "running") { fineTuneJob = check openAIFinetunes->/fine_tuning/jobs/[fineTuneJobId].get(); - ListFineTuningJobEventsResponse eventsResponse = - check openAIFinetunes->/fine_tuning/jobs/[fineTuneJobId]/events.get(); + eventsResponse = check openAIFinetunes->/fine_tuning/jobs/[fineTuneJobId]/events.get(); io:println(eventsResponse.data[0].message); runtime:sleep(1); } diff --git a/examples/Sarcastic-bot/types.bal b/examples/Sarcastic-bot/types.bal deleted file mode 100644 index 093632d..0000000 --- a/examples/Sarcastic-bot/types.bal +++ /dev/null @@ -1,361 +0,0 @@ -// AUTO-GENERATED FILE. DO NOT MODIFY. -// This file is auto-generated by the Ballerina OpenAPI tool. - -// Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com). -// -// WSO2 LLC. licenses this file to you under the Apache License, -// Version 2.0 (the "License"); you may not use this file except -// in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import ballerina/http; - -# The `File` object represents a document that has been uploaded to OpenAI. -public type OpenAIFile record { - # The file identifier, which can be referenced in the API endpoints. - string id; - # The size of the file, in bytes. - int bytes; - # The Unix timestamp (in seconds) for when the file was created. - int created_at; - # The name of the file. - string filename; - # The object type, which is always `file`. - "file" 'object; - # The intended purpose of the file. Supported values are `assistants`, `assistants_output`, `batch`, `batch_output`, `fine-tune`, `fine-tune-results` and `vision`. - "assistants"|"assistants_output"|"batch"|"batch_output"|"fine-tune"|"fine-tune-results"|"vision" purpose; - # Deprecated. The current status of the file, which can be either `uploaded`, `processed`, or `error`. - # - # # Deprecated - @deprecated - "uploaded"|"processed"|"error" status; - # Deprecated. For details on why a fine-tuning training file failed validation, see the `error` field on `fine_tuning.job`. - string? status_details?; -}; - -public type CreateFineTuningJobRequest record { - # The name of the model to fine-tune. You can select one of the - # [supported models](/docs/guides/fine-tuning/what-models-can-be-fine-tuned). - string|"babbage-002"|"davinci-002"|"gpt-3.5-turbo" model; - # The ID of an uploaded file that contains training data. - # - # See [upload file](/docs/api-reference/files/create) for how to upload a file. - # - # Your dataset must be formatted as a JSONL file. Additionally, you must upload your file with the purpose `fine-tune`. - # - # The contents of the file should differ depending on if the model uses the [chat](/docs/api-reference/fine-tuning/chat-input) or [completions](/docs/api-reference/fine-tuning/completions-input) format. - # - # See the [fine-tuning guide](/docs/guides/fine-tuning) for more details. - string training_file; - CreateFineTuningJobRequest_hyperparameters hyperparameters?; - # A string of up to 18 characters that will be added to your fine-tuned model name. - # - # For example, a `suffix` of "custom-model-name" would produce a model name like `ft:gpt-3.5-turbo:openai:custom-model-name:7p4lURel`. - string? suffix?; - # The ID of an uploaded file that contains validation data. - # - # If you provide this file, the data is used to generate validation - # metrics periodically during fine-tuning. These metrics can be viewed in - # the fine-tuning results file. - # The same data should not be present in both train and validation files. - # - # Your dataset must be formatted as a JSONL file. You must upload your file with the purpose `fine-tune`. - # - # See the [fine-tuning guide](/docs/guides/fine-tuning) for more details. - string? validation_file?; - # A list of integrations to enable for your fine-tuning job. - CreateFineTuningJobRequest_integrations[]? integrations?; - # The seed controls the reproducibility of the job. Passing in the same seed and job parameters should produce the same results, but may differ in rare cases. - # If a seed is not specified, one will be generated for you. - int? seed?; -}; - -# The `fine_tuning.job.checkpoint` object represents a model checkpoint for a fine-tuning job that is ready to use. -public type FineTuningJobCheckpoint record { - # The checkpoint identifier, which can be referenced in the API endpoints. - string id; - # The Unix timestamp (in seconds) for when the checkpoint was created. - int created_at; - # The name of the fine-tuned checkpoint model that is created. - string fine_tuned_model_checkpoint; - # The step number that the checkpoint was created at. - int step_number; - FineTuningJobCheckpoint_metrics metrics; - # The name of the fine-tuning job that this checkpoint was created from. - string fine_tuning_job_id; - # The object type, which is always "fine_tuning.job.checkpoint". - "fine_tuning.job.checkpoint" 'object; -}; - -public type ListPaginatedFineTuningJobsResponse record { - FineTuningJob[] data; - boolean has_more; - "list" 'object; -}; - -# The hyperparameters used for the fine-tuning job. -public type CreateFineTuningJobRequest_hyperparameters record { - # Number of examples in each batch. A larger batch size means that model parameters - # are updated less frequently, but with lower variance. - "auto"|int batch_size = "auto"; - # Scaling factor for the learning rate. A smaller learning rate may be useful to avoid - # overfitting. - "auto"|decimal learning_rate_multiplier = "auto"; - # The number of epochs to train the model for. An epoch refers to one full cycle - # through the training dataset. - "auto"|int n_epochs = "auto"; -}; - -# Represents the Queries record for the operation: listFineTuningEvents -public type ListFineTuningEventsQueries record { - # Number of events to retrieve. - int 'limit = 20; - # Identifier for the last event from the previous pagination request. - string after?; -}; - -# Represents the Queries record for the operation: listFineTuningJobCheckpoints -public type ListFineTuningJobCheckpointsQueries record { - # Number of checkpoints to retrieve. - int 'limit = 10; - # Identifier for the last checkpoint ID from the previous pagination request. - string after?; -}; - -# The settings for your integration with Weights and Biases. This payload specifies the project that -# metrics will be sent to. Optionally, you can set an explicit display name for your run, add tags -# to your run, and set a default entity (team, username, etc) to be associated with your run. -public type CreateFineTuningJobRequest_wandb record { - # The name of the project that the new run will be created under. - string project; - # A display name to set for the run. If not set, we will use the Job ID as the name. - string? name?; - # The entity to use for the run. This allows you to set the team or username of the WandB user that you would - # like associated with the run. If not set, the default entity for the registered WandB API key is used. - string? entity?; - # A list of tags to be attached to the newly created run. These tags are passed through directly to WandB. Some - # default tags are generated by OpenAI: "openai/finetune", "openai/{base-model}", "openai/{ftjob-abcdef}". - string[] tags?; -}; - -# The hyperparameters used for the fine-tuning job. See the [fine-tuning guide](/docs/guides/fine-tuning) for more details. -public type FineTuningJob_hyperparameters record { - # The number of epochs to train the model for. An epoch refers to one full cycle through the training dataset. - # "auto" decides the optimal number of epochs based on the size of the dataset. If setting the number manually, we support any number between 1 and 50 epochs. - "auto"|int n_epochs; -}; - -# Represents the Queries record for the operation: listFiles -public type ListFilesQueries record { - # Only return files with the given purpose. - string purpose?; -}; - -public type ListFineTuningJobCheckpointsResponse record { - FineTuningJobCheckpoint[] data; - "list" 'object; - string? first_id?; - string? last_id?; - boolean has_more; -}; - -public type DeleteModelResponse record { - string id; - boolean deleted; - string 'object; -}; - -public type FineTuningIntegration record { - # The type of the integration being enabled for the fine-tuning job - "wandb" 'type; - CreateFineTuningJobRequest_wandb wandb; -}; - -# Provides a set of configurations for controlling the behaviours when communicating with a remote HTTP endpoint. -@display {label: "Connection Config"} -public type ConnectionConfig record {| - # Configurations related to client authentication - http:BearerTokenConfig auth; - # The HTTP version understood by the client - http:HttpVersion httpVersion = http:HTTP_2_0; - # Configurations related to HTTP/1.x protocol - ClientHttp1Settings http1Settings?; - # Configurations related to HTTP/2 protocol - http:ClientHttp2Settings http2Settings?; - # The maximum time to wait (in seconds) for a response before closing the connection - decimal timeout = 60; - # The choice of setting `forwarded`/`x-forwarded` header - string forwarded = "disable"; - # Configurations associated with request pooling - http:PoolConfiguration poolConfig?; - # HTTP caching related configurations - http:CacheConfig cache?; - # Specifies the way of handling compression (`accept-encoding`) header - http:Compression compression = http:COMPRESSION_AUTO; - # Configurations associated with the behaviour of the Circuit Breaker - http:CircuitBreakerConfig circuitBreaker?; - # Configurations associated with retrying - http:RetryConfig retryConfig?; - # Configurations associated with inbound response size limits - http:ResponseLimitConfigs responseLimits?; - # SSL/TLS-related options - http:ClientSecureSocket secureSocket?; - # Proxy server related options - http:ProxyConfig proxy?; - # Enables the inbound payload validation functionality which provided by the constraint package. Enabled by default - boolean validation = true; -|}; - -# Metrics at the step number during the fine-tuning job. -public type FineTuningJobCheckpoint_metrics record { - decimal step?; - decimal train_loss?; - decimal train_mean_token_accuracy?; - decimal valid_loss?; - decimal valid_mean_token_accuracy?; - decimal full_valid_loss?; - decimal full_valid_mean_token_accuracy?; -}; - -public type CreateFileRequest record {| - # The File object (not file name) to be uploaded. - record {byte[] fileContent; string fileName;} file; - # The intended purpose of the uploaded file. - # - # Use "assistants" for [Assistants](/docs/api-reference/assistants) and [Message](/docs/api-reference/messages) files, "vision" for Assistants image file inputs, "batch" for [Batch API](/docs/guides/batch), and "fine-tune" for [Fine-tuning](/docs/api-reference/fine-tuning). - "assistants"|"batch"|"fine-tune"|"vision" purpose; -|}; - -# The `fine_tuning.job` object represents a fine-tuning job that has been created through the API. -public type FineTuningJob record { - # The object identifier, which can be referenced in the API endpoints. - string id; - # The Unix timestamp (in seconds) for when the fine-tuning job was created. - int created_at; - FineTuningJob_error? 'error; - # The name of the fine-tuned model that is being created. The value will be null if the fine-tuning job is still running. - string? fine_tuned_model; - # The Unix timestamp (in seconds) for when the fine-tuning job was finished. The value will be null if the fine-tuning job is still running. - int? finished_at; - FineTuningJob_hyperparameters hyperparameters; - # The base model that is being fine-tuned. - string model; - # The object type, which is always "fine_tuning.job". - "fine_tuning.job" 'object; - # The organization that owns the fine-tuning job. - string organization_id; - # The compiled results file ID(s) for the fine-tuning job. You can retrieve the results with the [Files API](/docs/api-reference/files/retrieve-contents). - string[] result_files; - # The current status of the fine-tuning job, which can be either `validating_files`, `queued`, `running`, `succeeded`, `failed`, or `cancelled`. - "validating_files"|"queued"|"running"|"succeeded"|"failed"|"cancelled" status; - # The total number of billable tokens processed by this fine-tuning job. The value will be null if the fine-tuning job is still running. - int? trained_tokens; - # The file ID used for training. You can retrieve the training data with the [Files API](/docs/api-reference/files/retrieve-contents). - string training_file; - # The file ID used for validation. You can retrieve the validation results with the [Files API](/docs/api-reference/files/retrieve-contents). - string? validation_file; - # A list of integrations to enable for this fine-tuning job. - (FineTuningIntegration)[]? integrations?; - # The seed used for the fine-tuning job. - int seed; - # The Unix timestamp (in seconds) for when the fine-tuning job is estimated to finish. The value will be null if the fine-tuning job is not running. - int? estimated_finish?; -}; - -# Proxy server configurations to be used with the HTTP client endpoint. -public type ProxyConfig record {| - # Host name of the proxy server - string host = ""; - # Proxy server port - int port = 0; - # Proxy server username - string userName = ""; - # Proxy server password - @display {label: "", kind: "password"} - string password = ""; -|}; - -# For fine-tuning jobs that have `failed`, this will contain more information on the cause of the failure. -public type FineTuningJob_error record { - # A machine-readable error code. - string code?; - # A human-readable error message. - string message?; - # The parameter that was invalid, usually `training_file` or `validation_file`. This field will be null if the failure was not parameter-specific. - string? param?; -}; - -public type CreateFineTuningJobRequest_integrations record { - # The type of integration to enable. Currently, only "wandb" (Weights and Biases) is supported. - "wandb" 'type; - CreateFineTuningJobRequest_wandb wandb; -}; - -public type ListModelsResponse record { - "list" 'object; - Model[] data; -}; - -# Fine-tuning job event object -public type FineTuningJobEvent record { - string id; - int created_at; - "info"|"warn"|"error" level; - string message; - "fine_tuning.job.event" 'object; -}; - -public type DeleteFileResponse record { - string id; - "file" 'object; - boolean deleted; -}; - -# Provides settings related to HTTP/1.x protocol. -public type ClientHttp1Settings record {| - # Specifies whether to reuse a connection for multiple requests - http:KeepAlive keepAlive = http:KEEPALIVE_AUTO; - # The chunking behaviour of the request - http:Chunking chunking = http:CHUNKING_AUTO; - # Proxy server related options - ProxyConfig proxy?; -|}; - -# Describes an OpenAI model offering that can be used with the API. -public type Model record { - # The model identifier, which can be referenced in the API endpoints. - string id; - # The Unix timestamp (in seconds) when the model was created. - int created; - # The object type, which is always "model". - "model" 'object; - # The organization that owns the model. - string owned_by; -}; - -public type ListFineTuningJobEventsResponse record { - FineTuningJobEvent[] data; - "list" 'object; -}; - -public type ListFilesResponse record { - OpenAIFile[] data; - "list" 'object; -}; - -# Represents the Queries record for the operation: listPaginatedFineTuningJobs -public type ListPaginatedFineTuningJobsQueries record { - # Number of fine-tuning jobs to retrieve. - int 'limit = 20; - # Identifier for the last job from the previous pagination request. - string after?; -}; diff --git a/examples/Sarcastic-bot/utils.bal b/examples/Sarcastic-bot/utils.bal deleted file mode 100644 index 0ce7ca0..0000000 --- a/examples/Sarcastic-bot/utils.bal +++ /dev/null @@ -1,252 +0,0 @@ -// AUTO-GENERATED FILE. DO NOT MODIFY. -// This file is auto-generated by the Ballerina OpenAPI tool. - -// Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com). -// -// WSO2 LLC. licenses this file to you under the Apache License, -// Version 2.0 (the "License"); you may not use this file except -// in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import ballerina/mime; -import ballerina/url; - -type SimpleBasicType string|boolean|int|float|decimal; - -# Represents encoding mechanism details. -type Encoding record { - # Defines how multiple values are delimited - string style = FORM; - # Specifies whether arrays and objects should generate as separate fields - boolean explode = true; - # Specifies the custom content type - string contentType?; - # Specifies the custom headers - map headers?; -}; - -enum EncodingStyle { - DEEPOBJECT, FORM, SPACEDELIMITED, PIPEDELIMITED -} - -final Encoding & readonly defaultEncoding = {}; - -# Serialize the record according to the deepObject style. -# -# + parent - Parent record name -# + anyRecord - Record to be serialized -# + return - Serialized record as a string -isolated function getDeepObjectStyleRequest(string parent, record {} anyRecord) returns string { - string[] recordArray = []; - foreach [string, anydata] [key, value] in anyRecord.entries() { - if value is SimpleBasicType { - recordArray.push(parent + "[" + key + "]" + "=" + getEncodedUri(value.toString())); - } else if value is SimpleBasicType[] { - recordArray.push(getSerializedArray(parent + "[" + key + "]" + "[]", value, DEEPOBJECT, true)); - } else if value is record {} { - string nextParent = parent + "[" + key + "]"; - recordArray.push(getDeepObjectStyleRequest(nextParent, value)); - } else if value is record {}[] { - string nextParent = parent + "[" + key + "]"; - recordArray.push(getSerializedRecordArray(nextParent, value, DEEPOBJECT)); - } - recordArray.push("&"); - } - _ = recordArray.pop(); - return string:'join("", ...recordArray); -} - -# Serialize the record according to the form style. -# -# + parent - Parent record name -# + anyRecord - Record to be serialized -# + explode - Specifies whether arrays and objects should generate separate parameters -# + return - Serialized record as a string -isolated function getFormStyleRequest(string parent, record {} anyRecord, boolean explode = true) returns string { - string[] recordArray = []; - if explode { - foreach [string, anydata] [key, value] in anyRecord.entries() { - if value is SimpleBasicType { - recordArray.push(key, "=", getEncodedUri(value.toString())); - } else if value is SimpleBasicType[] { - recordArray.push(getSerializedArray(key, value, explode = explode)); - } else if value is record {} { - recordArray.push(getFormStyleRequest(parent, value, explode)); - } - recordArray.push("&"); - } - _ = recordArray.pop(); - } else { - foreach [string, anydata] [key, value] in anyRecord.entries() { - if value is SimpleBasicType { - recordArray.push(key, ",", getEncodedUri(value.toString())); - } else if value is SimpleBasicType[] { - recordArray.push(getSerializedArray(key, value, explode = false)); - } else if value is record {} { - recordArray.push(getFormStyleRequest(parent, value, explode)); - } - recordArray.push(","); - } - _ = recordArray.pop(); - } - return string:'join("", ...recordArray); -} - -# Serialize arrays. -# -# + arrayName - Name of the field with arrays -# + anyArray - Array to be serialized -# + style - Defines how multiple values are delimited -# + explode - Specifies whether arrays and objects should generate separate parameters -# + return - Serialized array as a string -isolated function getSerializedArray(string arrayName, anydata[] anyArray, string style = "form", boolean explode = true) returns string { - string key = arrayName; - string[] arrayValues = []; - if anyArray.length() > 0 { - if style == FORM && !explode { - arrayValues.push(key, "="); - foreach anydata i in anyArray { - arrayValues.push(getEncodedUri(i.toString()), ","); - } - } else if style == SPACEDELIMITED && !explode { - arrayValues.push(key, "="); - foreach anydata i in anyArray { - arrayValues.push(getEncodedUri(i.toString()), "%20"); - } - } else if style == PIPEDELIMITED && !explode { - arrayValues.push(key, "="); - foreach anydata i in anyArray { - arrayValues.push(getEncodedUri(i.toString()), "|"); - } - } else if style == DEEPOBJECT { - foreach anydata i in anyArray { - arrayValues.push(key, "[]", "=", getEncodedUri(i.toString()), "&"); - } - } else { - foreach anydata i in anyArray { - arrayValues.push(key, "=", getEncodedUri(i.toString()), "&"); - } - } - _ = arrayValues.pop(); - } - return string:'join("", ...arrayValues); -} - -# Serialize the array of records according to the form style. -# -# + parent - Parent record name -# + value - Array of records to be serialized -# + style - Defines how multiple values are delimited -# + explode - Specifies whether arrays and objects should generate separate parameters -# + return - Serialized record as a string -isolated function getSerializedRecordArray(string parent, record {}[] value, string style = FORM, boolean explode = true) returns string { - string[] serializedArray = []; - if style == DEEPOBJECT { - int arayIndex = 0; - foreach var recordItem in value { - serializedArray.push(getDeepObjectStyleRequest(parent + "[" + arayIndex.toString() + "]", recordItem), "&"); - arayIndex = arayIndex + 1; - } - } else { - if !explode { - serializedArray.push(parent, "="); - } - foreach var recordItem in value { - serializedArray.push(getFormStyleRequest(parent, recordItem, explode), ","); - } - } - _ = serializedArray.pop(); - return string:'join("", ...serializedArray); -} - -# Get Encoded URI for a given value. -# -# + value - Value to be encoded -# + return - Encoded string -isolated function getEncodedUri(anydata value) returns string { - string|error encoded = url:encode(value.toString(), "UTF8"); - if encoded is string { - return encoded; - } else { - return value.toString(); - } -} - -# Generate query path with query parameter. -# -# + queryParam - Query parameter map -# + encodingMap - Details on serialization mechanism -# + return - Returns generated Path or error at failure of client initialization -isolated function getPathForQueryParam(map queryParam, map encodingMap = {}) returns string|error { - string[] param = []; - if queryParam.length() > 0 { - param.push("?"); - foreach var [key, value] in queryParam.entries() { - if value is () { - _ = queryParam.remove(key); - continue; - } - Encoding encodingData = encodingMap.hasKey(key) ? encodingMap.get(key) : defaultEncoding; - if value is SimpleBasicType { - param.push(key, "=", getEncodedUri(value.toString())); - } else if value is SimpleBasicType[] { - param.push(getSerializedArray(key, value, encodingData.style, encodingData.explode)); - } else if value is record {} { - if encodingData.style == DEEPOBJECT { - param.push(getDeepObjectStyleRequest(key, value)); - } else { - param.push(getFormStyleRequest(key, value, encodingData.explode)); - } - } else { - param.push(key, "=", value.toString()); - } - param.push("&"); - } - _ = param.pop(); - } - string restOfPath = string:'join("", ...param); - return restOfPath; -} - -isolated function createBodyParts(record {|anydata...;|} anyRecord, map encodingMap = {}) returns mime:Entity[]|error { - mime:Entity[] entities = []; - foreach [string, anydata] [key, value] in anyRecord.entries() { - Encoding encodingData = encodingMap.hasKey(key) ? encodingMap.get(key) : {}; - mime:Entity entity = new mime:Entity(); - if value is record {byte[] fileContent; string fileName;} { - entity.setContentDisposition(mime:getContentDispositionObject(string `form-data; name=${key}; filename=${value.fileName}`)); - entity.setByteArray(value.fileContent); - } else if value is byte[] { - entity.setContentDisposition(mime:getContentDispositionObject(string `form-data; name=${key};`)); - entity.setByteArray(value); - } else if value is SimpleBasicType|SimpleBasicType[] { - entity.setContentDisposition(mime:getContentDispositionObject(string `form-data; name=${key};`)); - entity.setText(value.toString()); - } else if value is record {}|record {}[] { - entity.setContentDisposition(mime:getContentDispositionObject(string `form-data; name=${key};`)); - entity.setJson(value.toJson()); - } - if encodingData?.contentType is string { - check entity.setContentType(encodingData?.contentType.toString()); - } - map? headers = encodingData?.headers; - if headers is map { - foreach var [headerName, headerValue] in headers.entries() { - if headerValue is SimpleBasicType { - entity.setHeader(headerName, headerValue.toString()); - } - } - } - entities.push(entity); - } - return entities; -} diff --git a/examples/Sports-headline-analyzer/Ballerina.toml b/examples/Sports-headline-analyzer/Ballerina.toml index 2700326..2c74cb0 100644 --- a/examples/Sports-headline-analyzer/Ballerina.toml +++ b/examples/Sports-headline-analyzer/Ballerina.toml @@ -6,3 +6,9 @@ distribution = "2201.9.2" [build-options] observabilityIncluded = true + +[[dependency]] +org = "ballerinax" +name = "openai.finetunes" +version = "1.0.7" +repository = "local" diff --git a/examples/Sports-headline-analyzer/client.bal b/examples/Sports-headline-analyzer/client.bal deleted file mode 100644 index 72c21eb..0000000 --- a/examples/Sports-headline-analyzer/client.bal +++ /dev/null @@ -1,222 +0,0 @@ -// AUTO-GENERATED FILE. DO NOT MODIFY. -// This file is auto-generated by the Ballerina OpenAPI tool. - -// Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com). -// -// WSO2 LLC. licenses this file to you under the Apache License, -// Version 2.0 (the "License"); you may not use this file except -// in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import ballerina/http; -import ballerina/mime; - -# The OpenAI REST API. Please see https://platform.openai.com/docs/api-reference for more details. -public isolated client class Client { - final http:Client clientEp; - # Gets invoked to initialize the `connector`. - # - # + config - The configurations to be used when initializing the `connector` - # + serviceUrl - URL of the target service - # + return - An error if connector initialization failed - public isolated function init(ConnectionConfig config, string serviceUrl = "https://api.openai.com/v1") returns error? { - http:ClientConfiguration httpClientConfig = {auth: config.auth, httpVersion: config.httpVersion, timeout: config.timeout, forwarded: config.forwarded, poolConfig: config.poolConfig, compression: config.compression, circuitBreaker: config.circuitBreaker, retryConfig: config.retryConfig, validation: config.validation}; - do { - if config.http1Settings is ClientHttp1Settings { - ClientHttp1Settings settings = check config.http1Settings.ensureType(ClientHttp1Settings); - httpClientConfig.http1Settings = {...settings}; - } - if config.http2Settings is http:ClientHttp2Settings { - httpClientConfig.http2Settings = check config.http2Settings.ensureType(http:ClientHttp2Settings); - } - if config.cache is http:CacheConfig { - httpClientConfig.cache = check config.cache.ensureType(http:CacheConfig); - } - if config.responseLimits is http:ResponseLimitConfigs { - httpClientConfig.responseLimits = check config.responseLimits.ensureType(http:ResponseLimitConfigs); - } - if config.secureSocket is http:ClientSecureSocket { - httpClientConfig.secureSocket = check config.secureSocket.ensureType(http:ClientSecureSocket); - } - if config.proxy is http:ProxyConfig { - httpClientConfig.proxy = check config.proxy.ensureType(http:ProxyConfig); - } - } - http:Client httpEp = check new (serviceUrl, httpClientConfig); - self.clientEp = httpEp; - return; - } - - # Delete a file. - # - # + file_id - The ID of the file to use for this request. - # + headers - Headers to be sent with the request - # + return - OK - resource isolated function delete files/[string file_id](map headers = {}) returns DeleteFileResponse|error { - string resourcePath = string `/files/${getEncodedUri(file_id)}`; - return self.clientEp->delete(resourcePath, headers = headers); - } - - # Delete a fine-tuned model. You must have the Owner role in your organization to delete a model. - # - # + model - The model to delete - # + headers - Headers to be sent with the request - # + return - OK - resource isolated function delete models/[string model](map headers = {}) returns DeleteModelResponse|error { - string resourcePath = string `/models/${getEncodedUri(model)}`; - return self.clientEp->delete(resourcePath, headers = headers); - } - - # Returns a list of files that belong to the user's organization. - # - # + headers - Headers to be sent with the request - # + queries - Queries to be sent with the request - # + return - OK - resource isolated function get files(map headers = {}, *ListFilesQueries queries) returns ListFilesResponse|error { - string resourcePath = string `/files`; - resourcePath = resourcePath + check getPathForQueryParam(queries); - return self.clientEp->get(resourcePath, headers); - } - - # Returns information about a specific file. - # - # + file_id - The ID of the file to use for this request. - # + headers - Headers to be sent with the request - # + return - OK - resource isolated function get files/[string file_id](map headers = {}) returns OpenAIFile|error { - string resourcePath = string `/files/${getEncodedUri(file_id)}`; - return self.clientEp->get(resourcePath, headers); - } - - # Returns the contents of the specified file. - # - # + file_id - The ID of the file to use for this request. - # + headers - Headers to be sent with the request - # + return - OK - resource isolated function get files/[string file_id]/content(map headers = {}) returns byte[]|error { - string resourcePath = string `/files/${getEncodedUri(file_id)}/content`; - return self.clientEp->get(resourcePath, headers); - } - - # List your organization's fine-tuning jobs - # - # + headers - Headers to be sent with the request - # + queries - Queries to be sent with the request - # + return - OK - resource isolated function get fine_tuning/jobs(map headers = {}, *ListPaginatedFineTuningJobsQueries queries) returns ListPaginatedFineTuningJobsResponse|error { - string resourcePath = string `/fine_tuning/jobs`; - resourcePath = resourcePath + check getPathForQueryParam(queries); - return self.clientEp->get(resourcePath, headers); - } - - # Get info about a fine-tuning job. - # - # [Learn more about fine-tuning](/docs/guides/fine-tuning) - # - # + fine_tuning_job_id - The ID of the fine-tuning job. - # + headers - Headers to be sent with the request - # + return - OK - resource isolated function get fine_tuning/jobs/[string fine_tuning_job_id](map headers = {}) returns FineTuningJob|error { - string resourcePath = string `/fine_tuning/jobs/${getEncodedUri(fine_tuning_job_id)}`; - return self.clientEp->get(resourcePath, headers); - } - - # List checkpoints for a fine-tuning job. - # - # + fine_tuning_job_id - The ID of the fine-tuning job to get checkpoints for. - # + headers - Headers to be sent with the request - # + queries - Queries to be sent with the request - # + return - OK - resource isolated function get fine_tuning/jobs/[string fine_tuning_job_id]/checkpoints(map headers = {}, *ListFineTuningJobCheckpointsQueries queries) returns ListFineTuningJobCheckpointsResponse|error { - string resourcePath = string `/fine_tuning/jobs/${getEncodedUri(fine_tuning_job_id)}/checkpoints`; - resourcePath = resourcePath + check getPathForQueryParam(queries); - return self.clientEp->get(resourcePath, headers); - } - - # Get status updates for a fine-tuning job. - # - # + fine_tuning_job_id - The ID of the fine-tuning job to get events for. - # + headers - Headers to be sent with the request - # + queries - Queries to be sent with the request - # + return - OK - resource isolated function get fine_tuning/jobs/[string fine_tuning_job_id]/events(map headers = {}, *ListFineTuningEventsQueries queries) returns ListFineTuningJobEventsResponse|error { - string resourcePath = string `/fine_tuning/jobs/${getEncodedUri(fine_tuning_job_id)}/events`; - resourcePath = resourcePath + check getPathForQueryParam(queries); - return self.clientEp->get(resourcePath, headers); - } - - # Lists the currently available models, and provides basic information about each one such as the owner and availability. - # - # + headers - Headers to be sent with the request - # + return - OK - resource isolated function get models(map headers = {}) returns ListModelsResponse|error { - string resourcePath = string `/models`; - return self.clientEp->get(resourcePath, headers); - } - - # Retrieves a model instance, providing basic information about the model such as the owner and permissioning. - # - # + model - The ID of the model to use for this request - # + headers - Headers to be sent with the request - # + return - OK - resource isolated function get models/[string model](map headers = {}) returns Model|error { - string resourcePath = string `/models/${getEncodedUri(model)}`; - return self.clientEp->get(resourcePath, headers); - } - - # Upload a file that can be used across various endpoints. Individual files can be up to 512 MB, and the size of all files uploaded by one organization can be up to 100 GB. - # - # The Assistants API supports files up to 2 million tokens and of specific file types. See the [Assistants Tools guide](/docs/assistants/tools) for details. - # - # The Fine-tuning API only supports `.jsonl` files. The input also has certain required formats for fine-tuning [chat](/docs/api-reference/fine-tuning/chat-input) or [completions](/docs/api-reference/fine-tuning/completions-input) models. - # - # The Batch API only supports `.jsonl` files up to 100 MB in size. The input also has a specific required [format](/docs/api-reference/batch/request-input). - # - # Please [contact us](https://help.openai.com/) if you need to increase these storage limits. - # - # + headers - Headers to be sent with the request - # + return - OK - resource isolated function post files(CreateFileRequest payload, map headers = {}) returns OpenAIFile|error { - string resourcePath = string `/files`; - http:Request request = new; - mime:Entity[] bodyParts = check createBodyParts(payload); - request.setBodyParts(bodyParts); - return self.clientEp->post(resourcePath, request, headers); - } - - # Creates a fine-tuning job which begins the process of creating a new model from a given dataset. - # - # Response includes details of the enqueued job including job status and the name of the fine-tuned models once complete. - # - # [Learn more about fine-tuning](/docs/guides/fine-tuning) - # - # + headers - Headers to be sent with the request - # + return - OK - resource isolated function post fine_tuning/jobs(CreateFineTuningJobRequest payload, map headers = {}) returns FineTuningJob|error { - string resourcePath = string `/fine_tuning/jobs`; - http:Request request = new; - json jsonBody = payload.toJson(); - request.setPayload(jsonBody, "application/json"); - return self.clientEp->post(resourcePath, request, headers); - } - - # Immediately cancel a fine-tune job. - # - # + fine_tuning_job_id - The ID of the fine-tuning job to cancel. - # + headers - Headers to be sent with the request - # + return - OK - resource isolated function post fine_tuning/jobs/[string fine_tuning_job_id]/cancel(map headers = {}) returns FineTuningJob|error { - string resourcePath = string `/fine_tuning/jobs/${getEncodedUri(fine_tuning_job_id)}/cancel`; - http:Request request = new; - return self.clientEp->post(resourcePath, request, headers); - } -} diff --git a/examples/Sports-headline-analyzer/main.bal b/examples/Sports-headline-analyzer/main.bal index c5f826b..85eb540 100644 --- a/examples/Sports-headline-analyzer/main.bal +++ b/examples/Sports-headline-analyzer/main.bal @@ -1,32 +1,31 @@ import ballerina/io; import ballerina/lang.runtime; - -// import ballerinax/openai.finetunes; +import ballerinax/openai.finetunes; configurable string token = ?; string serviceUrl = "https://api.openai.com/v1"; string trainingFileName = "training.jsonl"; string trainingFilePath = "./data/" + trainingFileName; -final ConnectionConfig config = {auth: {token}}; -final Client openAIFinetunes = check new Client(config, serviceUrl); +final finetunes:ConnectionConfig config = {auth: {token}}; +final finetunes:Client openAIFinetunes = check new Client(config, serviceUrl); public function main() returns error? { byte[] trainingFileContent = check io:fileReadBytes(trainingFilePath); - CreateFileRequest trainingFileRequest = { + finetunes:CreateFileRequest trainingFileRequest = { file: {fileContent: trainingFileContent, fileName: trainingFileName}, purpose: "fine-tune" }; - OpenAIFile trainingFileResponse = + finetunes:OpenAIFile trainingFileResponse = check openAIFinetunes->/files.post(trainingFileRequest); string trainingFileId = trainingFileResponse.id; io:println("Training file id: " + trainingFileId); - CreateFineTuningJobRequest fineTuneRequest = { + finetunes:CreateFineTuningJobRequest fineTuneRequest = { model: "gpt-4o-mini-2024-07-18", training_file: trainingFileId, hyperparameters: { @@ -36,12 +35,12 @@ public function main() returns error? { } }; - FineTuningJob fineTuneResponse = + finetunes:FineTuningJob fineTuneResponse = check openAIFinetunes->/fine_tuning/jobs.post(fineTuneRequest); string fineTuneJobId = fineTuneResponse.id; io:println("Fine-tuning job id: " + fineTuneJobId); - FineTuningJob fineTuneJob = + finetunes:FineTuningJob fineTuneJob = check openAIFinetunes->/fine_tuning/jobs/[fineTuneJobId].get(); io:print("Validating files..."); @@ -67,7 +66,7 @@ public function main() returns error? { if (fineTuneJob.status != "succeeded") { io:println("Fine-tuning job failed."); - DeleteFileResponse deleteFileResponse = + finetunes:DeleteFileResponse deleteFileResponse = check openAIFinetunes->/files/[trainingFileId].delete(); if (deleteFileResponse.deleted == true) { io:println("Training file deleted successfully."); @@ -79,10 +78,10 @@ public function main() returns error? { } io:println("\n"); - ListFineTuningJobCheckpointsResponse checkpointsResponse = + finetunes:ListFineTuningJobCheckpointsResponse checkpointsResponse = check openAIFinetunes->/fine_tuning/jobs/[fineTuneJobId]/checkpoints.get(); - foreach FineTuningJobCheckpoint item in checkpointsResponse.data.reverse() { + foreach finetunes:FineTuningJobCheckpoint item in checkpointsResponse.data.reverse() { io:print("step: ", item.metrics.step); io:print(", train loss: ", item.metrics.train_loss); io:println(", train mean token accuracy: ", item.metrics.train_mean_token_accuracy); @@ -93,12 +92,11 @@ public function main() returns error? { io:println("Model: ", fineTuneJob.model); io:println("Fine-tuning job completed successfully."); - DeleteFileResponse deleteFileResponse = + finetunes:DeleteFileResponse deleteFileResponse = check openAIFinetunes->/files/[trainingFileId].delete(); if (deleteFileResponse.deleted == true) { io:println("Training file deleted successfully."); } else { io:println("Failed to delete the training file."); } - } diff --git a/examples/Sports-headline-analyzer/types.bal b/examples/Sports-headline-analyzer/types.bal deleted file mode 100644 index 093632d..0000000 --- a/examples/Sports-headline-analyzer/types.bal +++ /dev/null @@ -1,361 +0,0 @@ -// AUTO-GENERATED FILE. DO NOT MODIFY. -// This file is auto-generated by the Ballerina OpenAPI tool. - -// Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com). -// -// WSO2 LLC. licenses this file to you under the Apache License, -// Version 2.0 (the "License"); you may not use this file except -// in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import ballerina/http; - -# The `File` object represents a document that has been uploaded to OpenAI. -public type OpenAIFile record { - # The file identifier, which can be referenced in the API endpoints. - string id; - # The size of the file, in bytes. - int bytes; - # The Unix timestamp (in seconds) for when the file was created. - int created_at; - # The name of the file. - string filename; - # The object type, which is always `file`. - "file" 'object; - # The intended purpose of the file. Supported values are `assistants`, `assistants_output`, `batch`, `batch_output`, `fine-tune`, `fine-tune-results` and `vision`. - "assistants"|"assistants_output"|"batch"|"batch_output"|"fine-tune"|"fine-tune-results"|"vision" purpose; - # Deprecated. The current status of the file, which can be either `uploaded`, `processed`, or `error`. - # - # # Deprecated - @deprecated - "uploaded"|"processed"|"error" status; - # Deprecated. For details on why a fine-tuning training file failed validation, see the `error` field on `fine_tuning.job`. - string? status_details?; -}; - -public type CreateFineTuningJobRequest record { - # The name of the model to fine-tune. You can select one of the - # [supported models](/docs/guides/fine-tuning/what-models-can-be-fine-tuned). - string|"babbage-002"|"davinci-002"|"gpt-3.5-turbo" model; - # The ID of an uploaded file that contains training data. - # - # See [upload file](/docs/api-reference/files/create) for how to upload a file. - # - # Your dataset must be formatted as a JSONL file. Additionally, you must upload your file with the purpose `fine-tune`. - # - # The contents of the file should differ depending on if the model uses the [chat](/docs/api-reference/fine-tuning/chat-input) or [completions](/docs/api-reference/fine-tuning/completions-input) format. - # - # See the [fine-tuning guide](/docs/guides/fine-tuning) for more details. - string training_file; - CreateFineTuningJobRequest_hyperparameters hyperparameters?; - # A string of up to 18 characters that will be added to your fine-tuned model name. - # - # For example, a `suffix` of "custom-model-name" would produce a model name like `ft:gpt-3.5-turbo:openai:custom-model-name:7p4lURel`. - string? suffix?; - # The ID of an uploaded file that contains validation data. - # - # If you provide this file, the data is used to generate validation - # metrics periodically during fine-tuning. These metrics can be viewed in - # the fine-tuning results file. - # The same data should not be present in both train and validation files. - # - # Your dataset must be formatted as a JSONL file. You must upload your file with the purpose `fine-tune`. - # - # See the [fine-tuning guide](/docs/guides/fine-tuning) for more details. - string? validation_file?; - # A list of integrations to enable for your fine-tuning job. - CreateFineTuningJobRequest_integrations[]? integrations?; - # The seed controls the reproducibility of the job. Passing in the same seed and job parameters should produce the same results, but may differ in rare cases. - # If a seed is not specified, one will be generated for you. - int? seed?; -}; - -# The `fine_tuning.job.checkpoint` object represents a model checkpoint for a fine-tuning job that is ready to use. -public type FineTuningJobCheckpoint record { - # The checkpoint identifier, which can be referenced in the API endpoints. - string id; - # The Unix timestamp (in seconds) for when the checkpoint was created. - int created_at; - # The name of the fine-tuned checkpoint model that is created. - string fine_tuned_model_checkpoint; - # The step number that the checkpoint was created at. - int step_number; - FineTuningJobCheckpoint_metrics metrics; - # The name of the fine-tuning job that this checkpoint was created from. - string fine_tuning_job_id; - # The object type, which is always "fine_tuning.job.checkpoint". - "fine_tuning.job.checkpoint" 'object; -}; - -public type ListPaginatedFineTuningJobsResponse record { - FineTuningJob[] data; - boolean has_more; - "list" 'object; -}; - -# The hyperparameters used for the fine-tuning job. -public type CreateFineTuningJobRequest_hyperparameters record { - # Number of examples in each batch. A larger batch size means that model parameters - # are updated less frequently, but with lower variance. - "auto"|int batch_size = "auto"; - # Scaling factor for the learning rate. A smaller learning rate may be useful to avoid - # overfitting. - "auto"|decimal learning_rate_multiplier = "auto"; - # The number of epochs to train the model for. An epoch refers to one full cycle - # through the training dataset. - "auto"|int n_epochs = "auto"; -}; - -# Represents the Queries record for the operation: listFineTuningEvents -public type ListFineTuningEventsQueries record { - # Number of events to retrieve. - int 'limit = 20; - # Identifier for the last event from the previous pagination request. - string after?; -}; - -# Represents the Queries record for the operation: listFineTuningJobCheckpoints -public type ListFineTuningJobCheckpointsQueries record { - # Number of checkpoints to retrieve. - int 'limit = 10; - # Identifier for the last checkpoint ID from the previous pagination request. - string after?; -}; - -# The settings for your integration with Weights and Biases. This payload specifies the project that -# metrics will be sent to. Optionally, you can set an explicit display name for your run, add tags -# to your run, and set a default entity (team, username, etc) to be associated with your run. -public type CreateFineTuningJobRequest_wandb record { - # The name of the project that the new run will be created under. - string project; - # A display name to set for the run. If not set, we will use the Job ID as the name. - string? name?; - # The entity to use for the run. This allows you to set the team or username of the WandB user that you would - # like associated with the run. If not set, the default entity for the registered WandB API key is used. - string? entity?; - # A list of tags to be attached to the newly created run. These tags are passed through directly to WandB. Some - # default tags are generated by OpenAI: "openai/finetune", "openai/{base-model}", "openai/{ftjob-abcdef}". - string[] tags?; -}; - -# The hyperparameters used for the fine-tuning job. See the [fine-tuning guide](/docs/guides/fine-tuning) for more details. -public type FineTuningJob_hyperparameters record { - # The number of epochs to train the model for. An epoch refers to one full cycle through the training dataset. - # "auto" decides the optimal number of epochs based on the size of the dataset. If setting the number manually, we support any number between 1 and 50 epochs. - "auto"|int n_epochs; -}; - -# Represents the Queries record for the operation: listFiles -public type ListFilesQueries record { - # Only return files with the given purpose. - string purpose?; -}; - -public type ListFineTuningJobCheckpointsResponse record { - FineTuningJobCheckpoint[] data; - "list" 'object; - string? first_id?; - string? last_id?; - boolean has_more; -}; - -public type DeleteModelResponse record { - string id; - boolean deleted; - string 'object; -}; - -public type FineTuningIntegration record { - # The type of the integration being enabled for the fine-tuning job - "wandb" 'type; - CreateFineTuningJobRequest_wandb wandb; -}; - -# Provides a set of configurations for controlling the behaviours when communicating with a remote HTTP endpoint. -@display {label: "Connection Config"} -public type ConnectionConfig record {| - # Configurations related to client authentication - http:BearerTokenConfig auth; - # The HTTP version understood by the client - http:HttpVersion httpVersion = http:HTTP_2_0; - # Configurations related to HTTP/1.x protocol - ClientHttp1Settings http1Settings?; - # Configurations related to HTTP/2 protocol - http:ClientHttp2Settings http2Settings?; - # The maximum time to wait (in seconds) for a response before closing the connection - decimal timeout = 60; - # The choice of setting `forwarded`/`x-forwarded` header - string forwarded = "disable"; - # Configurations associated with request pooling - http:PoolConfiguration poolConfig?; - # HTTP caching related configurations - http:CacheConfig cache?; - # Specifies the way of handling compression (`accept-encoding`) header - http:Compression compression = http:COMPRESSION_AUTO; - # Configurations associated with the behaviour of the Circuit Breaker - http:CircuitBreakerConfig circuitBreaker?; - # Configurations associated with retrying - http:RetryConfig retryConfig?; - # Configurations associated with inbound response size limits - http:ResponseLimitConfigs responseLimits?; - # SSL/TLS-related options - http:ClientSecureSocket secureSocket?; - # Proxy server related options - http:ProxyConfig proxy?; - # Enables the inbound payload validation functionality which provided by the constraint package. Enabled by default - boolean validation = true; -|}; - -# Metrics at the step number during the fine-tuning job. -public type FineTuningJobCheckpoint_metrics record { - decimal step?; - decimal train_loss?; - decimal train_mean_token_accuracy?; - decimal valid_loss?; - decimal valid_mean_token_accuracy?; - decimal full_valid_loss?; - decimal full_valid_mean_token_accuracy?; -}; - -public type CreateFileRequest record {| - # The File object (not file name) to be uploaded. - record {byte[] fileContent; string fileName;} file; - # The intended purpose of the uploaded file. - # - # Use "assistants" for [Assistants](/docs/api-reference/assistants) and [Message](/docs/api-reference/messages) files, "vision" for Assistants image file inputs, "batch" for [Batch API](/docs/guides/batch), and "fine-tune" for [Fine-tuning](/docs/api-reference/fine-tuning). - "assistants"|"batch"|"fine-tune"|"vision" purpose; -|}; - -# The `fine_tuning.job` object represents a fine-tuning job that has been created through the API. -public type FineTuningJob record { - # The object identifier, which can be referenced in the API endpoints. - string id; - # The Unix timestamp (in seconds) for when the fine-tuning job was created. - int created_at; - FineTuningJob_error? 'error; - # The name of the fine-tuned model that is being created. The value will be null if the fine-tuning job is still running. - string? fine_tuned_model; - # The Unix timestamp (in seconds) for when the fine-tuning job was finished. The value will be null if the fine-tuning job is still running. - int? finished_at; - FineTuningJob_hyperparameters hyperparameters; - # The base model that is being fine-tuned. - string model; - # The object type, which is always "fine_tuning.job". - "fine_tuning.job" 'object; - # The organization that owns the fine-tuning job. - string organization_id; - # The compiled results file ID(s) for the fine-tuning job. You can retrieve the results with the [Files API](/docs/api-reference/files/retrieve-contents). - string[] result_files; - # The current status of the fine-tuning job, which can be either `validating_files`, `queued`, `running`, `succeeded`, `failed`, or `cancelled`. - "validating_files"|"queued"|"running"|"succeeded"|"failed"|"cancelled" status; - # The total number of billable tokens processed by this fine-tuning job. The value will be null if the fine-tuning job is still running. - int? trained_tokens; - # The file ID used for training. You can retrieve the training data with the [Files API](/docs/api-reference/files/retrieve-contents). - string training_file; - # The file ID used for validation. You can retrieve the validation results with the [Files API](/docs/api-reference/files/retrieve-contents). - string? validation_file; - # A list of integrations to enable for this fine-tuning job. - (FineTuningIntegration)[]? integrations?; - # The seed used for the fine-tuning job. - int seed; - # The Unix timestamp (in seconds) for when the fine-tuning job is estimated to finish. The value will be null if the fine-tuning job is not running. - int? estimated_finish?; -}; - -# Proxy server configurations to be used with the HTTP client endpoint. -public type ProxyConfig record {| - # Host name of the proxy server - string host = ""; - # Proxy server port - int port = 0; - # Proxy server username - string userName = ""; - # Proxy server password - @display {label: "", kind: "password"} - string password = ""; -|}; - -# For fine-tuning jobs that have `failed`, this will contain more information on the cause of the failure. -public type FineTuningJob_error record { - # A machine-readable error code. - string code?; - # A human-readable error message. - string message?; - # The parameter that was invalid, usually `training_file` or `validation_file`. This field will be null if the failure was not parameter-specific. - string? param?; -}; - -public type CreateFineTuningJobRequest_integrations record { - # The type of integration to enable. Currently, only "wandb" (Weights and Biases) is supported. - "wandb" 'type; - CreateFineTuningJobRequest_wandb wandb; -}; - -public type ListModelsResponse record { - "list" 'object; - Model[] data; -}; - -# Fine-tuning job event object -public type FineTuningJobEvent record { - string id; - int created_at; - "info"|"warn"|"error" level; - string message; - "fine_tuning.job.event" 'object; -}; - -public type DeleteFileResponse record { - string id; - "file" 'object; - boolean deleted; -}; - -# Provides settings related to HTTP/1.x protocol. -public type ClientHttp1Settings record {| - # Specifies whether to reuse a connection for multiple requests - http:KeepAlive keepAlive = http:KEEPALIVE_AUTO; - # The chunking behaviour of the request - http:Chunking chunking = http:CHUNKING_AUTO; - # Proxy server related options - ProxyConfig proxy?; -|}; - -# Describes an OpenAI model offering that can be used with the API. -public type Model record { - # The model identifier, which can be referenced in the API endpoints. - string id; - # The Unix timestamp (in seconds) when the model was created. - int created; - # The object type, which is always "model". - "model" 'object; - # The organization that owns the model. - string owned_by; -}; - -public type ListFineTuningJobEventsResponse record { - FineTuningJobEvent[] data; - "list" 'object; -}; - -public type ListFilesResponse record { - OpenAIFile[] data; - "list" 'object; -}; - -# Represents the Queries record for the operation: listPaginatedFineTuningJobs -public type ListPaginatedFineTuningJobsQueries record { - # Number of fine-tuning jobs to retrieve. - int 'limit = 20; - # Identifier for the last job from the previous pagination request. - string after?; -}; diff --git a/examples/Sports-headline-analyzer/utils.bal b/examples/Sports-headline-analyzer/utils.bal deleted file mode 100644 index 0ce7ca0..0000000 --- a/examples/Sports-headline-analyzer/utils.bal +++ /dev/null @@ -1,252 +0,0 @@ -// AUTO-GENERATED FILE. DO NOT MODIFY. -// This file is auto-generated by the Ballerina OpenAPI tool. - -// Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com). -// -// WSO2 LLC. licenses this file to you under the Apache License, -// Version 2.0 (the "License"); you may not use this file except -// in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import ballerina/mime; -import ballerina/url; - -type SimpleBasicType string|boolean|int|float|decimal; - -# Represents encoding mechanism details. -type Encoding record { - # Defines how multiple values are delimited - string style = FORM; - # Specifies whether arrays and objects should generate as separate fields - boolean explode = true; - # Specifies the custom content type - string contentType?; - # Specifies the custom headers - map headers?; -}; - -enum EncodingStyle { - DEEPOBJECT, FORM, SPACEDELIMITED, PIPEDELIMITED -} - -final Encoding & readonly defaultEncoding = {}; - -# Serialize the record according to the deepObject style. -# -# + parent - Parent record name -# + anyRecord - Record to be serialized -# + return - Serialized record as a string -isolated function getDeepObjectStyleRequest(string parent, record {} anyRecord) returns string { - string[] recordArray = []; - foreach [string, anydata] [key, value] in anyRecord.entries() { - if value is SimpleBasicType { - recordArray.push(parent + "[" + key + "]" + "=" + getEncodedUri(value.toString())); - } else if value is SimpleBasicType[] { - recordArray.push(getSerializedArray(parent + "[" + key + "]" + "[]", value, DEEPOBJECT, true)); - } else if value is record {} { - string nextParent = parent + "[" + key + "]"; - recordArray.push(getDeepObjectStyleRequest(nextParent, value)); - } else if value is record {}[] { - string nextParent = parent + "[" + key + "]"; - recordArray.push(getSerializedRecordArray(nextParent, value, DEEPOBJECT)); - } - recordArray.push("&"); - } - _ = recordArray.pop(); - return string:'join("", ...recordArray); -} - -# Serialize the record according to the form style. -# -# + parent - Parent record name -# + anyRecord - Record to be serialized -# + explode - Specifies whether arrays and objects should generate separate parameters -# + return - Serialized record as a string -isolated function getFormStyleRequest(string parent, record {} anyRecord, boolean explode = true) returns string { - string[] recordArray = []; - if explode { - foreach [string, anydata] [key, value] in anyRecord.entries() { - if value is SimpleBasicType { - recordArray.push(key, "=", getEncodedUri(value.toString())); - } else if value is SimpleBasicType[] { - recordArray.push(getSerializedArray(key, value, explode = explode)); - } else if value is record {} { - recordArray.push(getFormStyleRequest(parent, value, explode)); - } - recordArray.push("&"); - } - _ = recordArray.pop(); - } else { - foreach [string, anydata] [key, value] in anyRecord.entries() { - if value is SimpleBasicType { - recordArray.push(key, ",", getEncodedUri(value.toString())); - } else if value is SimpleBasicType[] { - recordArray.push(getSerializedArray(key, value, explode = false)); - } else if value is record {} { - recordArray.push(getFormStyleRequest(parent, value, explode)); - } - recordArray.push(","); - } - _ = recordArray.pop(); - } - return string:'join("", ...recordArray); -} - -# Serialize arrays. -# -# + arrayName - Name of the field with arrays -# + anyArray - Array to be serialized -# + style - Defines how multiple values are delimited -# + explode - Specifies whether arrays and objects should generate separate parameters -# + return - Serialized array as a string -isolated function getSerializedArray(string arrayName, anydata[] anyArray, string style = "form", boolean explode = true) returns string { - string key = arrayName; - string[] arrayValues = []; - if anyArray.length() > 0 { - if style == FORM && !explode { - arrayValues.push(key, "="); - foreach anydata i in anyArray { - arrayValues.push(getEncodedUri(i.toString()), ","); - } - } else if style == SPACEDELIMITED && !explode { - arrayValues.push(key, "="); - foreach anydata i in anyArray { - arrayValues.push(getEncodedUri(i.toString()), "%20"); - } - } else if style == PIPEDELIMITED && !explode { - arrayValues.push(key, "="); - foreach anydata i in anyArray { - arrayValues.push(getEncodedUri(i.toString()), "|"); - } - } else if style == DEEPOBJECT { - foreach anydata i in anyArray { - arrayValues.push(key, "[]", "=", getEncodedUri(i.toString()), "&"); - } - } else { - foreach anydata i in anyArray { - arrayValues.push(key, "=", getEncodedUri(i.toString()), "&"); - } - } - _ = arrayValues.pop(); - } - return string:'join("", ...arrayValues); -} - -# Serialize the array of records according to the form style. -# -# + parent - Parent record name -# + value - Array of records to be serialized -# + style - Defines how multiple values are delimited -# + explode - Specifies whether arrays and objects should generate separate parameters -# + return - Serialized record as a string -isolated function getSerializedRecordArray(string parent, record {}[] value, string style = FORM, boolean explode = true) returns string { - string[] serializedArray = []; - if style == DEEPOBJECT { - int arayIndex = 0; - foreach var recordItem in value { - serializedArray.push(getDeepObjectStyleRequest(parent + "[" + arayIndex.toString() + "]", recordItem), "&"); - arayIndex = arayIndex + 1; - } - } else { - if !explode { - serializedArray.push(parent, "="); - } - foreach var recordItem in value { - serializedArray.push(getFormStyleRequest(parent, recordItem, explode), ","); - } - } - _ = serializedArray.pop(); - return string:'join("", ...serializedArray); -} - -# Get Encoded URI for a given value. -# -# + value - Value to be encoded -# + return - Encoded string -isolated function getEncodedUri(anydata value) returns string { - string|error encoded = url:encode(value.toString(), "UTF8"); - if encoded is string { - return encoded; - } else { - return value.toString(); - } -} - -# Generate query path with query parameter. -# -# + queryParam - Query parameter map -# + encodingMap - Details on serialization mechanism -# + return - Returns generated Path or error at failure of client initialization -isolated function getPathForQueryParam(map queryParam, map encodingMap = {}) returns string|error { - string[] param = []; - if queryParam.length() > 0 { - param.push("?"); - foreach var [key, value] in queryParam.entries() { - if value is () { - _ = queryParam.remove(key); - continue; - } - Encoding encodingData = encodingMap.hasKey(key) ? encodingMap.get(key) : defaultEncoding; - if value is SimpleBasicType { - param.push(key, "=", getEncodedUri(value.toString())); - } else if value is SimpleBasicType[] { - param.push(getSerializedArray(key, value, encodingData.style, encodingData.explode)); - } else if value is record {} { - if encodingData.style == DEEPOBJECT { - param.push(getDeepObjectStyleRequest(key, value)); - } else { - param.push(getFormStyleRequest(key, value, encodingData.explode)); - } - } else { - param.push(key, "=", value.toString()); - } - param.push("&"); - } - _ = param.pop(); - } - string restOfPath = string:'join("", ...param); - return restOfPath; -} - -isolated function createBodyParts(record {|anydata...;|} anyRecord, map encodingMap = {}) returns mime:Entity[]|error { - mime:Entity[] entities = []; - foreach [string, anydata] [key, value] in anyRecord.entries() { - Encoding encodingData = encodingMap.hasKey(key) ? encodingMap.get(key) : {}; - mime:Entity entity = new mime:Entity(); - if value is record {byte[] fileContent; string fileName;} { - entity.setContentDisposition(mime:getContentDispositionObject(string `form-data; name=${key}; filename=${value.fileName}`)); - entity.setByteArray(value.fileContent); - } else if value is byte[] { - entity.setContentDisposition(mime:getContentDispositionObject(string `form-data; name=${key};`)); - entity.setByteArray(value); - } else if value is SimpleBasicType|SimpleBasicType[] { - entity.setContentDisposition(mime:getContentDispositionObject(string `form-data; name=${key};`)); - entity.setText(value.toString()); - } else if value is record {}|record {}[] { - entity.setContentDisposition(mime:getContentDispositionObject(string `form-data; name=${key};`)); - entity.setJson(value.toJson()); - } - if encodingData?.contentType is string { - check entity.setContentType(encodingData?.contentType.toString()); - } - map? headers = encodingData?.headers; - if headers is map { - foreach var [headerName, headerValue] in headers.entries() { - if headerValue is SimpleBasicType { - entity.setHeader(headerName, headerValue.toString()); - } - } - } - entities.push(entity); - } - return entities; -} From c025a9d0fa1b8772be8553f834d0e9213dbca26e Mon Sep 17 00:00:00 2001 From: KATTA-00 Date: Wed, 14 Aug 2024 15:26:59 +0530 Subject: [PATCH 08/32] Update Ballerina.toml --- examples/Sarcastic-bot/Ballerina.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/Sarcastic-bot/Ballerina.toml b/examples/Sarcastic-bot/Ballerina.toml index 74e9661..46deadd 100644 --- a/examples/Sarcastic-bot/Ballerina.toml +++ b/examples/Sarcastic-bot/Ballerina.toml @@ -4,11 +4,11 @@ name = "Sarcastic_bot" version = "0.1.0" distribution = "2201.9.2" -[build-options] -observabilityIncluded = true - [[dependency]] org = "ballerinax" name = "openai.finetunes" version = "1.0.7" repository = "local" + +[build-options] +observabilityIncluded = true From d8a451234f4971e32f9aa4c41bee34fe63f4bbe7 Mon Sep 17 00:00:00 2001 From: KATTA-00 Date: Wed, 14 Aug 2024 15:37:27 +0530 Subject: [PATCH 09/32] Update README.md --- examples/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/README.md b/examples/README.md index 3a55ec8..9cb5ea1 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,6 +1,6 @@ # Examples -The `ballerinax/openai.finetunes` connector provides practical examples illustrating usage in various scenarios. Explore these [examples](https://github.com/ballerina-platform/module-ballerinax-openai.finetunes/tree/main/examples), covering use cases like file uploading, finetuning models, getting events/checkpoints of models and deleting files. +The `ballerinax/openai.finetunes` connector provides practical examples illustrating usage in various scenarios. Explore these [examples](https://github.com/ballerina-platform/module-ballerinax-openai.finetunes/tree/main/examples), covering use cases like file uploading, finetuning models, getting events/checkpoints of a job and deleting files. 1. [Sarcastic Bot](https://github.com/ballerina-platform/module-ballerinax-openai.finetunes/tree/main/examples/Sarcastic-bot) - Fine-tune the GPT-3.5-turbo model to generate sarcastic responses From d261808b75eb0d908a5ba00756b909b9ad89224a Mon Sep 17 00:00:00 2001 From: KATTA-00 Date: Wed, 14 Aug 2024 15:41:42 +0530 Subject: [PATCH 10/32] Add licenses to examples --- examples/Sarcastic-bot/main.bal | 16 ++++++++++++++++ examples/Sports-headline-analyzer/main.bal | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/examples/Sarcastic-bot/main.bal b/examples/Sarcastic-bot/main.bal index eef70df..67c16e9 100644 --- a/examples/Sarcastic-bot/main.bal +++ b/examples/Sarcastic-bot/main.bal @@ -1,3 +1,19 @@ +// Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + import ballerina/io; import ballerina/lang.runtime; import ballerinax/openai.finetunes; diff --git a/examples/Sports-headline-analyzer/main.bal b/examples/Sports-headline-analyzer/main.bal index 85eb540..565b14c 100644 --- a/examples/Sports-headline-analyzer/main.bal +++ b/examples/Sports-headline-analyzer/main.bal @@ -1,3 +1,19 @@ +// Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + import ballerina/io; import ballerina/lang.runtime; import ballerinax/openai.finetunes; From e98e0be2d1b49895a308a7504d58a683c62ce2eb Mon Sep 17 00:00:00 2001 From: KATTA-00 Date: Wed, 14 Aug 2024 17:10:37 +0530 Subject: [PATCH 11/32] Updatethe examples and versions --- examples/Sarcastic-bot/Ballerina.toml | 8 +- examples/Sarcastic-bot/Dependencies.toml | 312 ++++++++++++++++++ examples/Sarcastic-bot/main.bal | 2 +- .../Dependencies.toml | 30 +- examples/Sports-headline-analyzer/main.bal | 2 +- 5 files changed, 334 insertions(+), 20 deletions(-) create mode 100644 examples/Sarcastic-bot/Dependencies.toml diff --git a/examples/Sarcastic-bot/Ballerina.toml b/examples/Sarcastic-bot/Ballerina.toml index 46deadd..2c74cb0 100644 --- a/examples/Sarcastic-bot/Ballerina.toml +++ b/examples/Sarcastic-bot/Ballerina.toml @@ -1,14 +1,14 @@ [package] org = "kanishka" -name = "Sarcastic_bot" +name = "Sports_headline_analyzer" version = "0.1.0" distribution = "2201.9.2" +[build-options] +observabilityIncluded = true + [[dependency]] org = "ballerinax" name = "openai.finetunes" version = "1.0.7" repository = "local" - -[build-options] -observabilityIncluded = true diff --git a/examples/Sarcastic-bot/Dependencies.toml b/examples/Sarcastic-bot/Dependencies.toml new file mode 100644 index 0000000..5730488 --- /dev/null +++ b/examples/Sarcastic-bot/Dependencies.toml @@ -0,0 +1,312 @@ +# AUTO-GENERATED FILE. DO NOT MODIFY. + +# This file is auto-generated by Ballerina for managing dependency versions. +# It should not be modified by hand. + +[ballerina] +dependencies-toml-version = "2" +distribution-version = "2201.9.2" + +[[package]] +org = "ballerina" +name = "auth" +version = "2.11.2" +dependencies = [ + {org = "ballerina", name = "crypto"}, + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.array"}, + {org = "ballerina", name = "lang.string"}, + {org = "ballerina", name = "log"} +] + +[[package]] +org = "ballerina" +name = "cache" +version = "3.8.0" +dependencies = [ + {org = "ballerina", name = "constraint"}, + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "task"}, + {org = "ballerina", name = "time"} +] + +[[package]] +org = "ballerina" +name = "constraint" +version = "1.5.0" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] + +[[package]] +org = "ballerina" +name = "crypto" +version = "2.7.2" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "time"} +] + +[[package]] +org = "ballerina" +name = "file" +version = "1.9.0" +dependencies = [ + {org = "ballerina", name = "io"}, + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "os"}, + {org = "ballerina", name = "time"} +] + +[[package]] +org = "ballerina" +name = "http" +version = "2.11.3" +dependencies = [ + {org = "ballerina", name = "auth"}, + {org = "ballerina", name = "cache"}, + {org = "ballerina", name = "constraint"}, + {org = "ballerina", name = "crypto"}, + {org = "ballerina", name = "file"}, + {org = "ballerina", name = "io"}, + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "jwt"}, + {org = "ballerina", name = "lang.array"}, + {org = "ballerina", name = "lang.decimal"}, + {org = "ballerina", name = "lang.int"}, + {org = "ballerina", name = "lang.regexp"}, + {org = "ballerina", name = "lang.runtime"}, + {org = "ballerina", name = "lang.string"}, + {org = "ballerina", name = "lang.value"}, + {org = "ballerina", name = "log"}, + {org = "ballerina", name = "mime"}, + {org = "ballerina", name = "oauth2"}, + {org = "ballerina", name = "observe"}, + {org = "ballerina", name = "time"}, + {org = "ballerina", name = "url"} +] + +[[package]] +org = "ballerina" +name = "io" +version = "1.6.1" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.value"} +] +modules = [ + {org = "ballerina", packageName = "io", moduleName = "io"} +] + +[[package]] +org = "ballerina" +name = "jballerina.java" +version = "0.0.0" + +[[package]] +org = "ballerina" +name = "jwt" +version = "2.11.0" +dependencies = [ + {org = "ballerina", name = "cache"}, + {org = "ballerina", name = "crypto"}, + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.int"}, + {org = "ballerina", name = "lang.string"}, + {org = "ballerina", name = "log"}, + {org = "ballerina", name = "time"} +] + +[[package]] +org = "ballerina" +name = "lang.__internal" +version = "0.0.0" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.object"} +] + +[[package]] +org = "ballerina" +name = "lang.array" +version = "0.0.0" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.__internal"} +] + +[[package]] +org = "ballerina" +name = "lang.decimal" +version = "0.0.0" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] + +[[package]] +org = "ballerina" +name = "lang.int" +version = "0.0.0" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.__internal"}, + {org = "ballerina", name = "lang.object"} +] + +[[package]] +org = "ballerina" +name = "lang.object" +version = "0.0.0" + +[[package]] +org = "ballerina" +name = "lang.regexp" +version = "0.0.0" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] + +[[package]] +org = "ballerina" +name = "lang.runtime" +version = "0.0.0" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] +modules = [ + {org = "ballerina", packageName = "lang.runtime", moduleName = "lang.runtime"} +] + +[[package]] +org = "ballerina" +name = "lang.string" +version = "0.0.0" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.regexp"} +] + +[[package]] +org = "ballerina" +name = "lang.value" +version = "0.0.0" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] + +[[package]] +org = "ballerina" +name = "log" +version = "2.9.0" +dependencies = [ + {org = "ballerina", name = "io"}, + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.value"}, + {org = "ballerina", name = "observe"} +] + +[[package]] +org = "ballerina" +name = "mime" +version = "2.9.0" +dependencies = [ + {org = "ballerina", name = "io"}, + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.int"} +] + +[[package]] +org = "ballerina" +name = "oauth2" +version = "2.11.0" +dependencies = [ + {org = "ballerina", name = "cache"}, + {org = "ballerina", name = "crypto"}, + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "log"}, + {org = "ballerina", name = "time"}, + {org = "ballerina", name = "url"} +] + +[[package]] +org = "ballerina" +name = "observe" +version = "1.2.3" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] + +[[package]] +org = "ballerina" +name = "os" +version = "1.8.0" +dependencies = [ + {org = "ballerina", name = "io"}, + {org = "ballerina", name = "jballerina.java"} +] + +[[package]] +org = "ballerina" +name = "task" +version = "2.5.0" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "time"} +] + +[[package]] +org = "ballerina" +name = "time" +version = "2.4.0" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] + +[[package]] +org = "ballerina" +name = "url" +version = "2.4.0" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] + +[[package]] +org = "ballerinai" +name = "observe" +version = "0.0.0" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "observe"} +] +modules = [ + {org = "ballerinai", packageName = "observe", moduleName = "observe"} +] + +[[package]] +org = "ballerinax" +name = "openai.finetunes" +version = "1.0.7" +dependencies = [ + {org = "ballerina", name = "http"}, + {org = "ballerina", name = "mime"}, + {org = "ballerina", name = "url"}, + {org = "ballerinai", name = "observe"} +] +modules = [ + {org = "ballerinax", packageName = "openai.finetunes", moduleName = "openai.finetunes"} +] + +[[package]] +org = "kanishka" +name = "Sports_headline_analyzer" +version = "0.1.0" +dependencies = [ + {org = "ballerina", name = "io"}, + {org = "ballerina", name = "lang.runtime"}, + {org = "ballerinai", name = "observe"}, + {org = "ballerinax", name = "openai.finetunes"} +] +modules = [ + {org = "kanishka", packageName = "Sports_headline_analyzer", moduleName = "Sports_headline_analyzer"} +] + diff --git a/examples/Sarcastic-bot/main.bal b/examples/Sarcastic-bot/main.bal index 67c16e9..24b6cf2 100644 --- a/examples/Sarcastic-bot/main.bal +++ b/examples/Sarcastic-bot/main.bal @@ -26,7 +26,7 @@ string trainingFilePath = "./data/" + trainingFileName; string validationFilePath = "./data/" + validationFileName; final finetunes:ConnectionConfig config = {auth: {token}}; -final finetunes:Client openAIFinetunes = check new Client(config, serviceUrl); +final finetunes:Client openAIFinetunes = check new finetunes:Client(config, serviceUrl); public function main() returns error? { diff --git a/examples/Sports-headline-analyzer/Dependencies.toml b/examples/Sports-headline-analyzer/Dependencies.toml index 38d3e3f..1bbaf58 100644 --- a/examples/Sports-headline-analyzer/Dependencies.toml +++ b/examples/Sports-headline-analyzer/Dependencies.toml @@ -85,10 +85,6 @@ dependencies = [ {org = "ballerina", name = "time"}, {org = "ballerina", name = "url"} ] -modules = [ - {org = "ballerina", packageName = "http", moduleName = "http"}, - {org = "ballerina", packageName = "http", moduleName = "http.httpscerr"} -] [[package]] org = "ballerina" @@ -218,9 +214,6 @@ dependencies = [ {org = "ballerina", name = "jballerina.java"}, {org = "ballerina", name = "lang.int"} ] -modules = [ - {org = "ballerina", packageName = "mime", moduleName = "mime"} -] [[package]] org = "ballerina" @@ -276,9 +269,6 @@ version = "2.4.0" dependencies = [ {org = "ballerina", name = "jballerina.java"} ] -modules = [ - {org = "ballerina", packageName = "url", moduleName = "url"} -] [[package]] org = "ballerinai" @@ -292,17 +282,29 @@ modules = [ {org = "ballerinai", packageName = "observe", moduleName = "observe"} ] +[[package]] +org = "ballerinax" +name = "openai.finetunes" +version = "1.0.7" +dependencies = [ + {org = "ballerina", name = "http"}, + {org = "ballerina", name = "mime"}, + {org = "ballerina", name = "url"}, + {org = "ballerinai", name = "observe"} +] +modules = [ + {org = "ballerinax", packageName = "openai.finetunes", moduleName = "openai.finetunes"} +] + [[package]] org = "kanishka" name = "Sports_headline_analyzer" version = "0.1.0" dependencies = [ - {org = "ballerina", name = "http"}, {org = "ballerina", name = "io"}, {org = "ballerina", name = "lang.runtime"}, - {org = "ballerina", name = "mime"}, - {org = "ballerina", name = "url"}, - {org = "ballerinai", name = "observe"} + {org = "ballerinai", name = "observe"}, + {org = "ballerinax", name = "openai.finetunes"} ] modules = [ {org = "kanishka", packageName = "Sports_headline_analyzer", moduleName = "Sports_headline_analyzer"} diff --git a/examples/Sports-headline-analyzer/main.bal b/examples/Sports-headline-analyzer/main.bal index 565b14c..927297f 100644 --- a/examples/Sports-headline-analyzer/main.bal +++ b/examples/Sports-headline-analyzer/main.bal @@ -24,7 +24,7 @@ string trainingFileName = "training.jsonl"; string trainingFilePath = "./data/" + trainingFileName; final finetunes:ConnectionConfig config = {auth: {token}}; -final finetunes:Client openAIFinetunes = check new Client(config, serviceUrl); +final finetunes:Client openAIFinetunes = check new finetunes:Client(config, serviceUrl); public function main() returns error? { From 0a9127ae1adba09218753d0891b8073791c133b4 Mon Sep 17 00:00:00 2001 From: KATTA-00 Date: Wed, 14 Aug 2024 17:12:49 +0530 Subject: [PATCH 12/32] remove unwanted spaces --- examples/Sports-headline-analyzer/main.bal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/Sports-headline-analyzer/main.bal b/examples/Sports-headline-analyzer/main.bal index 927297f..585291b 100644 --- a/examples/Sports-headline-analyzer/main.bal +++ b/examples/Sports-headline-analyzer/main.bal @@ -95,7 +95,7 @@ public function main() returns error? { io:println("\n"); finetunes:ListFineTuningJobCheckpointsResponse checkpointsResponse = - check openAIFinetunes->/fine_tuning/jobs/[fineTuneJobId]/checkpoints.get(); + check openAIFinetunes->/fine_tuning/jobs/[fineTuneJobId]/checkpoints.get(); foreach finetunes:FineTuningJobCheckpoint item in checkpointsResponse.data.reverse() { io:print("step: ", item.metrics.step); From 2a75cdd8e89d4af7966734029720bd60ac1ce0b7 Mon Sep 17 00:00:00 2001 From: Kanishka Gunawardana <83855665+KATTA-00@users.noreply.github.com> Date: Wed, 14 Aug 2024 21:29:23 +0530 Subject: [PATCH 13/32] Update examples/Sarcastic-bot/Ballerina.toml Co-authored-by: Nipuna Ransinghe --- examples/Sarcastic-bot/Ballerina.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/Sarcastic-bot/Ballerina.toml b/examples/Sarcastic-bot/Ballerina.toml index 2c74cb0..dc33b73 100644 --- a/examples/Sarcastic-bot/Ballerina.toml +++ b/examples/Sarcastic-bot/Ballerina.toml @@ -1,6 +1,6 @@ [package] org = "kanishka" -name = "Sports_headline_analyzer" +name = "sarcastic_bot" version = "0.1.0" distribution = "2201.9.2" From 8228921094b0de76f8385f1555004e6bbb04e47e Mon Sep 17 00:00:00 2001 From: Kanishka Gunawardana <83855665+KATTA-00@users.noreply.github.com> Date: Wed, 14 Aug 2024 21:29:34 +0530 Subject: [PATCH 14/32] Update examples/Sarcastic-bot/Ballerina.toml Co-authored-by: Nipuna Ransinghe --- examples/Sarcastic-bot/Ballerina.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/Sarcastic-bot/Ballerina.toml b/examples/Sarcastic-bot/Ballerina.toml index dc33b73..d061e40 100644 --- a/examples/Sarcastic-bot/Ballerina.toml +++ b/examples/Sarcastic-bot/Ballerina.toml @@ -1,5 +1,5 @@ [package] -org = "kanishka" +org = "wso2" name = "sarcastic_bot" version = "0.1.0" distribution = "2201.9.2" From ba483c0c7eae2870466663db5c5fe557dfdcb67d Mon Sep 17 00:00:00 2001 From: Kanishka Gunawardana <83855665+KATTA-00@users.noreply.github.com> Date: Wed, 14 Aug 2024 21:29:56 +0530 Subject: [PATCH 15/32] Update examples/Sarcastic-bot/main.bal Co-authored-by: Nipuna Ransinghe --- examples/Sarcastic-bot/main.bal | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/Sarcastic-bot/main.bal b/examples/Sarcastic-bot/main.bal index 24b6cf2..8e21c6c 100644 --- a/examples/Sarcastic-bot/main.bal +++ b/examples/Sarcastic-bot/main.bal @@ -19,11 +19,11 @@ import ballerina/lang.runtime; import ballerinax/openai.finetunes; configurable string token = ?; -string serviceUrl = "https://api.openai.com/v1"; -string trainingFileName = "training.jsonl"; -string validationFileName = "validation.jsonl"; -string trainingFilePath = "./data/" + trainingFileName; -string validationFilePath = "./data/" + validationFileName; +const SERVICE_URL = "https://api.openai.com/v1"; +const TRAINING_FILENAME = "training.jsonl"; +const VALIDATION_FILENAME = "validation.jsonl"; +const TRAINING_FILEPATH = "./data/" + trainingFileName; +const VALIDATION_FILEPATH = "./data/" + validationFileName; final finetunes:ConnectionConfig config = {auth: {token}}; final finetunes:Client openAIFinetunes = check new finetunes:Client(config, serviceUrl); From d78bbbb42ae16577c787832eb52b46683ed2faa1 Mon Sep 17 00:00:00 2001 From: Kanishka Gunawardana <83855665+KATTA-00@users.noreply.github.com> Date: Wed, 14 Aug 2024 21:30:26 +0530 Subject: [PATCH 16/32] Update examples/Sports-headline-analyzer/Ballerina.toml Co-authored-by: Nipuna Ransinghe --- examples/Sports-headline-analyzer/Ballerina.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/Sports-headline-analyzer/Ballerina.toml b/examples/Sports-headline-analyzer/Ballerina.toml index 2c74cb0..4a97b85 100644 --- a/examples/Sports-headline-analyzer/Ballerina.toml +++ b/examples/Sports-headline-analyzer/Ballerina.toml @@ -1,6 +1,6 @@ [package] org = "kanishka" -name = "Sports_headline_analyzer" +name = "sports_headline_analyzer" version = "0.1.0" distribution = "2201.9.2" From 7a4a2b912b72dd35886c7f8fcd5484c7e84593d9 Mon Sep 17 00:00:00 2001 From: Kanishka Gunawardana <83855665+KATTA-00@users.noreply.github.com> Date: Wed, 14 Aug 2024 21:30:39 +0530 Subject: [PATCH 17/32] Update examples/Sports-headline-analyzer/main.bal Co-authored-by: Nipuna Ransinghe --- examples/Sports-headline-analyzer/main.bal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/Sports-headline-analyzer/main.bal b/examples/Sports-headline-analyzer/main.bal index 585291b..a003241 100644 --- a/examples/Sports-headline-analyzer/main.bal +++ b/examples/Sports-headline-analyzer/main.bal @@ -80,7 +80,7 @@ public function main() returns error? { runtime:sleep(1); } - if (fineTuneJob.status != "succeeded") { + if fineTuneJob.status != "succeeded" { io:println("Fine-tuning job failed."); finetunes:DeleteFileResponse deleteFileResponse = check openAIFinetunes->/files/[trainingFileId].delete(); From 3cc6f6ac4cba252e06738daabbdc54c249487c5d Mon Sep 17 00:00:00 2001 From: Kanishka Gunawardana <83855665+KATTA-00@users.noreply.github.com> Date: Wed, 14 Aug 2024 21:30:52 +0530 Subject: [PATCH 18/32] Update examples/Sports-headline-analyzer/main.bal Co-authored-by: Nipuna Ransinghe --- examples/Sports-headline-analyzer/main.bal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/Sports-headline-analyzer/main.bal b/examples/Sports-headline-analyzer/main.bal index a003241..61921c5 100644 --- a/examples/Sports-headline-analyzer/main.bal +++ b/examples/Sports-headline-analyzer/main.bal @@ -110,7 +110,7 @@ public function main() returns error? { finetunes:DeleteFileResponse deleteFileResponse = check openAIFinetunes->/files/[trainingFileId].delete(); - if (deleteFileResponse.deleted == true) { + if (deleteFileResponse.deleted) { io:println("Training file deleted successfully."); } else { io:println("Failed to delete the training file."); From 2e8522160c3d8348dbba8a1747b262d9998affbe Mon Sep 17 00:00:00 2001 From: Kanishka Gunawardana <83855665+KATTA-00@users.noreply.github.com> Date: Wed, 14 Aug 2024 21:31:04 +0530 Subject: [PATCH 19/32] Update examples/Sports-headline-analyzer/Ballerina.toml Co-authored-by: Nipuna Ransinghe --- examples/Sports-headline-analyzer/Ballerina.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/Sports-headline-analyzer/Ballerina.toml b/examples/Sports-headline-analyzer/Ballerina.toml index 4a97b85..d0b8829 100644 --- a/examples/Sports-headline-analyzer/Ballerina.toml +++ b/examples/Sports-headline-analyzer/Ballerina.toml @@ -1,5 +1,5 @@ [package] -org = "kanishka" +org = "wso2" name = "sports_headline_analyzer" version = "0.1.0" distribution = "2201.9.2" From 623d8637b902294ca1a64da0a956edda03042c04 Mon Sep 17 00:00:00 2001 From: KATTA-00 Date: Wed, 14 Aug 2024 21:50:52 +0530 Subject: [PATCH 20/32] Update examples --- ballerina/Ballerina.toml | 2 +- examples/Sarcastic-bot/Ballerina.toml | 6 - examples/Sarcastic-bot/Dependencies.toml | 312 ------------------ examples/Sarcastic-bot/main.bal | 23 +- .../Sports-headline-analyzer/Ballerina.toml | 6 - .../Dependencies.toml | 312 ------------------ examples/Sports-headline-analyzer/main.bal | 22 +- 7 files changed, 23 insertions(+), 660 deletions(-) delete mode 100644 examples/Sarcastic-bot/Dependencies.toml delete mode 100644 examples/Sports-headline-analyzer/Dependencies.toml diff --git a/ballerina/Ballerina.toml b/ballerina/Ballerina.toml index 03e6c34..d2c7c22 100644 --- a/ballerina/Ballerina.toml +++ b/ballerina/Ballerina.toml @@ -2,7 +2,7 @@ distribution = "2201.9.2" org = "ballerinax" name = "openai.finetunes" -version = "1.0.7" +version = "1.0.5" license = ["Apache-2.0"] authors = ["Ballerina"] keywords = [] # TODO: Add keywords diff --git a/examples/Sarcastic-bot/Ballerina.toml b/examples/Sarcastic-bot/Ballerina.toml index d061e40..3702f81 100644 --- a/examples/Sarcastic-bot/Ballerina.toml +++ b/examples/Sarcastic-bot/Ballerina.toml @@ -6,9 +6,3 @@ distribution = "2201.9.2" [build-options] observabilityIncluded = true - -[[dependency]] -org = "ballerinax" -name = "openai.finetunes" -version = "1.0.7" -repository = "local" diff --git a/examples/Sarcastic-bot/Dependencies.toml b/examples/Sarcastic-bot/Dependencies.toml deleted file mode 100644 index 5730488..0000000 --- a/examples/Sarcastic-bot/Dependencies.toml +++ /dev/null @@ -1,312 +0,0 @@ -# AUTO-GENERATED FILE. DO NOT MODIFY. - -# This file is auto-generated by Ballerina for managing dependency versions. -# It should not be modified by hand. - -[ballerina] -dependencies-toml-version = "2" -distribution-version = "2201.9.2" - -[[package]] -org = "ballerina" -name = "auth" -version = "2.11.2" -dependencies = [ - {org = "ballerina", name = "crypto"}, - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "lang.array"}, - {org = "ballerina", name = "lang.string"}, - {org = "ballerina", name = "log"} -] - -[[package]] -org = "ballerina" -name = "cache" -version = "3.8.0" -dependencies = [ - {org = "ballerina", name = "constraint"}, - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "task"}, - {org = "ballerina", name = "time"} -] - -[[package]] -org = "ballerina" -name = "constraint" -version = "1.5.0" -dependencies = [ - {org = "ballerina", name = "jballerina.java"} -] - -[[package]] -org = "ballerina" -name = "crypto" -version = "2.7.2" -dependencies = [ - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "time"} -] - -[[package]] -org = "ballerina" -name = "file" -version = "1.9.0" -dependencies = [ - {org = "ballerina", name = "io"}, - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "os"}, - {org = "ballerina", name = "time"} -] - -[[package]] -org = "ballerina" -name = "http" -version = "2.11.3" -dependencies = [ - {org = "ballerina", name = "auth"}, - {org = "ballerina", name = "cache"}, - {org = "ballerina", name = "constraint"}, - {org = "ballerina", name = "crypto"}, - {org = "ballerina", name = "file"}, - {org = "ballerina", name = "io"}, - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "jwt"}, - {org = "ballerina", name = "lang.array"}, - {org = "ballerina", name = "lang.decimal"}, - {org = "ballerina", name = "lang.int"}, - {org = "ballerina", name = "lang.regexp"}, - {org = "ballerina", name = "lang.runtime"}, - {org = "ballerina", name = "lang.string"}, - {org = "ballerina", name = "lang.value"}, - {org = "ballerina", name = "log"}, - {org = "ballerina", name = "mime"}, - {org = "ballerina", name = "oauth2"}, - {org = "ballerina", name = "observe"}, - {org = "ballerina", name = "time"}, - {org = "ballerina", name = "url"} -] - -[[package]] -org = "ballerina" -name = "io" -version = "1.6.1" -dependencies = [ - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "lang.value"} -] -modules = [ - {org = "ballerina", packageName = "io", moduleName = "io"} -] - -[[package]] -org = "ballerina" -name = "jballerina.java" -version = "0.0.0" - -[[package]] -org = "ballerina" -name = "jwt" -version = "2.11.0" -dependencies = [ - {org = "ballerina", name = "cache"}, - {org = "ballerina", name = "crypto"}, - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "lang.int"}, - {org = "ballerina", name = "lang.string"}, - {org = "ballerina", name = "log"}, - {org = "ballerina", name = "time"} -] - -[[package]] -org = "ballerina" -name = "lang.__internal" -version = "0.0.0" -dependencies = [ - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "lang.object"} -] - -[[package]] -org = "ballerina" -name = "lang.array" -version = "0.0.0" -dependencies = [ - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "lang.__internal"} -] - -[[package]] -org = "ballerina" -name = "lang.decimal" -version = "0.0.0" -dependencies = [ - {org = "ballerina", name = "jballerina.java"} -] - -[[package]] -org = "ballerina" -name = "lang.int" -version = "0.0.0" -dependencies = [ - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "lang.__internal"}, - {org = "ballerina", name = "lang.object"} -] - -[[package]] -org = "ballerina" -name = "lang.object" -version = "0.0.0" - -[[package]] -org = "ballerina" -name = "lang.regexp" -version = "0.0.0" -dependencies = [ - {org = "ballerina", name = "jballerina.java"} -] - -[[package]] -org = "ballerina" -name = "lang.runtime" -version = "0.0.0" -dependencies = [ - {org = "ballerina", name = "jballerina.java"} -] -modules = [ - {org = "ballerina", packageName = "lang.runtime", moduleName = "lang.runtime"} -] - -[[package]] -org = "ballerina" -name = "lang.string" -version = "0.0.0" -dependencies = [ - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "lang.regexp"} -] - -[[package]] -org = "ballerina" -name = "lang.value" -version = "0.0.0" -dependencies = [ - {org = "ballerina", name = "jballerina.java"} -] - -[[package]] -org = "ballerina" -name = "log" -version = "2.9.0" -dependencies = [ - {org = "ballerina", name = "io"}, - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "lang.value"}, - {org = "ballerina", name = "observe"} -] - -[[package]] -org = "ballerina" -name = "mime" -version = "2.9.0" -dependencies = [ - {org = "ballerina", name = "io"}, - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "lang.int"} -] - -[[package]] -org = "ballerina" -name = "oauth2" -version = "2.11.0" -dependencies = [ - {org = "ballerina", name = "cache"}, - {org = "ballerina", name = "crypto"}, - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "log"}, - {org = "ballerina", name = "time"}, - {org = "ballerina", name = "url"} -] - -[[package]] -org = "ballerina" -name = "observe" -version = "1.2.3" -dependencies = [ - {org = "ballerina", name = "jballerina.java"} -] - -[[package]] -org = "ballerina" -name = "os" -version = "1.8.0" -dependencies = [ - {org = "ballerina", name = "io"}, - {org = "ballerina", name = "jballerina.java"} -] - -[[package]] -org = "ballerina" -name = "task" -version = "2.5.0" -dependencies = [ - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "time"} -] - -[[package]] -org = "ballerina" -name = "time" -version = "2.4.0" -dependencies = [ - {org = "ballerina", name = "jballerina.java"} -] - -[[package]] -org = "ballerina" -name = "url" -version = "2.4.0" -dependencies = [ - {org = "ballerina", name = "jballerina.java"} -] - -[[package]] -org = "ballerinai" -name = "observe" -version = "0.0.0" -dependencies = [ - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "observe"} -] -modules = [ - {org = "ballerinai", packageName = "observe", moduleName = "observe"} -] - -[[package]] -org = "ballerinax" -name = "openai.finetunes" -version = "1.0.7" -dependencies = [ - {org = "ballerina", name = "http"}, - {org = "ballerina", name = "mime"}, - {org = "ballerina", name = "url"}, - {org = "ballerinai", name = "observe"} -] -modules = [ - {org = "ballerinax", packageName = "openai.finetunes", moduleName = "openai.finetunes"} -] - -[[package]] -org = "kanishka" -name = "Sports_headline_analyzer" -version = "0.1.0" -dependencies = [ - {org = "ballerina", name = "io"}, - {org = "ballerina", name = "lang.runtime"}, - {org = "ballerinai", name = "observe"}, - {org = "ballerinax", name = "openai.finetunes"} -] -modules = [ - {org = "kanishka", packageName = "Sports_headline_analyzer", moduleName = "Sports_headline_analyzer"} -] - diff --git a/examples/Sarcastic-bot/main.bal b/examples/Sarcastic-bot/main.bal index 8e21c6c..8fc50e1 100644 --- a/examples/Sarcastic-bot/main.bal +++ b/examples/Sarcastic-bot/main.bal @@ -22,23 +22,23 @@ configurable string token = ?; const SERVICE_URL = "https://api.openai.com/v1"; const TRAINING_FILENAME = "training.jsonl"; const VALIDATION_FILENAME = "validation.jsonl"; -const TRAINING_FILEPATH = "./data/" + trainingFileName; -const VALIDATION_FILEPATH = "./data/" + validationFileName; +const TRAINING_FILEPATH = "./data/" + TRAINING_FILENAME; +const VALIDATION_FILEPATH = "./data/" + VALIDATION_FILENAME; final finetunes:ConnectionConfig config = {auth: {token}}; -final finetunes:Client openAIFinetunes = check new finetunes:Client(config, serviceUrl); +final finetunes:Client openAIFinetunes = check new finetunes:Client(config, SERVICE_URL); public function main() returns error? { - byte[] trainingFileContent = check io:fileReadBytes(trainingFilePath); - byte[] validationFileContent = check io:fileReadBytes(validationFilePath); + byte[] trainingFileContent = check io:fileReadBytes(TRAINING_FILEPATH); + byte[] validationFileContent = check io:fileReadBytes(VALIDATION_FILEPATH); finetunes:CreateFileRequest trainingFileRequest = { - file: {fileContent: trainingFileContent, fileName: trainingFileName}, + file: {fileContent: trainingFileContent, fileName: TRAINING_FILENAME}, purpose: "fine-tune" }; finetunes:CreateFileRequest validationFileRequest = { - file: {fileContent: validationFileContent, fileName: validationFileName}, + file: {fileContent: validationFileContent, fileName: VALIDATION_FILENAME}, purpose: "fine-tune" }; @@ -72,15 +72,14 @@ public function main() returns error? { check openAIFinetunes->/fine_tuning/jobs/[fineTuneJobId].get(); io:print("Validating files..."); - while (fineTuneJob.status == "validating_files") { + while fineTuneJob.status == "validating_files" { io:print("."); fineTuneJob = check openAIFinetunes->/fine_tuning/jobs/[fineTuneJobId].get(); - runtime:sleep(1); } io:print("\nFiles validated successfully."); - while (fineTuneJob.status == "queued") { + while fineTuneJob.status == "queued" { io:print("."); fineTuneJob = check openAIFinetunes->/fine_tuning/jobs/[fineTuneJobId].get(); runtime:sleep(1); @@ -88,14 +87,14 @@ public function main() returns error? { io:println("\nTraining..."); finetunes:ListFineTuningJobEventsResponse eventsResponse; - while (fineTuneJob.status == "running") { + while fineTuneJob.status == "running" { fineTuneJob = check openAIFinetunes->/fine_tuning/jobs/[fineTuneJobId].get(); eventsResponse = check openAIFinetunes->/fine_tuning/jobs/[fineTuneJobId]/events.get(); io:println(eventsResponse.data[0].message); runtime:sleep(1); } - if (fineTuneJob.status != "succeeded") { + if fineTuneJob.status != "succeeded" { io:println("Fine-tuning job failed."); return; } diff --git a/examples/Sports-headline-analyzer/Ballerina.toml b/examples/Sports-headline-analyzer/Ballerina.toml index d0b8829..de7996e 100644 --- a/examples/Sports-headline-analyzer/Ballerina.toml +++ b/examples/Sports-headline-analyzer/Ballerina.toml @@ -6,9 +6,3 @@ distribution = "2201.9.2" [build-options] observabilityIncluded = true - -[[dependency]] -org = "ballerinax" -name = "openai.finetunes" -version = "1.0.7" -repository = "local" diff --git a/examples/Sports-headline-analyzer/Dependencies.toml b/examples/Sports-headline-analyzer/Dependencies.toml deleted file mode 100644 index 1bbaf58..0000000 --- a/examples/Sports-headline-analyzer/Dependencies.toml +++ /dev/null @@ -1,312 +0,0 @@ -# AUTO-GENERATED FILE. DO NOT MODIFY. - -# This file is auto-generated by Ballerina for managing dependency versions. -# It should not be modified by hand. - -[ballerina] -dependencies-toml-version = "2" -distribution-version = "2201.9.2" - -[[package]] -org = "ballerina" -name = "auth" -version = "2.11.2" -dependencies = [ - {org = "ballerina", name = "crypto"}, - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "lang.array"}, - {org = "ballerina", name = "lang.string"}, - {org = "ballerina", name = "log"} -] - -[[package]] -org = "ballerina" -name = "cache" -version = "3.8.0" -dependencies = [ - {org = "ballerina", name = "constraint"}, - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "task"}, - {org = "ballerina", name = "time"} -] - -[[package]] -org = "ballerina" -name = "constraint" -version = "1.5.0" -dependencies = [ - {org = "ballerina", name = "jballerina.java"} -] - -[[package]] -org = "ballerina" -name = "crypto" -version = "2.7.2" -dependencies = [ - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "time"} -] - -[[package]] -org = "ballerina" -name = "file" -version = "1.9.0" -dependencies = [ - {org = "ballerina", name = "io"}, - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "os"}, - {org = "ballerina", name = "time"} -] - -[[package]] -org = "ballerina" -name = "http" -version = "2.11.3" -dependencies = [ - {org = "ballerina", name = "auth"}, - {org = "ballerina", name = "cache"}, - {org = "ballerina", name = "constraint"}, - {org = "ballerina", name = "crypto"}, - {org = "ballerina", name = "file"}, - {org = "ballerina", name = "io"}, - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "jwt"}, - {org = "ballerina", name = "lang.array"}, - {org = "ballerina", name = "lang.decimal"}, - {org = "ballerina", name = "lang.int"}, - {org = "ballerina", name = "lang.regexp"}, - {org = "ballerina", name = "lang.runtime"}, - {org = "ballerina", name = "lang.string"}, - {org = "ballerina", name = "lang.value"}, - {org = "ballerina", name = "log"}, - {org = "ballerina", name = "mime"}, - {org = "ballerina", name = "oauth2"}, - {org = "ballerina", name = "observe"}, - {org = "ballerina", name = "time"}, - {org = "ballerina", name = "url"} -] - -[[package]] -org = "ballerina" -name = "io" -version = "1.6.1" -dependencies = [ - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "lang.value"} -] -modules = [ - {org = "ballerina", packageName = "io", moduleName = "io"} -] - -[[package]] -org = "ballerina" -name = "jballerina.java" -version = "0.0.0" - -[[package]] -org = "ballerina" -name = "jwt" -version = "2.12.1" -dependencies = [ - {org = "ballerina", name = "cache"}, - {org = "ballerina", name = "crypto"}, - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "lang.int"}, - {org = "ballerina", name = "lang.string"}, - {org = "ballerina", name = "log"}, - {org = "ballerina", name = "time"} -] - -[[package]] -org = "ballerina" -name = "lang.__internal" -version = "0.0.0" -dependencies = [ - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "lang.object"} -] - -[[package]] -org = "ballerina" -name = "lang.array" -version = "0.0.0" -dependencies = [ - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "lang.__internal"} -] - -[[package]] -org = "ballerina" -name = "lang.decimal" -version = "0.0.0" -dependencies = [ - {org = "ballerina", name = "jballerina.java"} -] - -[[package]] -org = "ballerina" -name = "lang.int" -version = "0.0.0" -dependencies = [ - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "lang.__internal"}, - {org = "ballerina", name = "lang.object"} -] - -[[package]] -org = "ballerina" -name = "lang.object" -version = "0.0.0" - -[[package]] -org = "ballerina" -name = "lang.regexp" -version = "0.0.0" -dependencies = [ - {org = "ballerina", name = "jballerina.java"} -] - -[[package]] -org = "ballerina" -name = "lang.runtime" -version = "0.0.0" -dependencies = [ - {org = "ballerina", name = "jballerina.java"} -] -modules = [ - {org = "ballerina", packageName = "lang.runtime", moduleName = "lang.runtime"} -] - -[[package]] -org = "ballerina" -name = "lang.string" -version = "0.0.0" -dependencies = [ - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "lang.regexp"} -] - -[[package]] -org = "ballerina" -name = "lang.value" -version = "0.0.0" -dependencies = [ - {org = "ballerina", name = "jballerina.java"} -] - -[[package]] -org = "ballerina" -name = "log" -version = "2.9.0" -dependencies = [ - {org = "ballerina", name = "io"}, - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "lang.value"}, - {org = "ballerina", name = "observe"} -] - -[[package]] -org = "ballerina" -name = "mime" -version = "2.9.0" -dependencies = [ - {org = "ballerina", name = "io"}, - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "lang.int"} -] - -[[package]] -org = "ballerina" -name = "oauth2" -version = "2.11.0" -dependencies = [ - {org = "ballerina", name = "cache"}, - {org = "ballerina", name = "crypto"}, - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "log"}, - {org = "ballerina", name = "time"}, - {org = "ballerina", name = "url"} -] - -[[package]] -org = "ballerina" -name = "observe" -version = "1.2.3" -dependencies = [ - {org = "ballerina", name = "jballerina.java"} -] - -[[package]] -org = "ballerina" -name = "os" -version = "1.8.0" -dependencies = [ - {org = "ballerina", name = "io"}, - {org = "ballerina", name = "jballerina.java"} -] - -[[package]] -org = "ballerina" -name = "task" -version = "2.5.0" -dependencies = [ - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "time"} -] - -[[package]] -org = "ballerina" -name = "time" -version = "2.4.0" -dependencies = [ - {org = "ballerina", name = "jballerina.java"} -] - -[[package]] -org = "ballerina" -name = "url" -version = "2.4.0" -dependencies = [ - {org = "ballerina", name = "jballerina.java"} -] - -[[package]] -org = "ballerinai" -name = "observe" -version = "0.0.0" -dependencies = [ - {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "observe"} -] -modules = [ - {org = "ballerinai", packageName = "observe", moduleName = "observe"} -] - -[[package]] -org = "ballerinax" -name = "openai.finetunes" -version = "1.0.7" -dependencies = [ - {org = "ballerina", name = "http"}, - {org = "ballerina", name = "mime"}, - {org = "ballerina", name = "url"}, - {org = "ballerinai", name = "observe"} -] -modules = [ - {org = "ballerinax", packageName = "openai.finetunes", moduleName = "openai.finetunes"} -] - -[[package]] -org = "kanishka" -name = "Sports_headline_analyzer" -version = "0.1.0" -dependencies = [ - {org = "ballerina", name = "io"}, - {org = "ballerina", name = "lang.runtime"}, - {org = "ballerinai", name = "observe"}, - {org = "ballerinax", name = "openai.finetunes"} -] -modules = [ - {org = "kanishka", packageName = "Sports_headline_analyzer", moduleName = "Sports_headline_analyzer"} -] - diff --git a/examples/Sports-headline-analyzer/main.bal b/examples/Sports-headline-analyzer/main.bal index 61921c5..42a0cea 100644 --- a/examples/Sports-headline-analyzer/main.bal +++ b/examples/Sports-headline-analyzer/main.bal @@ -19,19 +19,19 @@ import ballerina/lang.runtime; import ballerinax/openai.finetunes; configurable string token = ?; -string serviceUrl = "https://api.openai.com/v1"; -string trainingFileName = "training.jsonl"; -string trainingFilePath = "./data/" + trainingFileName; +string SERVICE_URL = "https://api.openai.com/v1"; +string TRAINING_FILENAME = "training.jsonl"; +string TRAINING_FILEPATH = "./data/" + TRAINING_FILENAME; final finetunes:ConnectionConfig config = {auth: {token}}; -final finetunes:Client openAIFinetunes = check new finetunes:Client(config, serviceUrl); +final finetunes:Client openAIFinetunes = check new finetunes:Client(config, SERVICE_URL); public function main() returns error? { - byte[] trainingFileContent = check io:fileReadBytes(trainingFilePath); + byte[] trainingFileContent = check io:fileReadBytes(TRAINING_FILEPATH); finetunes:CreateFileRequest trainingFileRequest = { - file: {fileContent: trainingFileContent, fileName: trainingFileName}, + file: {fileContent: trainingFileContent, fileName: TRAINING_FILENAME}, purpose: "fine-tune" }; @@ -60,21 +60,21 @@ public function main() returns error? { check openAIFinetunes->/fine_tuning/jobs/[fineTuneJobId].get(); io:print("Validating files..."); - while (fineTuneJob.status == "validating_files") { + while fineTuneJob.status == "validating_files" { io:print("."); fineTuneJob = check openAIFinetunes->/fine_tuning/jobs/[fineTuneJobId].get(); runtime:sleep(1); } io:print("\nFiles validated successfully."); - while (fineTuneJob.status == "queued") { + while fineTuneJob.status == "queued" { io:print("."); fineTuneJob = check openAIFinetunes->/fine_tuning/jobs/[fineTuneJobId].get(); runtime:sleep(1); } io:print("\nTraining..."); - while (fineTuneJob.status == "running") { + while fineTuneJob.status == "running" { fineTuneJob = check openAIFinetunes->/fine_tuning/jobs/[fineTuneJobId].get(); io:print("."); runtime:sleep(1); @@ -84,7 +84,7 @@ public function main() returns error? { io:println("Fine-tuning job failed."); finetunes:DeleteFileResponse deleteFileResponse = check openAIFinetunes->/files/[trainingFileId].delete(); - if (deleteFileResponse.deleted == true) { + if deleteFileResponse.deleted { io:println("Training file deleted successfully."); } else { io:println("Failed to delete the training file."); @@ -110,7 +110,7 @@ public function main() returns error? { finetunes:DeleteFileResponse deleteFileResponse = check openAIFinetunes->/files/[trainingFileId].delete(); - if (deleteFileResponse.deleted) { + if deleteFileResponse.deleted { io:println("Training file deleted successfully."); } else { io:println("Failed to delete the training file."); From 84d75ee72ff0a4808f63d871cebd70606caf1824 Mon Sep 17 00:00:00 2001 From: Kanishka Gunawardana <83855665+KATTA-00@users.noreply.github.com> Date: Wed, 14 Aug 2024 22:34:11 +0530 Subject: [PATCH 21/32] Update examples/README.md Co-authored-by: Nipuna Ransinghe --- examples/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/README.md b/examples/README.md index 9cb5ea1..c9a8178 100644 --- a/examples/README.md +++ b/examples/README.md @@ -8,7 +8,7 @@ The `ballerinax/openai.finetunes` connector provides practical examples illustra ## Prerequisites -1. Generate a API as described in the [Setup guide](https://central.ballerina.io/ballerinax/openai.finetunes/latest#setup-guide). +1. Generate an API key as described in the [Setup guide](https://central.ballerina.io/ballerinax/openai.finetunes/latest#setup-guide). 2. For each example, create a `Config.toml` file the related configuration. Here's an example of how your `Config.toml` file should look: From ed5c95734ccc310fd83acf8493c279a5bd567af5 Mon Sep 17 00:00:00 2001 From: Kanishka Gunawardana <83855665+KATTA-00@users.noreply.github.com> Date: Wed, 14 Aug 2024 22:35:27 +0530 Subject: [PATCH 22/32] Update examples/Sports-headline-analyzer/Sports headline analyzer.md Co-authored-by: Nipuna Ransinghe --- examples/Sports-headline-analyzer/Sports headline analyzer.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/Sports-headline-analyzer/Sports headline analyzer.md b/examples/Sports-headline-analyzer/Sports headline analyzer.md index 888af3b..afa3c87 100644 --- a/examples/Sports-headline-analyzer/Sports headline analyzer.md +++ b/examples/Sports-headline-analyzer/Sports headline analyzer.md @@ -1,5 +1,5 @@ -## Sports Headlines Analyzer +## Sports headlines analyzer This use case illustrates how the OpenAI Fine-tunes API v1 can be used to fine-tune the GPT-4o-mini model for extracting structured information from sports headlines. The example outlines a series of steps that include using the OpenAI Files API v1 to upload training data, then employing the OpenAI Fine-tunes API v1 to fine-tune the GPT-4o-mini model with this data, and finally printing the model's checkpoints and deleting the data file. From c18fa99eaeec977c112d6987341a0ed2f35a8813 Mon Sep 17 00:00:00 2001 From: Kanishka Gunawardana <83855665+KATTA-00@users.noreply.github.com> Date: Wed, 14 Aug 2024 22:35:36 +0530 Subject: [PATCH 23/32] Update examples/Sports-headline-analyzer/Sports headline analyzer.md Co-authored-by: Nipuna Ransinghe --- examples/Sports-headline-analyzer/Sports headline analyzer.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/Sports-headline-analyzer/Sports headline analyzer.md b/examples/Sports-headline-analyzer/Sports headline analyzer.md index afa3c87..c00bbdd 100644 --- a/examples/Sports-headline-analyzer/Sports headline analyzer.md +++ b/examples/Sports-headline-analyzer/Sports headline analyzer.md @@ -17,7 +17,7 @@ Create a `Config.toml` file in the example's root directory as follows: token = "" ``` -## Run the Example +## Run the example Execute the following command to run the example: From 38ed1dae86793644d127af946f621367a22e8aaf Mon Sep 17 00:00:00 2001 From: KATTA-00 Date: Wed, 14 Aug 2024 22:36:41 +0530 Subject: [PATCH 24/32] Remove gitignore from examples --- examples/Sarcastic-bot/.gitignore | 11 ----------- examples/Sports-headline-analyzer/.gitignore | 11 ----------- 2 files changed, 22 deletions(-) delete mode 100644 examples/Sarcastic-bot/.gitignore delete mode 100644 examples/Sports-headline-analyzer/.gitignore diff --git a/examples/Sarcastic-bot/.gitignore b/examples/Sarcastic-bot/.gitignore deleted file mode 100644 index d5fc29a..0000000 --- a/examples/Sarcastic-bot/.gitignore +++ /dev/null @@ -1,11 +0,0 @@ -# Ballerina generates this directory during the compilation of a package. -# It contains compiler-generated artifacts and the final executable if this is an application package. -target/ - -# Ballerina maintains the compiler-generated source code here. -# Remove this if you want to commit generated sources. -generated/ - -# Contains configuration values used during development time. -# See https://ballerina.io/learn/provide-values-to-configurable-variables/ for more details. -Config.toml diff --git a/examples/Sports-headline-analyzer/.gitignore b/examples/Sports-headline-analyzer/.gitignore deleted file mode 100644 index d5fc29a..0000000 --- a/examples/Sports-headline-analyzer/.gitignore +++ /dev/null @@ -1,11 +0,0 @@ -# Ballerina generates this directory during the compilation of a package. -# It contains compiler-generated artifacts and the final executable if this is an application package. -target/ - -# Ballerina maintains the compiler-generated source code here. -# Remove this if you want to commit generated sources. -generated/ - -# Contains configuration values used during development time. -# See https://ballerina.io/learn/provide-values-to-configurable-variables/ for more details. -Config.toml From 972a15c4e336b33a51afa70318d5f3be42137021 Mon Sep 17 00:00:00 2001 From: KATTA-00 Date: Wed, 14 Aug 2024 22:39:52 +0530 Subject: [PATCH 25/32] Update API key generation step in Sports headline analyzer.md --- examples/Sports-headline-analyzer/Sports headline analyzer.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/Sports-headline-analyzer/Sports headline analyzer.md b/examples/Sports-headline-analyzer/Sports headline analyzer.md index c00bbdd..50b1784 100644 --- a/examples/Sports-headline-analyzer/Sports headline analyzer.md +++ b/examples/Sports-headline-analyzer/Sports headline analyzer.md @@ -5,7 +5,7 @@ This use case illustrates how the OpenAI Fine-tunes API v1 can be used to fine-t ## Prerequisites -### 1. Generate an API Key +### 1. Generate an API key Refer to the [Setup guide](https://central.ballerina.io/ballerinax/openai.finetunes/latest#setup-guide) to obtain the API key. From c1c2ba79430f571a4a041297d1a575f5ac735a23 Mon Sep 17 00:00:00 2001 From: KATTA-00 Date: Wed, 14 Aug 2024 22:45:56 +0530 Subject: [PATCH 26/32] Add the examples to README.md --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index aa47bdd..46a7c22 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,9 @@ The `OpenAI Finetunes` connector provides practical examples illustrating usage in various scenarios. Explore these [examples](https://github.com/module-ballerinax-openai.finetunes/tree/main/examples/), covering the following use cases: -[//]: # (TODO: Add examples) +1. [Sarcastic Bot](https://github.com/ballerina-platform/module-ballerinax-openai.finetunes/tree/main/examples/Sarcastic-bot) - Fine-tune the GPT-3.5-turbo model to generate sarcastic responses + +2. [Sports Headline Analyzer](https://github.com/ballerina-platform/module-ballerinax-openai.finetunes/tree/main/examples/Sports-headline-analyzer) - Fine-tune the GPT-4o-mini model to extract structured information (player, team, sport, and gender) from sports headlines. ## Build from the source From edbd1b8e702265b185b4175858f9556f7e2973b1 Mon Sep 17 00:00:00 2001 From: Kanishka Gunawardana <83855665+KATTA-00@users.noreply.github.com> Date: Thu, 15 Aug 2024 07:23:19 +0530 Subject: [PATCH 27/32] Update examples/Sarcastic-bot/Sarcastic bot.md Co-authored-by: Nipuna Ransinghe --- examples/Sarcastic-bot/Sarcastic bot.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/Sarcastic-bot/Sarcastic bot.md b/examples/Sarcastic-bot/Sarcastic bot.md index 56cb18f..97c8f4a 100644 --- a/examples/Sarcastic-bot/Sarcastic bot.md +++ b/examples/Sarcastic-bot/Sarcastic bot.md @@ -1,5 +1,5 @@ -## Sarcastic Bot +## Sarcastic bot This use case demonstrates how the OpenAI Fine-tunes API v1 can be harnessed to fine-tune the GPT-3.5-turbo model to produce sarcastic responses. The example showcases a series of steps that utilize the OpenAI Files API v1 to upload training data, followed by using the OpenAI Fine-tunes API v1 to fine-tune the GPT-3.5-turbo model with that data, getting the traning model events. From 8de9072d44af5b94a57be485e9a70fe3fe7b4758 Mon Sep 17 00:00:00 2001 From: Kanishka Gunawardana <83855665+KATTA-00@users.noreply.github.com> Date: Thu, 15 Aug 2024 07:23:39 +0530 Subject: [PATCH 28/32] Update README.md Co-authored-by: Nipuna Ransinghe --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 46a7c22..f27a938 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ The `OpenAI Finetunes` connector provides practical examples illustrating usage in various scenarios. Explore these [examples](https://github.com/module-ballerinax-openai.finetunes/tree/main/examples/), covering the following use cases: -1. [Sarcastic Bot](https://github.com/ballerina-platform/module-ballerinax-openai.finetunes/tree/main/examples/Sarcastic-bot) - Fine-tune the GPT-3.5-turbo model to generate sarcastic responses +1. [Sarcastic bot](https://github.com/ballerina-platform/module-ballerinax-openai.finetunes/tree/main/examples/Sarcastic-bot) - Fine-tune the GPT-3.5-turbo model to generate sarcastic responses 2. [Sports Headline Analyzer](https://github.com/ballerina-platform/module-ballerinax-openai.finetunes/tree/main/examples/Sports-headline-analyzer) - Fine-tune the GPT-4o-mini model to extract structured information (player, team, sport, and gender) from sports headlines. From 38bd08d8cff381853cf5419ac341ff3b0d62390a Mon Sep 17 00:00:00 2001 From: Kanishka Gunawardana <83855665+KATTA-00@users.noreply.github.com> Date: Thu, 15 Aug 2024 07:23:52 +0530 Subject: [PATCH 29/32] Update README.md Co-authored-by: Nipuna Ransinghe --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f27a938..25a855f 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ The `OpenAI Finetunes` connector provides practical examples illustrating usage 1. [Sarcastic bot](https://github.com/ballerina-platform/module-ballerinax-openai.finetunes/tree/main/examples/Sarcastic-bot) - Fine-tune the GPT-3.5-turbo model to generate sarcastic responses -2. [Sports Headline Analyzer](https://github.com/ballerina-platform/module-ballerinax-openai.finetunes/tree/main/examples/Sports-headline-analyzer) - Fine-tune the GPT-4o-mini model to extract structured information (player, team, sport, and gender) from sports headlines. +2. [Sports headline analyzer](https://github.com/ballerina-platform/module-ballerinax-openai.finetunes/tree/main/examples/Sports-headline-analyzer) - Fine-tune the GPT-4o-mini model to extract structured information (player, team, sport, and gender) from sports headlines. ## Build from the source From d74d6f31cd03c82579cdad7972271c36f20779e3 Mon Sep 17 00:00:00 2001 From: KATTA-00 Date: Thu, 15 Aug 2024 07:27:19 +0530 Subject: [PATCH 30/32] Add examples in Module.md and Package.md --- ballerina/Module.md | 4 +++- ballerina/Package.md | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ballerina/Module.md b/ballerina/Module.md index f0b0909..440a7cf 100644 --- a/ballerina/Module.md +++ b/ballerina/Module.md @@ -14,4 +14,6 @@ The `OpenAI Finetunes` connector provides practical examples illustrating usage in various scenarios. Explore these [examples](https://github.com/module-ballerinax-openai.finetunes/tree/main/examples/), covering the following use cases: -[//]: # (TODO: Add examples) \ No newline at end of file +1. [Sarcastic bot](https://github.com/ballerina-platform/module-ballerinax-openai.finetunes/tree/main/examples/Sarcastic-bot) - Fine-tune the GPT-3.5-turbo model to generate sarcastic responses + +2. [Sports headline analyzer](https://github.com/ballerina-platform/module-ballerinax-openai.finetunes/tree/main/examples/Sports-headline-analyzer) - Fine-tune the GPT-4o-mini model to extract structured information (player, team, sport, and gender) from sports headlines. \ No newline at end of file diff --git a/ballerina/Package.md b/ballerina/Package.md index f0b0909..440a7cf 100644 --- a/ballerina/Package.md +++ b/ballerina/Package.md @@ -14,4 +14,6 @@ The `OpenAI Finetunes` connector provides practical examples illustrating usage in various scenarios. Explore these [examples](https://github.com/module-ballerinax-openai.finetunes/tree/main/examples/), covering the following use cases: -[//]: # (TODO: Add examples) \ No newline at end of file +1. [Sarcastic bot](https://github.com/ballerina-platform/module-ballerinax-openai.finetunes/tree/main/examples/Sarcastic-bot) - Fine-tune the GPT-3.5-turbo model to generate sarcastic responses + +2. [Sports headline analyzer](https://github.com/ballerina-platform/module-ballerinax-openai.finetunes/tree/main/examples/Sports-headline-analyzer) - Fine-tune the GPT-4o-mini model to extract structured information (player, team, sport, and gender) from sports headlines. \ No newline at end of file From fb9ae0febd5597e6ecf5e965e8e913e59a4e7f0d Mon Sep 17 00:00:00 2001 From: KATTA-00 Date: Thu, 15 Aug 2024 07:30:57 +0530 Subject: [PATCH 31/32] Update README.md --- examples/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/README.md b/examples/README.md index c9a8178..0f1d308 100644 --- a/examples/README.md +++ b/examples/README.md @@ -16,7 +16,7 @@ The `ballerinax/openai.finetunes` connector provides practical examples illustra token = "" ``` -## Running an Example +## Running an example Execute the following commands to build an example from the source: From 5091637bda51a59d7bc9d60263c4472912d63c31 Mon Sep 17 00:00:00 2001 From: Nipuna Ransinghe Date: Thu, 15 Aug 2024 10:05:23 +0530 Subject: [PATCH 32/32] Apply suggestions from code review --- ballerina/Dependencies.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ballerina/Dependencies.toml b/ballerina/Dependencies.toml index 88106dc..d6606d1 100644 --- a/ballerina/Dependencies.toml +++ b/ballerina/Dependencies.toml @@ -107,7 +107,7 @@ version = "0.0.0" [[package]] org = "ballerina" name = "jwt" -version = "2.11.0" +version = "2.12.1" dependencies = [ {org = "ballerina", name = "cache"}, {org = "ballerina", name = "crypto"}, @@ -318,7 +318,7 @@ modules = [ [[package]] org = "ballerinax" name = "openai.finetunes" -version = "1.0.7" +version = "1.0.5" dependencies = [ {org = "ballerina", name = "http"}, {org = "ballerina", name = "log"},