Init the repo (#1)

* init the repo

* update working dir and node version

* finish tool

.github/CODEOWNERS

@@ -0,0 +1,2 @@
+# global
+@doriansmiley

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,38 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Desktop (please complete the following information):**
+ - OS: [e.g. iOS]
+ - Browser [e.g. chrome, safari]
+ - Version [e.g. 22]
+
+**Smartphone (please complete the following information):**
+ - Device: [e.g. iPhone6]
+ - OS: [e.g. iOS8.1]
+ - Browser [e.g. stock browser, safari]
+ - Version [e.g. 22]
+
+**Additional context**
+Add any other context about the problem here.

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,13 @@
+# What is the purpose of this change?
+Closes [issue](#)
+<!--
+What functionality does this introduce, or what feature does this fix? How and why?
+Please link to github issue.
+-->
+
+
+<!-- You can erase any parts of this template not applicable to your Pull Request. -->
+
+* [ ] Have you run linked to the issue this PR closes?
+* [ ] Have added tests?
+* [ ] Are all tests passing?

.github/workflows/build.yml

@@ -0,0 +1,22 @@
+name: Deploy
+on:
+  push:
+    tags:
+      - '*'
+defaults:
+  run:
+    shell: bash
+    working-directory: .
+jobs:
+  deploy:
+    name: Deploy
+    runs-on: ubuntu-latest
+    env:
+      SERVERLESS_ACCESS_KEY: ${{ secrets.SERVERLESS_ACCESS_KEY }}
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
+        with:
+          node-version: 17.5
+      - run: npm ci
+      # TODO add build, test, deploy commands

.github/workflows/test.yml

@@ -0,0 +1,22 @@
+name: Test
+on:
+  pull_request:
+    branches:
+      - master
+defaults:
+  run:
+    shell: bash
+    working-directory: .
+jobs:
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+    env:
+      SERVERLESS_ACCESS_KEY: ${{ secrets.SERVERLESS_ACCESS_KEY }}
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
+        with:
+          node-version: 17.5
+      - run: npm ci
+      #TODO add test command

.gitignore

@@ -102,3 +102,12 @@ dist
 
 # TernJS port file
 .tern-port
+
+# ENV vars
+**/.env
+
+# mac
+**/.DS_Store
+
+# Webstorm
+**/.idea

README.md

@@ -1,2 +1,73 @@
 # openai-nodejs-cli
-NodeJS CLI tool for OpenAI. Submit fine tuning jobs and more.
+NodeJS CLI tool for submitting fine-tuning jobs to OpenAI.
+
+# Options:
+```
+-cft --create-fine-tune-job [specFile, trainingData, validationData]
+    specFile -       relative path (from current directory) to the job spec json file.
+    trainingData -   relative path (from current directory) to the job training data json file.
+    validationData - relative path (from current directory) to the validation data json file.
+
+-gjb --get-job [jobId]
+    jobId -          The id of the job returned in the create call.
+
+-h --help
+    Prints the help menu.
+```
+
+# Example Usage
+```shell
+node index -cft ./specFile.json ./trainingData.jsonl ./validationData.jsonl
+```
+Note the referenced files are included in this repo. Please refer to the
+[OpenAI Fine-Tuning](https://beta.openai.com/docs/api-reference/fine-tunes/create) guide for more information about the file format
+and settings.
+
+Below is a typical response with the identifiers replaced with `xxx`
+```json
+{
+	"object": "fine-tune",
+	"id": "ft-xxx",
+	"hyperparams": {
+		"n_epochs": 4,
+		"batch_size": null,
+		"prompt_loss_weight": 0.01,
+		"learning_rate_multiplier": null
+	},
+	"organization_id": "xxx",
+	"model": "davinci",
+	"training_files": [{
+		"object": "file",
+		"id": "file-xxx",
+		"purpose": "fine-tune",
+		"filename": "xxx.jsonl",
+		"bytes": 698,
+		"created_at": 1673736863,
+		"status": "uploaded",
+		"status_details": null
+	}],
+	"validation_files": [],
+	"result_files": [],
+	"created_at": 1673736863,
+	"updated_at": 1673736863,
+	"status": "pending",
+	"fine_tuned_model": null,
+	"events": [{
+		"object": "fine-tune-event",
+		"level": "info",
+		"message": "Created fine-tune: ft-xxx",
+		"created_at": 1673736863
+	}]
+}
+```
+You can use the returned `id` property to poll the API:
+```shell
+node index -gjb ft-PSUaVtJNV3JzJAhUuAFh7g6M
+```
+IMPORTANT: it can take anywhere from 20-30 minutes for a job to succeed or days
+depending on OpenAI service health.
+
+There is a command under `./commands/pollFineTuningJob.js` that will poll the API, 
+but I have not implemented it due to the long response times
+
+

commands/createFineTuningJob.js

@@ -0,0 +1,85 @@
+const FormData = require("form-data");
+const axios = require("axios");
+const debug = require("debug")("openai-nodejs-cli");
+const fs = require("fs");
+const path = require("path");
+const { v4: uuidv4 } = require('uuid');
+
+const pollJob = require("./pollFineTuningJob");
+
+module.exports = async function f({specFile, trainingData, validationData}) {
+  const rootDir = path.resolve(".");
+  const specFileBuffer = fs.readFileSync(rootDir + "/" + specFile);
+  const spec = JSON.parse(specFileBuffer.toString());
+  const files = [trainingData, validationData];
+  const fileMap = new Map();
+
+  const promises = files.map((file, index) => {
+    if (!file) {
+      return undefined;
+    }
+    const rootDir = path.resolve(".");
+    const fileBuffer = fs.readFileSync(rootDir + "/" + trainingData);
+    const form = new FormData();
+    const filename = `${uuidv4()}.jsonl`;
+    switch (index) {
+      case 0:
+        fileMap.set('trainingData', filename);
+        break;
+      case 1:
+        fileMap.set('validationData', filename);
+        break;
+  }
+    form.append("file", fileBuffer, { filename });
+    form.append('purpose', 'fine-tune');
+    const headers = form.getHeaders();
+    headers["Authorization"] = `Bearer ${process.env.OPENAI_API_KEY}`;
+    debug(`headers ${headers}`);
+
+    return axios.post(`${process.env.OPEN_API_FILES_ENDPOINT}`, form, {
+      headers: {
+        ...form.getHeaders(),
+        ...headers
+      },
+    });
+  }).filter((file) => file !== undefined);
+
+  const results = await Promise.all(promises);
+  results.forEach((result) => {
+    switch (result.data.filename){
+      case fileMap.get('trainingData'):
+        fileMap.set('trainingData', result.data);
+        break;
+      case fileMap.get('validationData'):
+        fileMap.set('validationData', result.data);
+        break;
+      default:
+    }
+  });
+
+  const headers = {};
+  headers["Content-Type"] = 'application/json';
+  headers["Authorization"] = `Bearer ${process.env.OPENAI_API_KEY}`;
+  debug(`headers ${headers}`);
+  const params = {
+    ...spec,
+    training_file: fileMap.get('trainingData').id,
+  }
+  // remove null keys or the API call will fail
+  Object.keys(params).forEach(key => {
+    if (params[key] === null) {
+      delete params[key];
+    }
+  })
+  if (fileMap.get('validationData')?.id) {
+    params.validation_file = fileMap.get('validationData')?.id;
+  }
+  debug(`params ${params}`);
+  const result = await axios.post(`${process.env.OPEN_API_FINE_TUNE_ENDPOINT}`, params, {
+    headers: {
+      ...headers
+    },
+  });
+
+  return result.data;
+};

commands/getFineTuningJob.js

@@ -0,0 +1,15 @@
+const axios = require('axios');
+const debug = require('debug')('openai-nodejs-cli');
+
+module.exports = async function f({jobId}) {
+  const headers = {};
+  headers['Content-Type'] = 'application/json';
+  headers['Authorization'] = `Bearer ${process.env.OPENAI_API_KEY}`
+  debug(`headers ${headers}`);
+  const result = await axios.get(`${process.env.OPEN_API_FINE_TUNE_ENDPOINT}/${jobId}`, {
+    headers: {
+      ...headers
+    },
+  });
+  return result.data;
+};

commands/pollFineTuningJob.js

@@ -0,0 +1,36 @@
+const axios = require('axios');
+const debug = require('debug')('openai-nodejs-cli');
+
+// IMPORTANT: this code does work  but fine tuning jobs take a long time to complete, 10/30 minutes but sometimes days
+module.exports = async function f(job) {
+  return new Promise(async (resolve, reject) => {
+    const checkTimer = setInterval(async function () {
+      const headers = {};
+      headers['Content-Type'] = 'application/json';
+      headers['Authorization'] = `Bearer ${process.env.OPENAI_API_KEY}`
+      debug(`headers ${headers}`);
+      const result = await axios.get(`${process.env.OPEN_API_FINE_TUNE_ENDPOINT}/${job.id}`, {
+        headers: {
+          ...headers
+        },
+      });
+
+      switch(result.data.status){
+        case 'succeeded':
+          clearInterval(checkTimer);
+          debug('job complete');
+          resolve(result.data);
+          break;
+        case 'error':
+          // TODO figure out if they return the status error, it's not in the docs
+          clearInterval(checkTimer);
+          debug(`job failed: ${result.data}`);
+          reject(new Error('job failed'));
+          break;
+        default:
+          debug('poll job: ' + job.id.toString());
+      }
+
+    }, 1000);
+  });
+};

config/profiles.json

@@ -0,0 +1,16 @@
+{
+  "default": {
+    "daoCode": 1,
+    "mongoUrl": "mongodb://spiUser:spiPassword@localhost:27017/spi",
+    "assetStorageRoot": "~/workspace/NodeServices/webAssetManager/test/images/managedCache/",
+    "assetStorageType": 1,
+    "baseUrl": "http://localhost/",
+    "webRoot": "/",
+    "tokenExpiryOffset": 360000000,
+    "parserCode": "node",
+    "httpServiceCode": "node",
+    "eventDispatcherCode": "abstract",
+    "webAssetMangerApiURL": "http://localhost:3008",
+    "templateApiUrl": "http://localhost:3003"
+  }
+}