diff --git a/README.md b/README.md index 2d07e04c..c45ece39 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ GPU accelerated Neural networks in JavaScript for Browsers and Node.js - [For training with NeuralNetwork](#for-training-with-neuralnetwork) - [For training with `RNNTimeStep`, `LSTMTimeStep` and `GRUTimeStep`](#for-training-with-rnntimestep-lstmtimestep-and-grutimestep) - [For training with `RNN`, `LSTM` and `GRU`](#for-training-with-rnn-lstm-and-gru) - - [For training with `AE`](#for-training-with-ae) + - [For training with `AutoencoderGPU`](#for-training-with-ae) - [Training Options](#training-options) - [Async Training](#async-training) - [Cross Validation](#cross-validation) @@ -318,7 +318,7 @@ net.train([ const output = net.run('I feel great about the world!'); // 'happy' ``` -#### For training with `AE` +#### For training with `AutoencoderGPU` Each training pattern can either: @@ -328,7 +328,7 @@ Each training pattern can either: Training an autoencoder to compress the values of a XOR calculation: ```javascript -const net = new brain.AE( +const net = new brain.AutoencoderGPU( { hiddenLayers: [ 5, 2, 5 ] } @@ -644,7 +644,7 @@ The user interface used: - [`brain.NeuralNetwork`](src/neural-network.ts) - [Feedforward Neural Network](https://en.wikipedia.org/wiki/Feedforward_neural_network) with backpropagation - [`brain.NeuralNetworkGPU`](src/neural-network-gpu.ts) - [Feedforward Neural Network](https://en.wikipedia.org/wiki/Feedforward_neural_network) with backpropagation, GPU version -- [`brain.AE`](src/autoencoder.ts) - [Autoencoder or "AE"](https://en.wikipedia.org/wiki/Autoencoder) with backpropogation and GPU support +- [`brain.AutoencoderGPU`](src/autoencoder.ts) - [Autoencoder or "AutoencoderGPU"](https://en.wikipedia.org/wiki/Autoencoder) with backpropogation and GPU support - [`brain.recurrent.RNNTimeStep`](src/recurrent/rnn-time-step.ts) - [Time Step Recurrent Neural Network or "RNN"](https://en.wikipedia.org/wiki/Recurrent_neural_network) - [`brain.recurrent.LSTMTimeStep`](src/recurrent/lstm-time-step.ts) - [Time Step Long Short Term Memory Neural Network or "LSTM"](https://en.wikipedia.org/wiki/Long_short-term_memory) - [`brain.recurrent.GRUTimeStep`](src/recurrent/gru-time-step.ts) - [Time Step Gated Recurrent Unit or "GRU"](https://en.wikipedia.org/wiki/Gated_recurrent_unit) diff --git a/src/autoencoder.test.ts b/src/autoencoder.test.ts index 7838fe1e..a841ea01 100644 --- a/src/autoencoder.test.ts +++ b/src/autoencoder.test.ts @@ -1,4 +1,4 @@ -import AE from "./autoencoder"; +import AutoencoderGPU from "./autoencoder"; const trainingData = [ [0, 0, 0], @@ -7,9 +7,9 @@ const trainingData = [ [1, 1, 0] ]; -const xornet = new AE( +const xornet = new AutoencoderGPU( { - decodedSize: 3, + inputSize: 3, hiddenLayers: [ 5, 2, 5 ] } ); diff --git a/src/autoencoder.ts b/src/autoencoder.ts index e799b042..bebed4dc 100644 --- a/src/autoencoder.ts +++ b/src/autoencoder.ts @@ -1,41 +1,54 @@ -import { KernelOutput, Texture, TextureArrayOutput } from "gpu.js"; -import { IJSONLayer, INeuralNetworkData, INeuralNetworkDatum, INeuralNetworkTrainOptions } from "./neural-network"; +import { IKernelFunctionThis, KernelOutput, Texture, TextureArrayOutput } from "gpu.js"; +import { IJSONLayer, INeuralNetworkData, INeuralNetworkDatum, INeuralNetworkTrainOptions, NeuralNetworkIO, NeuralNetworkRAM } from "./neural-network"; import { INeuralNetworkGPUOptions, NeuralNetworkGPU } from "./neural-network-gpu"; import { INeuralNetworkState } from "./neural-network-types"; import { UntrainedNeuralNetworkError } from "./errors/untrained-neural-network-error"; -export interface IAEOptions { - binaryThresh: number; - decodedSize: number; - hiddenLayers: number[]; +function loss( + this: IKernelFunctionThis, + actual: number, + expected: number, + inputs: NeuralNetworkIO, + ram: NeuralNetworkRAM +) { + let error = expected - actual; + + // if ( o ≈ i0 ) then return 10% of the loss value. + // Otherwise, return 1000% of the full loss value. + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + if (Math.round(actual) !== Math.round(inputs[this.thread.x])) error *= 32; + else error *= 0.03125; + + return error; } /** * An autoencoder learns to compress input data down to relevant features and reconstruct input data from its compressed representation. */ -export class AE { +export class AutoencoderGPU extends NeuralNetworkGPU { private decoder?: NeuralNetworkGPU; - private denoiser: NeuralNetworkGPU; constructor ( - options?: Partial + options?: Partial ) { // Create default options for the autoencoder. options ??= {}; - // Create default options for the autoencoder's denoiser subnet. - const denoiserOptions: Partial = {}; - // Inherit the binary threshold of the parent autoencoder. - denoiserOptions.binaryThresh = options.binaryThresh; + options.binaryThresh = options.binaryThresh; // Inherit the hidden layers of the parent autoencoder. - denoiserOptions.hiddenLayers = options.hiddenLayers; + options.hiddenLayers = options.hiddenLayers; + + const decodedSize = options.inputSize ?? options.outputSize ?? 1; // Define the denoiser subnet's input and output sizes. - if (options.decodedSize) denoiserOptions.inputSize = denoiserOptions.outputSize = options.decodedSize; + if (decodedSize) options.inputSize = options.outputSize = decodedSize; + + options.loss ??= loss; - // Create the denoiser subnet of the autoencoder. - this.denoiser = new NeuralNetworkGPU(options); + // Create the autoencoder. + super(options); } /** @@ -51,7 +64,7 @@ export class AE} options * @returns {INeuralNetworkState} */ - train(data: DecodedData[], options?: Partial): INeuralNetworkState { + train(data: Partial[] | INeuralNetworkDatum, Partial>[], options?: Partial): INeuralNetworkState { const preprocessedData: INeuralNetworkDatum, Partial>[] = []; + if (data.length && data.length > 0) for (let datum of data) { - preprocessedData.push( { input: datum, output: datum } ); + preprocessedData.push( { input: datum as Partial, output: datum as Partial } ); } - const results = this.denoiser.train(preprocessedData, options); + const results = super.train(preprocessedData, options); this.decoder = this.createDecoder(); @@ -151,12 +165,12 @@ export class AE} */ private createDecoder() { - const json = this.denoiser.toJSON(); + const json = this.toJSON(); const layers: IJSONLayer[] = []; const sizes: number[] = []; - for (let i = this.encodedLayerIndex; i < this.denoiser.sizes.length; i++) { + for (let i = this.encodedLayerIndex; i < this.sizes.length; i++) { layers.push(json.layers[i]); sizes.push(json.sizes[i]); } @@ -175,15 +189,15 @@ export class AE number; export type RAMFunction = ( @@ -108,7 +108,7 @@ function loss( actual: number, expected: number, inputs: NeuralNetworkIO, - memory: NeuralNetworkRAM + ram: NeuralNetworkRAM ) { return expected - actual; }