diff --git a/README.md b/README.md index 2d07e04c..12e75e48 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ GPU accelerated Neural networks in JavaScript for Browsers and Node.js - [For training with NeuralNetwork](#for-training-with-neuralnetwork) - [For training with `RNNTimeStep`, `LSTMTimeStep` and `GRUTimeStep`](#for-training-with-rnntimestep-lstmtimestep-and-grutimestep) - [For training with `RNN`, `LSTM` and `GRU`](#for-training-with-rnn-lstm-and-gru) - - [For training with `AE`](#for-training-with-ae) + - [For training with `AutoencoderGPU`](#for-training-with-ae) - [Training Options](#training-options) - [Async Training](#async-training) - [Cross Validation](#cross-validation) @@ -318,7 +318,7 @@ net.train([ const output = net.run('I feel great about the world!'); // 'happy' ``` -#### For training with `AE` +#### For training with `AutoencoderGPU` Each training pattern can either: @@ -328,7 +328,7 @@ Each training pattern can either: Training an autoencoder to compress the values of a XOR calculation: ```javascript -const net = new brain.AE( +const net = new brain.AutoencoderGPU( { hiddenLayers: [ 5, 2, 5 ] } @@ -362,8 +362,8 @@ const data = net.denoise(noisyData); Test for anomalies in data samples: ```javascript -const shouldBeFalse = net.includesAnomalies([0, 1, 1]); -const shouldBeTrue = net.includesAnomalies([0, 1, 0]); +const shouldBeFalse = net.likelyIncludesAnomalies([0, 1, 1]); +const shouldBeTrue = net.likelyIncludesAnomalies([0, 1, 0]); ``` ### Training Options @@ -644,7 +644,7 @@ The user interface used: - [`brain.NeuralNetwork`](src/neural-network.ts) - [Feedforward Neural Network](https://en.wikipedia.org/wiki/Feedforward_neural_network) with backpropagation - [`brain.NeuralNetworkGPU`](src/neural-network-gpu.ts) - [Feedforward Neural Network](https://en.wikipedia.org/wiki/Feedforward_neural_network) with backpropagation, GPU version -- [`brain.AE`](src/autoencoder.ts) - [Autoencoder or "AE"](https://en.wikipedia.org/wiki/Autoencoder) with backpropogation and GPU support +- [`brain.AutoencoderGPU`](src/autoencoder.ts) - [Autoencoder or "AutoencoderGPU"](https://en.wikipedia.org/wiki/Autoencoder) with backpropogation and GPU support - [`brain.recurrent.RNNTimeStep`](src/recurrent/rnn-time-step.ts) - [Time Step Recurrent Neural Network or "RNN"](https://en.wikipedia.org/wiki/Recurrent_neural_network) - [`brain.recurrent.LSTMTimeStep`](src/recurrent/lstm-time-step.ts) - [Time Step Long Short Term Memory Neural Network or "LSTM"](https://en.wikipedia.org/wiki/Long_short-term_memory) - [`brain.recurrent.GRUTimeStep`](src/recurrent/gru-time-step.ts) - [Time Step Gated Recurrent Unit or "GRU"](https://en.wikipedia.org/wiki/Gated_recurrent_unit) diff --git a/src/autoencoder-gpu.test.ts b/src/autoencoder-gpu.test.ts new file mode 100644 index 00000000..8f5073bd --- /dev/null +++ b/src/autoencoder-gpu.test.ts @@ -0,0 +1,76 @@ +import AutoencoderGPU from './autoencoder-gpu'; +import { INeuralNetworkTrainOptions } from './neural-network'; + +const trainingData = [ + [0, 0, 0], + [0, 1, 1], + [1, 0, 1], + [1, 1, 0], +]; + +const xornet = new AutoencoderGPU({ + inputSize: 3, + hiddenLayers: [4, 2, 4], + outputSize: 3, +}); + +const errorThresh = 0.0011; + +const trainOptions: Partial = { + errorThresh, + iterations: 250000, + learningRate: 0.1, + log: (details) => console.log(details), + // logPeriod: 500, + logPeriod: 500, +}; + +const result = xornet.train(trainingData, trainOptions); + +test('denoise a data sample', async () => { + expect(result.error).toBeLessThanOrEqual(errorThresh); + + function xor(...args: number[]) { + return Math.round(xornet.denoise(args)[2]); + } + + const run1 = xor(0, 0, 0); + const run2 = xor(0, 1, 1); + const run3 = xor(1, 0, 1); + const run4 = xor(1, 1, 0); + + expect(run1).toBe(0); + expect(run2).toBe(1); + expect(run3).toBe(1); + expect(run4).toBe(0); +}); + +test('encode and decode a data sample', async () => { + expect(result.error).toBeLessThanOrEqual(errorThresh); + + const run1$input = [0, 0, 0]; + const run1$encoded = xornet.encode(run1$input); + const run1$decoded = xornet.decode(run1$encoded); + + const run2$input = [0, 1, 1]; + const run2$encoded = xornet.encode(run2$input); + const run2$decoded = xornet.decode(run2$encoded); + + for (let i = 0; i < 3; i++) + expect(Math.round(run1$decoded[i])).toBe(run1$input[i]); + for (let i = 0; i < 3; i++) + expect(Math.round(run2$decoded[i])).toBe(run2$input[i]); +}); + +test('test a data sample for anomalies', async () => { + expect(result.error).toBeLessThanOrEqual(errorThresh); + + function likelyIncludesAnomalies(...args: number[]) { + expect(xornet.likelyIncludesAnomalies(args, 0.5)).toBe(false); + } + + likelyIncludesAnomalies(0, 0, 0); + likelyIncludesAnomalies(0, 1, 1); + likelyIncludesAnomalies(1, 0, 1); + likelyIncludesAnomalies(1, 1, 0); +}); diff --git a/src/autoencoder-gpu.ts b/src/autoencoder-gpu.ts new file mode 100644 index 00000000..ee196acb --- /dev/null +++ b/src/autoencoder-gpu.ts @@ -0,0 +1,235 @@ +import { + IKernelFunctionThis, + KernelOutput, + Texture, + TextureArrayOutput, +} from 'gpu.js'; +import { + IJSONLayer, + INeuralNetworkData, + INeuralNetworkDatum, + INeuralNetworkTrainOptions, + NeuralNetworkIO, + NeuralNetworkRAM, +} from './neural-network'; +import { + INeuralNetworkGPUOptions, + NeuralNetworkGPU, +} from './neural-network-gpu'; +import { INeuralNetworkState } from './neural-network-types'; +import { UntrainedNeuralNetworkError } from './errors/untrained-neural-network-error'; +import { DEFAULT_ANOMALY_THRESHOLD } from './autoencoder'; + +function loss( + this: IKernelFunctionThis, + actual: number, + expected: number, + inputs: NeuralNetworkIO, + ram: NeuralNetworkRAM +) { + let error = expected - actual; + + // if ( o ≈ i0 ) then return 3.125% of the loss value. + // Otherwise, return 3200% of the full loss value. + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-expect-error + if (Math.round(actual) !== Math.round(inputs[this.thread.x])) error *= 32; + else error *= 0.03125; + + return error; +} + +/** + * An autoencoder learns to compress input data down to relevant features and reconstruct input data from its compressed representation. + */ +export class AutoencoderGPU< + DecodedData extends INeuralNetworkData, + EncodedData extends INeuralNetworkData +> extends NeuralNetworkGPU { + private decoder?: NeuralNetworkGPU; + + constructor(options?: Partial) { + // Create default options for the autoencoder. + options ??= {}; + + const decodedSize = options.inputSize ?? options.outputSize ?? 1; + + // Define the denoiser subnet's input and output sizes. + options.inputSize = options.outputSize = decodedSize; + + options.hiddenLayers ??= [Math.round(decodedSize * 0.66)]; + + options.loss ??= loss; + + // Create the autoencoder. + super(options); + } + + /** + * Denoise input data, removing any anomalies from the data. + * @param {DecodedData} input + * @returns {DecodedData} + */ + denoise(input: DecodedData): DecodedData { + // Run the input through the generic denoiser. + // This isn't the best denoiser implementation, but it's efficient. + // Efficiency is important here because training should focus on + // optimizing for feature extraction as quickly as possible rather than + // denoising and anomaly detection; there are other specialized topologies + // better suited for these tasks anyways, many of which can be implemented + // by using an autoencoder. + return this.run(input); + } + + /** + * Decode `EncodedData` into an approximation of its original form. + * + * @param {EncodedData} input + * @returns {DecodedData} + */ + decode(input: EncodedData): DecodedData { + // If the decoder has not been trained yet, throw an error. + if (!this.decoder) throw new UntrainedNeuralNetworkError(this); + + // Decode the encoded input. + return this.decoder.run(input); + } + + /** + * Encode data to extract features, reduce dimensionality, etc. + * + * @param {DecodedData} input + * @returns {EncodedData} + */ + encode(input: DecodedData): EncodedData { + // If the decoder has not been trained yet, throw an error. + if (!this) throw new UntrainedNeuralNetworkError(this); + + // Process the input. + this.run(input); + + // Get the auto-encoded input. + let encodedInput: TextureArrayOutput = this + .encodedLayer as TextureArrayOutput; + + // If the encoded input is a `Texture`, convert it into an `Array`. + if (encodedInput instanceof Texture) encodedInput = encodedInput.toArray(); + else encodedInput = encodedInput.slice(0); + + // Return the encoded input. + return encodedInput as EncodedData; + } + + /** + * Test whether or not a data sample likely contains anomalies. + * If anomalies are likely present in the sample, returns `true`. + * Otherwise, returns `false`. + * + * @param {DecodedData} input + * @returns {boolean} + */ + likelyIncludesAnomalies( + input: DecodedData, + anomalyThreshold: number + ): boolean { + anomalyThreshold ??= DEFAULT_ANOMALY_THRESHOLD; + // Create the anomaly vector. + const anomalies: number[] = []; + + // Attempt to denoise the input. + const denoised = this.denoise(input); + + // Calculate the anomaly vector. + for (let i = 0; i < (input.length ?? 0); i++) { + anomalies[i] = Math.abs( + (input as number[])[i] - (denoised as number[])[i] + ); + } + + // Calculate the sum of all anomalies within the vector. + const sum = anomalies.reduce( + (previousValue, value) => previousValue + value + ); + + // Calculate the mean anomaly. + const mean = sum / (input as number[]).length; + + // Return whether or not the mean anomaly rate is greater than the anomaly threshold. + return mean > anomalyThreshold; + } + + /** + * Train the auto encoder. + * + * @param {DecodedData[]} data + * @param {Partial} options + * @returns {INeuralNetworkState} + */ + train( + data: + | Array> + | Array, Partial>>, + options?: Partial + ): INeuralNetworkState { + const preprocessedData: Array, + Partial + >> = []; + + if (data.length && data.length > 0) + for (const datum of data) { + preprocessedData.push({ + input: datum as Partial, + output: datum as Partial, + }); + } + + const results = super.train(preprocessedData, options); + + this.decoder = this.createDecoder(); + + return results; + } + + /** + * Create a new decoder from the trained denoiser. + * + * @returns {NeuralNetworkGPU} + */ + private createDecoder() { + const json = this.toJSON(); + + const layers: IJSONLayer[] = []; + const sizes: number[] = []; + + for (let i = this.encodedLayerIndex; i < this.sizes.length; i++) { + layers.push(json.layers[i]); + sizes.push(json.sizes[i]); + } + + json.layers = layers; + json.sizes = sizes; + + json.options.inputSize = json.sizes[0]; + + const decoder = new NeuralNetworkGPU().fromJSON(json); + + return (decoder as unknown) as NeuralNetworkGPU; + } + + /** + * Get the layer containing the encoded representation. + */ + private get encodedLayer(): KernelOutput { + return this.outputs[this.encodedLayerIndex]; + } + + /** + * Get the offset of the encoded layer. + */ + private get encodedLayerIndex(): number { + return Math.round(this.outputs.length * 0.5) - 1; + } +} + +export default AutoencoderGPU; diff --git a/src/autoencoder.test.ts b/src/autoencoder.test.ts index 166903b6..dcd1ec54 100644 --- a/src/autoencoder.test.ts +++ b/src/autoencoder.test.ts @@ -1,23 +1,32 @@ -import AE from './autoencoder'; +import Autoencoder from './autoencoder'; +import { INeuralNetworkTrainOptions } from './neural-network'; const trainingData = [ [0, 0, 0], [0, 1, 1], [1, 0, 1], [1, 1, 0], + [1, 1, 0], ]; -const xornet = new AE({ - decodedSize: 3, - hiddenLayers: [5, 2, 5], +const xornet = new Autoencoder({ + inputSize: 3, + hiddenLayers: [4, 2, 4], + outputSize: 3, }); -const errorThresh = 0.011; +const errorThresh = 0.0011; -const result = xornet.train(trainingData, { - iterations: 100000, +const trainOptions: Partial = { errorThresh, -}); + iterations: 250000, + learningRate: 0.1, + log: (details) => console.log(details), + // logPeriod: 500, + logPeriod: 500, +}; + +const result = xornet.train(trainingData, trainOptions); test('denoise a data sample', async () => { expect(result.error).toBeLessThanOrEqual(errorThresh); @@ -57,12 +66,12 @@ test('encode and decode a data sample', async () => { test('test a data sample for anomalies', async () => { expect(result.error).toBeLessThanOrEqual(errorThresh); - function includesAnomalies(...args: number[]) { - expect(xornet.likelyIncludesAnomalies(args)).toBe(false); + function likelyIncludesAnomalies(...args: number[]) { + expect(xornet.likelyIncludesAnomalies(args, 0.5)).toBe(false); } - includesAnomalies(0, 0, 0); - includesAnomalies(0, 1, 1); - includesAnomalies(1, 0, 1); - includesAnomalies(1, 1, 0); + likelyIncludesAnomalies(0, 0, 0); + likelyIncludesAnomalies(0, 1, 1); + likelyIncludesAnomalies(1, 0, 1); + likelyIncludesAnomalies(1, 1, 0); }); diff --git a/src/autoencoder.ts b/src/autoencoder.ts index 357d7fcd..6b921350 100644 --- a/src/autoencoder.ts +++ b/src/autoencoder.ts @@ -1,52 +1,82 @@ -import { KernelOutput, Texture, TextureArrayOutput } from 'gpu.js'; +import { + IKernelFunctionThis, + KernelOutput, + Texture, + TextureArrayOutput, +} from 'gpu.js'; import { IJSONLayer, INeuralNetworkData, INeuralNetworkDatum, INeuralNetworkTrainOptions, + NeuralNetworkIO, + NeuralNetworkRAM, + INeuralNetworkOptions, + NeuralNetwork, } from './neural-network'; -import { - INeuralNetworkGPUOptions, - NeuralNetworkGPU, -} from './neural-network-gpu'; + import { INeuralNetworkState } from './neural-network-types'; import { UntrainedNeuralNetworkError } from './errors/untrained-neural-network-error'; -export interface IAEOptions { - binaryThresh: number; - decodedSize: number; - hiddenLayers: number[]; +export const DEFAULT_ANOMALY_THRESHOLD = 0.5; + +function loss( + this: IKernelFunctionThis, + actual: number, + expected: number, + inputs: NeuralNetworkIO, + ram: NeuralNetworkRAM +) { + let error = expected - actual; + + // if ( o ≈ i0 ) then return 3.125% of the loss value. + // Otherwise, return 3200% of the full loss value. + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-expect-error + if (Math.round(actual) !== Math.round(inputs[this.thread.x])) error *= 32; + else error *= 0.03125; + + return error; } /** * An autoencoder learns to compress input data down to relevant features and reconstruct input data from its compressed representation. */ -export class AE< +export class Autoencoder< DecodedData extends INeuralNetworkData, EncodedData extends INeuralNetworkData -> { - private decoder?: NeuralNetworkGPU; - private readonly denoiser: NeuralNetworkGPU; +> extends NeuralNetwork { + private decoder?: NeuralNetwork; - constructor(options?: Partial) { + constructor(options?: Partial) { // Create default options for the autoencoder. options ??= {}; - // Create default options for the autoencoder's denoiser subnet. - const denoiserOptions: Partial = {}; - - // Inherit the binary threshold of the parent autoencoder. - denoiserOptions.binaryThresh = options.binaryThresh; - // Inherit the hidden layers of the parent autoencoder. - denoiserOptions.hiddenLayers = options.hiddenLayers; + const decodedSize = options.inputSize ?? options.outputSize ?? 1; // Define the denoiser subnet's input and output sizes. - if (options.decodedSize) - denoiserOptions.inputSize = denoiserOptions.outputSize = - options.decodedSize; + options.inputSize = options.outputSize = decodedSize; + + options.hiddenLayers ??= [Math.round(decodedSize * 0.66)]; + + options.loss ??= loss; + + // Create the autoencoder. + super(options); + } + + /** + * Get the layer containing the encoded representation. + */ + private get encodedLayer(): KernelOutput { + return this.outputs[this.encodedLayerIndex]; + } - // Create the denoiser subnet of the autoencoder. - this.denoiser = new NeuralNetworkGPU(options); + /** + * Get the offset of the encoded layer. + */ + private get encodedLayerIndex(): number { + return Math.round(this.outputs.length * 0.5) - 1; } /** @@ -62,7 +92,7 @@ export class AE< // denoising and anomaly detection; there are other specialized topologies // better suited for these tasks anyways, many of which can be implemented // by using an autoencoder. - return this.denoiser.run(input); + return this.run(input); } /** @@ -87,10 +117,10 @@ export class AE< */ encode(input: DecodedData): EncodedData { // If the decoder has not been trained yet, throw an error. - if (!this.denoiser) throw new UntrainedNeuralNetworkError(this); + if (!this) throw new UntrainedNeuralNetworkError(this); // Process the input. - this.denoiser.run(input); + this.run(input); // Get the auto-encoded input. let encodedInput: TextureArrayOutput = this @@ -112,7 +142,12 @@ export class AE< * @param {DecodedData} input * @returns {boolean} */ - likelyIncludesAnomalies(input: DecodedData, anomalyThreshold = 0.2): boolean { + likelyIncludesAnomalies( + input: DecodedData, + anomalyThreshold: number + ): boolean { + anomalyThreshold ??= DEFAULT_ANOMALY_THRESHOLD; + // Create the anomaly vector. const anomalies: number[] = []; @@ -124,6 +159,9 @@ export class AE< anomalies[i] = Math.abs( (input as number[])[i] - (denoised as number[])[i] ); + anomalies[i] = Math.abs( + (input as number[])[i] - (denoised as number[])[i] + ); } // Calculate the sum of all anomalies within the vector. @@ -146,7 +184,9 @@ export class AE< * @returns {INeuralNetworkState} */ train( - data: DecodedData[], + data: + | Array> + | Array, Partial>>, options?: Partial ): INeuralNetworkState { const preprocessedData: Array >> = []; - for (const datum of data) { - preprocessedData.push({ input: datum, output: datum }); - } + if (data.length && data.length > 0) + for (const datum of data) { + preprocessedData.push({ + input: datum as Partial, + output: datum as Partial, + }); + } - const results = this.denoiser.train(preprocessedData, options); + const results = super.train(preprocessedData, options); this.decoder = this.createDecoder(); @@ -168,15 +212,15 @@ export class AE< /** * Create a new decoder from the trained denoiser. * - * @returns {NeuralNetworkGPU} + * @returns {NeuralNetwork} */ - private createDecoder() { - const json = this.denoiser.toJSON(); + private createDecoder(): NeuralNetwork { + const json = this.toJSON(); const layers: IJSONLayer[] = []; const sizes: number[] = []; - for (let i = this.encodedLayerIndex; i < this.denoiser.sizes.length; i++) { + for (let i = this.encodedLayerIndex; i < this.sizes.length; i++) { layers.push(json.layers[i]); sizes.push(json.sizes[i]); } @@ -186,24 +230,10 @@ export class AE< json.options.inputSize = json.sizes[0]; - const decoder = new NeuralNetworkGPU().fromJSON(json); + const decoder = new NeuralNetwork().fromJSON(json); - return (decoder as unknown) as NeuralNetworkGPU; - } - - /** - * Get the layer containing the encoded representation. - */ - private get encodedLayer(): KernelOutput { - return this.denoiser.outputs[this.encodedLayerIndex]; - } - - /** - * Get the offset of the encoded layer. - */ - private get encodedLayerIndex(): number { - return Math.round(this.denoiser.outputs.length * 0.5) - 1; + return (decoder as unknown) as NeuralNetwork; } } -export default AE; +export default Autoencoder; diff --git a/src/errors/untrained-neural-network-error.ts b/src/errors/untrained-neural-network-error.ts index f37984bf..b27af312 100644 --- a/src/errors/untrained-neural-network-error.ts +++ b/src/errors/untrained-neural-network-error.ts @@ -1,7 +1,13 @@ -export class UntrainedNeuralNetworkError< - T extends { constructor: { name: string } } -> extends Error { - constructor(neuralNetwork: T) { +interface IErrorableNeuralNetworkConstructor { + name: string; +} + +interface IErrorableNeuralNetwork { + constructor: IErrorableNeuralNetworkConstructor; +} + +export class UntrainedNeuralNetworkError extends Error { + constructor(neuralNetwork: IErrorableNeuralNetwork) { super( `Cannot run a ${neuralNetwork.constructor.name} before it is trained.` ); diff --git a/src/index.ts b/src/index.ts index 1d410f76..a6617532 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,5 +1,6 @@ import * as activation from './activation'; -import { AE } from './autoencoder'; +import { Autoencoder } from './autoencoder'; +import { AutoencoderGPU } from './autoencoder-gpu'; import CrossValidate from './cross-validate'; import { FeedForward } from './feed-forward'; import * as layer from './layer'; @@ -54,7 +55,8 @@ const utilities = { export { activation, - AE, + Autoencoder, + AutoencoderGPU, CrossValidate, likely, layer, diff --git a/src/neural-network-gpu.ts b/src/neural-network-gpu.ts index 01bd6ea9..4c237918 100644 --- a/src/neural-network-gpu.ts +++ b/src/neural-network-gpu.ts @@ -4,6 +4,7 @@ import { GPUFunction, IKernelFunctionThis, IKernelMapRunShortcut, + IKernelRunShortcut, IMappedKernelResult, KernelOutput, Texture, @@ -20,6 +21,10 @@ import { INeuralNetworkPreppedTrainingData, INeuralNetworkTrainOptions, NeuralNetwork, + LossFunction, + NeuralNetworkIO, + RAMFunction, + NeuralNetworkRAM, } from './neural-network'; import { release } from './utilities/kernel'; @@ -96,8 +101,8 @@ function weightedSumTanh( return Math.tanh(sum); } -function calcErrorOutput(output: number, target: number): number { - return target - output; +function calcErrorOutput(value: number): number { + return value; } function calcDeltasSigmoid(error: number, output: number): number { @@ -180,7 +185,9 @@ export interface INeuralNetworkGPUOptions extends INeuralNetworkOptions { export type BackPropagateOutput = ( this: IKernelFunctionThis, outputs: KernelOutput, - targets: KernelOutput + targets: KernelOutput, + inputs: NeuralNetworkIO, + ram: NeuralNetworkRAM ) => { result: KernelOutput; error: KernelOutput }; export type BackPropagateLayer = ( @@ -260,10 +267,47 @@ export class NeuralNetworkGPU< // @ts-expect-error biases: KernelOutput[] = []; + _ramKernel?: IKernelRunShortcut; + constructor(options: Partial = {}) { super(options); this.errorCheckInterval = 100; this.gpu = new GPU({ mode: options.mode }); + // Compile the accelerated learning functions. + this.lossFunction = this._lossFunction; + this.ramFunction = this._ramFunction; + } + + public get lossFunction(): LossFunction { + return super.lossFunction; + } + + public set lossFunction(value: LossFunction) { + this.gpu.addFunction(value); + super.lossFunction = value; + } + + public get ramFunction(): RAMFunction | undefined { + return super.ramFunction; + } + + public set ramFunction(value: RAMFunction | undefined) { + if (!value) { + if (this._ramKernel) delete this._ramKernel; + } else { + const layerCount = this.sizes.length; + const maxNeuronsPerLayer = this.sizes.reduce((eax, edx) => + edx > eax ? edx : eax + ); + const ramSize = this.ramSize; + this._ramKernel = this.gpu.createKernel(value, { + constants: { + ramSize, + }, + output: [layerCount, maxNeuronsPerLayer, ramSize], + }); + } + super.ramFunction = value; } initialize(): void { @@ -376,6 +420,23 @@ export class NeuralNetworkGPU< ); output = input = this.outputs[layer]; } + const updateRAM: IKernelRunShortcut | undefined = this._ramKernel; + if (updateRAM) { + const input = this.outputs[0]; + const output = this.outputs[this.outputLayer]; + const loss = this.loss.current.mean; + const deltaLoss = loss - this.loss.previous.mean; + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-expect-error + this._ram = updateRAM( + this.ram, + input, + output, + this.sizes, + loss, + deltaLoss + ); + } return output; }; @@ -400,11 +461,14 @@ export class NeuralNetworkGPU< ); } + const loss: LossFunction = this.lossFunction; + calcDeltas = alias( utils.getMinifySafeName(() => calcDeltas), calcDeltas ); this.gpu.addFunction(calcDeltas); + this.gpu.addFunction(loss); for (let layer = this.outputLayer; layer > 0; layer--) { if (layer === this.outputLayer) { // eslint-disable-next-line @typescript-eslint/ban-ts-comment @@ -416,13 +480,20 @@ export class NeuralNetworkGPU< function ( this: IKernelFunctionThis, outputs: number[], - targets: number[] + targets: number[], + inputs: NeuralNetworkIO, + ram: NeuralNetworkRAM ): number { const output = outputs[this.thread.x]; const target = targets[this.thread.x]; // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error - return calcDeltas(calcErrorOutput(output, target), output); + return calcDeltas( + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-expect-error + calcErrorOutput(loss(output, target, inputs, ram)), + output + ); }, { output: [this.sizes[this.outputLayer]], @@ -478,7 +549,12 @@ export class NeuralNetworkGPU< if (layer === this.outputLayer) { // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error - output = this.backwardPropagate[layer](this.outputs[layer], target); + output = this.backwardPropagate[layer]( + this.outputs[layer], + target, + this.outputs[0], + this.ram + ); } else { // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error @@ -704,12 +780,17 @@ export class NeuralNetworkGPU< : (layerBiases as Float32Array) ) ); + const jsonLayerRAM = this.ram.map((layerMemory, layerIndex) => + layerMemory.map((nodeRAM) => Array.from(nodeRAM)) + ); const jsonLayers: IJSONLayer[] = []; for (let i = 0; i <= this.outputLayer; i++) { - jsonLayers.push({ + const jsonLayer: IJSONLayer = { weights: jsonLayerWeights[i] ?? [], biases: jsonLayerBiases[i] ?? [], - }); + ram: jsonLayerRAM[i] ?? [], + }; + jsonLayers.push(jsonLayer); } return { type: 'NeuralNetworkGPU', @@ -721,6 +802,7 @@ export class NeuralNetworkGPU< outputLookupLength: this.outputLookupLength, options: { ...this.options }, trainOpts: this.getTrainOptsJSON(), + ramSize: this.ramSize, }; } } diff --git a/src/neural-network.ts b/src/neural-network.ts index 5e851d40..b1e75fd0 100644 --- a/src/neural-network.ts +++ b/src/neural-network.ts @@ -12,6 +12,104 @@ import { max } from './utilities/max'; import { mse } from './utilities/mse'; import { randos } from './utilities/randos'; import { zeros } from './utilities/zeros'; +import { IKernelFunctionThis } from 'gpu.js'; + +/** + * An input or output layer of a neural network. + * This data type exists to allow kernel functions to operate on individual layers. + * This functionality is essential to custom loss functions. + */ +export type NeuralNetworkIO = number[] | number[][] | number[][][]; + +/** + * A read-write state matrix designed to hold metadata for use by `loss` functions. + * This data is read-only to the `loss` function. + * To the `updateRAM` kernel function, this data is read-write. + * To the neural network consumer, + * the `ram` property is made public to allow for modifications to be made in addition to reading. + */ +export type NeuralNetworkRAM = Float32Array[][]; + +/** + * A loss function determines how fit a neural network currently is. + * The higher the value returned by this function, + * the less accurate the network is. + * The lower the value returned by this function is, + * the more accurate the network is. + * + * Here, `ram` is read-only. + */ +export type LossFunction = ( + this: IKernelFunctionThis, + actual: number, + expected: number, + inputs: NeuralNetworkIO, + ram: NeuralNetworkRAM +) => number; + +/** + * A RAM function updates the RAM matrix of the neural network. + * + * Here, `ram` is read-write. + * The actual matrix passed to the function is read-only. + * However, the return value of the function directly corresponds to a value within the RAM matrix. + */ +export type RAMFunction = ( + this: IKernelFunctionThis, + ram: NeuralNetworkRAM, + inputs: NeuralNetworkIO, + outputs: NeuralNetworkIO, + sizes: number[], + loss: number, + lossDelta: number +) => number; + +/** + * Each time the loss is calculated, + * a snapshot is taken of various loss analytics. + */ +export interface ILossAnalyticsSnapshot { + mean: number; + median: number; + total: number; +} + +const EMPTY_LOSS_SNAPSHOT: ILossAnalyticsSnapshot = { + mean: Number.MAX_SAFE_INTEGER, + median: Number.MAX_SAFE_INTEGER, + total: Number.MAX_SAFE_INTEGER, +}; + +Object.freeze(EMPTY_LOSS_SNAPSHOT); + +function createLossAnalyticsSnapshot(): ILossAnalyticsSnapshot { + return JSON.parse(JSON.stringify(EMPTY_LOSS_SNAPSHOT)); +} + +/** + * A collection of analytics pertaining to the results of the loss function. + */ +export interface ILossAnalytics { + current: ILossAnalyticsSnapshot; + max: ILossAnalyticsSnapshot; + min: ILossAnalyticsSnapshot; + previous: ILossAnalyticsSnapshot; + projected: ILossAnalyticsSnapshot; +} + +const EMPTY_LOSS: ILossAnalytics = { + current: createLossAnalyticsSnapshot(), + max: createLossAnalyticsSnapshot(), + min: createLossAnalyticsSnapshot(), + previous: createLossAnalyticsSnapshot(), + projected: createLossAnalyticsSnapshot(), +}; + +Object.freeze(EMPTY_LOSS); + +export function createLossAnalytics(): ILossAnalytics { + return JSON.parse(JSON.stringify(EMPTY_LOSS)); +} type NeuralNetworkFormatter = | ((v: INumberHash) => Float32Array) @@ -40,6 +138,16 @@ export function getTypedArrayFn( }; } +function loss( + this: IKernelFunctionThis, + actual: number, + expected: number, + inputs: NeuralNetworkIO, + ram: NeuralNetworkRAM +): number { + return expected - actual; +} + export type NeuralNetworkActivation = | 'sigmoid' | 'relu' @@ -49,6 +157,7 @@ export type NeuralNetworkActivation = export interface IJSONLayer { biases: number[]; weights: number[][]; + ram: number[][]; } export interface INeuralNetworkJSON { @@ -61,6 +170,7 @@ export interface INeuralNetworkJSON { outputLookupLength: number; options: INeuralNetworkOptions; trainOpts: INeuralNetworkTrainOptionsJSON; + ramSize: number; } export interface INeuralNetworkOptions { @@ -68,6 +178,8 @@ export interface INeuralNetworkOptions { outputSize: number; binaryThresh: number; hiddenLayers?: number[]; + loss: LossFunction; + ramSize: number; } export function defaults(): INeuralNetworkOptions { @@ -75,6 +187,8 @@ export function defaults(): INeuralNetworkOptions { inputSize: 0, outputSize: 0, binaryThresh: 0.5, + loss, + ramSize: 1, }; } @@ -107,6 +221,8 @@ export interface INeuralNetworkTrainOptions { errorThresh: number; log: boolean | ((status: INeuralNetworkState) => void); logPeriod: number; + loss?: LossFunction; + updateRAM?: RAMFunction; leakyReluAlpha: number; learningRate: number; momentum: number; @@ -126,6 +242,7 @@ export function trainDefaults(): INeuralNetworkTrainOptions { errorThresh: 0.005, // the acceptable error percentage from training data log: false, // true to use console.log, when a function is supplied it is used logPeriod: 10, // iterations between logging out + loss, leakyReluAlpha: 0.01, learningRate: 0.3, // multiply's against the input and the delta then adds to momentum momentum: 0.1, // multiply's against the specified "change" then adds to learning rate for change @@ -176,18 +293,28 @@ export class NeuralNetwork< _formatInput: NeuralNetworkFormatter | null = null; _formatOutput: NeuralNetworkFormatter | null = null; + _lossAnalytics: ILossAnalytics = createLossAnalytics(); + _ram: NeuralNetworkRAM = []; + _ramSize = 1; + runInput: (input: Float32Array) => Float32Array = (input: Float32Array) => { this.setActivation(); - return this.runInput(input); + const output = this.runInput(input); + this._updateRAM(); + return output; }; - calculateDeltas: (output: Float32Array) => void = ( - output: Float32Array + calculateDeltas: (output: Float32Array, input: Float32Array) => void = ( + output: Float32Array, + input: Float32Array ): void => { this.setActivation(); - return this.calculateDeltas(output); + return this.calculateDeltas(output, input); }; + _lossFunction: LossFunction = loss; + _ramFunction?: RAMFunction; + // adam biasChangesLow: Float32Array[] = []; biasChangesHigh: Float32Array[] = []; @@ -198,13 +325,28 @@ export class NeuralNetwork< constructor( options: Partial = {} ) { - this.options = { ...this.options, ...options }; + const defaultOptions = defaults(); + this.options.binaryThresh = + options.binaryThresh ?? defaultOptions.binaryThresh; + this.options.hiddenLayers = + options.hiddenLayers ?? defaultOptions.hiddenLayers; + this.options.inputSize = options.inputSize ?? defaultOptions.inputSize; + this.options.loss = options.loss ?? defaultOptions.loss; + this.options.outputSize = options.outputSize ?? defaultOptions.outputSize; + this.options.ramSize = options.ramSize ?? defaultOptions.ramSize; this.updateTrainingOptions(options); const { inputSize, hiddenLayers, outputSize } = this.options; if (inputSize && outputSize) { this.sizes = [inputSize].concat(hiddenLayers ?? []).concat([outputSize]); } + + // Initialize the memory matrix. + if (options.ramSize) this._ramSize = options.ramSize; + this.ram = this.createRAM(this.ramSize); + // Initialize the loss function. + if (options.loss) this._lossFunction = options.loss; + if (options.updateRAM) this._ramFunction = options.updateRAM; } /** @@ -220,6 +362,7 @@ export class NeuralNetwork< this.biases = new Array(this.outputLayer); // weights for bias nodes this.weights = new Array(this.outputLayer); this.outputs = new Array(this.outputLayer); + this.ram = this.createRAM(this.ramSize); // state for training this.deltas = new Array(this.outputLayer); @@ -281,6 +424,39 @@ export class NeuralNetwork< return this.sizes.length > 0; } + public get loss(): ILossAnalytics { + return this._lossAnalytics; + } + + public get lossFunction(): LossFunction { + return typeof this._lossFunction === 'function' ? this._lossFunction : loss; + } + + public set lossFunction(value: LossFunction) { + this._lossFunction = value; + } + + public get ram(): NeuralNetworkRAM { + return this._ram; + } + + public set ram(ram: NeuralNetworkRAM) { + this._ram = ram; + } + + public get ramFunction(): RAMFunction | undefined { + return this._ramFunction; + } + + public set ramFunction(value: RAMFunction | undefined) { + this._ramFunction = value; + } + + public get ramSize(): number { + if (!isFinite(this._ramSize) || this._ramSize < 1) return 1; + return this._ramSize; + } + run(input: Partial): OutputType { if (!this.isRunnable) { throw new Error('network not runnable'); @@ -297,6 +473,7 @@ export class NeuralNetwork< } this.validateInput(formattedInput); const output = this.runInput(formattedInput).slice(0); + this._updateRAM(); if (this.outputLookup) { return (lookup.toObject( this.outputLookup, @@ -306,6 +483,78 @@ export class NeuralNetwork< return (output as unknown) as OutputType; } + protected _updateRAM(): void { + if (this.ram) { + const updateRAM: RAMFunction | undefined = this.ramFunction; + if (updateRAM) { + const input = this.outputs[0]; + const output = this.outputs[this.outputLayer]; + const loss = this.loss.current.mean; + const deltaLoss = loss - this.loss.previous.mean; + this._ram = this.ram.map((layerRAM, layer) => + layerRAM.map((neuronRAM, neuron) => + neuronRAM.map((value, index) => { + return updateRAM.call( + this._getRAMKernelFunctionThis(layer, neuron, index), + this.ram, + (input as unknown) as NeuralNetworkIO, + (output as unknown) as NeuralNetworkIO, + this.sizes, + loss, + deltaLoss + ); + }) + ) + ); + } + } + } + + private _getLossKernelFunctionThis( + layer: number, + neuron: number + ): IKernelFunctionThis { + return { + color: function color(r: number, g = 0, b = 0, a = 0) {}, + constants: { + ramSize: this.ramSize, + }, + output: { + x: this.ram[0][0].length, + y: this.ram[0].length, + z: this.ram.length, + }, + thread: { + x: neuron, + y: layer, + z: 0, + }, + }; + } + + private _getRAMKernelFunctionThis( + layer: number, + neuron: number, + index: number + ): IKernelFunctionThis { + return { + color: function color(r: number, g = 0, b = 0, a = 0) {}, + constants: { + ramSize: this.ramSize, + }, + output: { + x: this.ram[0][0].length, + y: this.ram[0].length, + z: this.ram.length, + }, + thread: { + x: index, + y: neuron, + z: layer, + }, + }; + } + _runInputSigmoid(input: Float32Array): Float32Array { this.outputs[0] = input; // set output state of input layer @@ -470,6 +719,10 @@ export class NeuralNetwork< const val = options.logPeriod; return typeof val === 'number' && val > 0; }, + loss: () => { + const val = options.loss; + return typeof val === 'function' || typeof val === 'boolean'; + }, leakyReluAlpha: () => { const val = options.leakyReluAlpha; return typeof val === 'number' && val > 0 && val < 1; @@ -666,6 +919,12 @@ export class NeuralNetwork< data: Array, Partial>>, options: Partial = {} ): INeuralNetworkState { + let lossFunctionBackup; + if (options.loss) { + lossFunctionBackup = this.lossFunction; + this.lossFunction = options.loss; + } + const { preparedData, status, endTime } = this.prepTraining( data as Array>, options @@ -676,6 +935,9 @@ export class NeuralNetwork< break; } } + + if (lossFunctionBackup) this.lossFunction = lossFunctionBackup; + return status; } @@ -710,9 +972,10 @@ export class NeuralNetwork< ): number | null { // forward propagate this.runInput(value.input); + this._updateRAM(); // back propagate - this.calculateDeltas(value.output); + this.calculateDeltas(value.output, value.input); this.adjustWeights(); if (logErrorRate) { @@ -721,7 +984,7 @@ export class NeuralNetwork< return null; } - _calculateDeltasSigmoid(target: Float32Array): void { + _calculateDeltasSigmoid(target: Float32Array, input: Float32Array): void { for (let layer = this.outputLayer; layer >= 0; layer--) { const activeSize = this.sizes[layer]; const activeOutput = this.outputs[layer]; @@ -734,7 +997,15 @@ export class NeuralNetwork< let error = 0; if (layer === this.outputLayer) { - error = target[node] - output; + if (typeof this._lossFunction === 'function') { + error = this._lossFunction.call( + this._getLossKernelFunctionThis(layer, node), + output, + target[node], + (input as unknown) as NeuralNetworkIO, + this.ram + ); + } else error = target[node] - output; } else { const deltas = this.deltas[layer + 1]; for (let k = 0; k < deltas.length; k++) { @@ -747,7 +1018,7 @@ export class NeuralNetwork< } } - _calculateDeltasRelu(target: Float32Array): void { + _calculateDeltasRelu(target: Float32Array, input: Float32Array): void { for (let layer = this.outputLayer; layer >= 0; layer--) { const currentSize = this.sizes[layer]; const currentOutputs = this.outputs[layer]; @@ -761,7 +1032,15 @@ export class NeuralNetwork< let error = 0; if (layer === this.outputLayer) { - error = target[node] - output; + if (typeof this._lossFunction === 'function') { + error = this._lossFunction.call( + this._getLossKernelFunctionThis(layer, node), + output, + target[node], + (input as unknown) as NeuralNetworkIO, + this.ram + ); + } else error = target[node] - output; } else { for (let k = 0; k < nextDeltas.length; k++) { error += nextDeltas[k] * nextWeights[k][node]; @@ -773,7 +1052,7 @@ export class NeuralNetwork< } } - _calculateDeltasLeakyRelu(target: Float32Array): void { + _calculateDeltasLeakyRelu(target: Float32Array, input: Float32Array): void { const alpha = this.trainOpts.leakyReluAlpha; for (let layer = this.outputLayer; layer >= 0; layer--) { const currentSize = this.sizes[layer]; @@ -788,7 +1067,15 @@ export class NeuralNetwork< let error = 0; if (layer === this.outputLayer) { - error = target[node] - output; + if (typeof this._lossFunction === 'function') { + error = this._lossFunction.call( + this._getLossKernelFunctionThis(layer, node), + output, + target[node], + (input as unknown) as NeuralNetworkIO, + this.ram + ); + } else error = target[node] - output; } else { for (let k = 0; k < nextDeltas.length; k++) { error += nextDeltas[k] * nextWeights[k][node]; @@ -800,7 +1087,7 @@ export class NeuralNetwork< } } - _calculateDeltasTanh(target: Float32Array): void { + _calculateDeltasTanh(target: Float32Array, input: Float32Array): void { for (let layer = this.outputLayer; layer >= 0; layer--) { const currentSize = this.sizes[layer]; const currentOutputs = this.outputs[layer]; @@ -814,7 +1101,15 @@ export class NeuralNetwork< let error = 0; if (layer === this.outputLayer) { - error = target[node] - output; + if (typeof this._lossFunction === 'function') { + error = this._lossFunction.call( + this._getLossKernelFunctionThis(layer, node), + output, + target[node], + (input as unknown) as NeuralNetworkIO, + this.ram + ); + } else error = target[node] - output; } else { for (let k = 0; k < nextDeltas.length; k++) { error += nextDeltas[k] * nextWeights[k][node]; @@ -1088,6 +1383,7 @@ export class NeuralNetwork< for (let i = 0; i < preparedData.length; i++) { const output = this.runInput(preparedData[i].input); + this._updateRAM(); const target = preparedData[i].output; const actual = output[0] > this.options.binaryThresh ? 1 : 0; const expected = target[0]; @@ -1135,6 +1431,7 @@ export class NeuralNetwork< for (let i = 0; i < preparedData.length; i++) { const output = this.runInput(preparedData[i].input); + this._updateRAM(); const target = preparedData[i].output; const actual = output.indexOf(max(output)); const expected = target.indexOf(max(target)); @@ -1173,13 +1470,20 @@ export class NeuralNetwork< const jsonLayerBiases = this.biases.map((layerBiases) => Array.from(layerBiases) ); + const jsonLayerRAM = this.ram?.map((layerMemory) => + layerMemory.map((nodeRAM) => Array.from(nodeRAM)) + ); const jsonLayers: IJSONLayer[] = []; const outputLength = this.sizes.length - 1; + const ramSize = this.ramSize; for (let i = 0; i <= outputLength; i++) { - jsonLayers.push({ + const jsonLayer: IJSONLayer = { weights: jsonLayerWeights[i] ?? [], biases: jsonLayerBiases[i] ?? [], - }); + ram: + jsonLayerRAM[i] ?? new Array(this.sizes[i]).fill(new Array(ramSize)), + }; + jsonLayers.push(jsonLayer); } return { type: 'NeuralNetwork', @@ -1191,6 +1495,7 @@ export class NeuralNetwork< outputLookupLength: this.outputLookupLength, options: { ...this.options }, trainOpts: this.getTrainOptsJSON(), + ramSize: this.ramSize, }; } @@ -1223,9 +1528,19 @@ export class NeuralNetwork< const layerBiases = this.biases.map((layerBiases, layerIndex) => Float32Array.from(jsonLayers[layerIndex].biases) ); + const ramSize = (this._ramSize = json.ramSize); + const layerRAM = isFinite(ramSize) + ? this.ram.map((ram, layerIndex) => + Array.from(jsonLayers[layerIndex].ram).map((nodeRAM) => + Float32Array.from(nodeRAM ?? new Float32Array(ramSize)) + ) + ) + : undefined; + this.ram = this.createRAM(ramSize); for (let i = 0; i <= this.outputLayer; i++) { this.weights[i] = layerWeights[i] || []; this.biases[i] = layerBiases[i] || []; + if (layerRAM) this._ram[i] = layerRAM[i] || new Float32Array(ramSize); } return this; } @@ -1328,4 +1643,19 @@ export class NeuralNetwork< input: Partial ) => OutputType; } + + private createRAM(ramSize: number): NeuralNetworkRAM { + if (!isFinite(ramSize) || ramSize < 0) ramSize = 1; + const ram: NeuralNetworkRAM = []; + for (let layer = 0; layer < this.sizes.length; layer++) { + ram[layer] = []; + for (let neuron = 0; neuron < this.sizes.length; neuron++) { + ram[layer][neuron] = new Float32Array(ramSize).fill(0); + } + } + if (!ram[0]) ram[0] = []; + if (!ram[0][0]) ram[0][0] = new Float32Array(ramSize); + if (!ram[0][0][0]) ram[0][0][0] = 0; + return ram; + } } diff --git a/src/utilities/to-svg.ts b/src/utilities/to-svg.ts index 6e4bb835..20c010fc 100644 --- a/src/utilities/to-svg.ts +++ b/src/utilities/to-svg.ts @@ -467,7 +467,9 @@ export function toSVG< // Get network size array for NeuralNetwork or NeuralNetworkGPU let sizes: number[] = []; if (net instanceof NeuralNetwork || net instanceof NeuralNetworkGPU) { - sizes = getNeuralNetworkSizes(net); + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-expect-error + sizes = getNeuralNetworkSizes((net as unknown) as NeuralNetwork); } // get network size for Recurrent else if (net instanceof Recurrent) { pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy