From 5518d7a9567873c393b3c3a7740468d91d8a01f7 Mon Sep 17 00:00:00 2001 From: JulioJerez Date: Wed, 15 Nov 2023 06:55:05 -0800 Subject: [PATCH] adding dropout as funtion of the layer rather that making it its own layer (wip) it is always a dilemma, because adding funtionality to a layer make it more inflexible to use --- .../toolbox/ndCifar10ImageClassification.cpp | 3 +- .../ndSandbox/toolbox/ndHandWrittenDigits.cpp | 9 +- newton-4.00/sdk/dBrain/ndBrainInc.h | 1 - .../dBrain/ndBrainLayerConvolutional_2d.cpp | 5 + .../sdk/dBrain/ndBrainLayerConvolutional_2d.h | 1 + .../sdk/dBrain/ndBrainLayerDropOut.cpp | 105 ------------------ newton-4.00/sdk/dBrain/ndBrainLayerDropOut.h | 47 -------- .../ndBrainLayerLeakyReluActivation.cpp | 1 - newton-4.00/sdk/dBrain/ndBrainLayerLinear.cpp | 11 ++ newton-4.00/sdk/dBrain/ndBrainLayerLinear.h | 3 + newton-4.00/sdk/dBrain/ndBrainSaveLoad.cpp | 5 - 11 files changed, 24 insertions(+), 167 deletions(-) delete mode 100644 newton-4.00/sdk/dBrain/ndBrainLayerDropOut.cpp delete mode 100644 newton-4.00/sdk/dBrain/ndBrainLayerDropOut.h diff --git a/newton-4.00/applications/ndSandbox/toolbox/ndCifar10ImageClassification.cpp b/newton-4.00/applications/ndSandbox/toolbox/ndCifar10ImageClassification.cpp index a812c0f3e1..ab3b736d50 100644 --- a/newton-4.00/applications/ndSandbox/toolbox/ndCifar10ImageClassification.cpp +++ b/newton-4.00/applications/ndSandbox/toolbox/ndCifar10ImageClassification.cpp @@ -387,7 +387,8 @@ static void Cifar10TrainingSet() const ndBrainLayerConvolutionalMaxPooling_2d* pooling; #if 0 - #define ACTIVATION_TYPE ndBrainLayerReluActivation + //#define ACTIVATION_TYPE ndBrainLayerReluActivation + #define DIGIT_ACTIVATION_TYPE ndBrainLayerLeakyReluActivation #else #define ACTIVATION_TYPE ndBrainLayerTanhActivation #endif diff --git a/newton-4.00/applications/ndSandbox/toolbox/ndHandWrittenDigits.cpp b/newton-4.00/applications/ndSandbox/toolbox/ndHandWrittenDigits.cpp index 7eeb9d7c3f..374ea6360d 100644 --- a/newton-4.00/applications/ndSandbox/toolbox/ndHandWrittenDigits.cpp +++ b/newton-4.00/applications/ndSandbox/toolbox/ndHandWrittenDigits.cpp @@ -273,6 +273,7 @@ static void MnistTrainingSet() } for (ndInt32 epoch = 0; epoch < 1000; ++epoch) + //for (ndInt32 epoch = 0; epoch < 1; ++epoch) { ndInt32 start = 0; ndMemSet(failCount, ndUnsigned32(0), D_MAX_THREADS_COUNT); @@ -410,37 +411,31 @@ static void MnistTrainingSet() layers.PushBack(new ndBrainLayerConvolutional_2d(width, height, 1, 3, 16)); conv = (ndBrainLayerConvolutional_2d*)(layers[layers.GetCount() - 1]); - layers.PushBack(new ndBrainLayerDropOut(layers[layers.GetCount() - 1]->GetOutputSize())); layers.PushBack(new DIGIT_ACTIVATION_TYPE(conv->GetOutputSize())); layers.PushBack(new ndBrainLayerConvolutionalMaxPooling_2d(conv->GetOutputWidth(), conv->GetOutputHeight(), conv->GetOutputChannels())); pooling = (ndBrainLayerConvolutionalMaxPooling_2d*)(layers[layers.GetCount() - 1]); layers.PushBack(new ndBrainLayerConvolutional_2d(pooling->GetOutputWidth(), pooling->GetOutputHeight(), pooling->GetOutputChannels(), 3, 32)); conv = (ndBrainLayerConvolutional_2d*)(layers[layers.GetCount() - 1]); - layers.PushBack(new ndBrainLayerDropOut(layers[layers.GetCount() - 1]->GetOutputSize())); layers.PushBack(new DIGIT_ACTIVATION_TYPE(conv->GetOutputSize())); layers.PushBack(new ndBrainLayerConvolutionalMaxPooling_2d(conv->GetOutputWidth(), conv->GetOutputHeight(), conv->GetOutputChannels())); pooling = (ndBrainLayerConvolutionalMaxPooling_2d*)(layers[layers.GetCount() - 1]); layers.PushBack(new ndBrainLayerConvolutional_2d(pooling->GetOutputWidth(), pooling->GetOutputHeight(), pooling->GetOutputChannels(), 3, 32)); - layers.PushBack(new ndBrainLayerDropOut(layers[layers.GetCount() - 1]->GetOutputSize())); conv = (ndBrainLayerConvolutional_2d*)(layers[layers.GetCount() - 1]); layers.PushBack(new DIGIT_ACTIVATION_TYPE(conv->GetOutputSize())); #else layers.PushBack(new ndBrainLayerLinear(trainingDigits->GetColumns(), neuronsPerLayers)); - layers.PushBack(new ndBrainLayerDropOut(layers[layers.GetCount() - 1]->GetOutputSize())); - layers.PushBack(new ndBrainLayerTanhActivation(layers[layers.GetCount() - 1]->GetOutputSize())); + layers.PushBack(new DIGIT_ACTIVATION_TYPE(layers[layers.GetCount() - 1]->GetOutputSize())); #endif layers.PushBack(new ndBrainLayerLinear(layers[layers.GetCount() - 1]->GetOutputSize(), neuronsPerLayers)); - layers.PushBack(new ndBrainLayerDropOut(layers[layers.GetCount() - 1]->GetOutputSize())); layers.PushBack(new DIGIT_ACTIVATION_TYPE(layers[layers.GetCount() - 1]->GetOutputSize())); layers.PushBack(new ndBrainLayerLinear(layers[layers.GetCount() - 1]->GetOutputSize(), neuronsPerLayers)); - layers.PushBack(new ndBrainLayerDropOut(layers[layers.GetCount() - 1]->GetOutputSize())); layers.PushBack(new DIGIT_ACTIVATION_TYPE(layers[layers.GetCount() - 1]->GetOutputSize())); layers.PushBack(new ndBrainLayerLinear(layers[layers.GetCount() - 1]->GetOutputSize(), trainingLabels->GetColumns())); diff --git a/newton-4.00/sdk/dBrain/ndBrainInc.h b/newton-4.00/sdk/dBrain/ndBrainInc.h index 92698e1419..d8d5acc998 100644 --- a/newton-4.00/sdk/dBrain/ndBrainInc.h +++ b/newton-4.00/sdk/dBrain/ndBrainInc.h @@ -38,7 +38,6 @@ #include #include #include -#include #include #include #include diff --git a/newton-4.00/sdk/dBrain/ndBrainLayerConvolutional_2d.cpp b/newton-4.00/sdk/dBrain/ndBrainLayerConvolutional_2d.cpp index e2b4e1cb48..acc2aec5bf 100644 --- a/newton-4.00/sdk/dBrain/ndBrainLayerConvolutional_2d.cpp +++ b/newton-4.00/sdk/dBrain/ndBrainLayerConvolutional_2d.cpp @@ -231,6 +231,11 @@ void ndBrainLayerConvolutional_2d::AdamUpdate(const ndBrainLayer& u, const ndBra } } +void ndBrainLayerConvolutional_2d::UpdateDropOut() +{ + ndAssert(0); +} + void ndBrainLayerConvolutional_2d::MakePrediction(const ndBrainVector& input, ndBrainVector& output) const { ndAssert(input.GetCount() == GetInputSize()); diff --git a/newton-4.00/sdk/dBrain/ndBrainLayerConvolutional_2d.h b/newton-4.00/sdk/dBrain/ndBrainLayerConvolutional_2d.h index b240939183..7f59a1038d 100644 --- a/newton-4.00/sdk/dBrain/ndBrainLayerConvolutional_2d.h +++ b/newton-4.00/sdk/dBrain/ndBrainLayerConvolutional_2d.h @@ -45,6 +45,7 @@ class ndBrainLayerConvolutional_2d : public ndBrainLayer virtual const char* GetLabelId() const; virtual void Blend(const ndBrainLayer& src, ndBrainFloat blend); + virtual void UpdateDropOut(); virtual void InitWeightsXavierMethod(); virtual void InitWeights(ndBrainFloat weighVariance, ndBrainFloat biasVariance); virtual void AdamUpdate(const ndBrainLayer& u, const ndBrainLayer& v, ndBrainFloat epsilon); diff --git a/newton-4.00/sdk/dBrain/ndBrainLayerDropOut.cpp b/newton-4.00/sdk/dBrain/ndBrainLayerDropOut.cpp deleted file mode 100644 index 1bdea8ef98..0000000000 --- a/newton-4.00/sdk/dBrain/ndBrainLayerDropOut.cpp +++ /dev/null @@ -1,105 +0,0 @@ -/* Copyright (c) <2003-2022> -* -* This software is provided 'as-is', without any express or implied -* warranty. In no event will the authors be held liable for any damages -* arising from the use of this software. -* -* Permission is granted to anyone to use this software for any purpose, -* including commercial applications, and to alter it and redistribute it -* freely, subject to the following restrictions: -* -* 1. The origin of this software must not be misrepresented; you must not -* claim that you wrote the original software. If you use this software -* in a product, an acknowledgment in the product documentation would be -* appreciated but is not required. -* -* 2. Altered source versions must be plainly marked as such, and must not be -* misrepresented as being the original software. -* -* 3. This notice may not be removed or altered from any source distribution. -*/ - -#include "ndBrainStdafx.h" -#include "ndBrainSaveLoad.h" -#include "ndBrainLayerDropOut.h" - -ndBrainLayerDropOut::ndBrainLayerDropOut(ndInt32 neurons, ndBrainFloat dropFraction) - :ndBrainLayerActivation(neurons) - ,m_droppedNeuron() - ,m_dropFraction(dropFraction) -{ - m_droppedNeuron.SetCount(neurons); - m_droppedNeuron.Set(ndBrainFloat(1.0f)); -} - -ndBrainLayerDropOut::ndBrainLayerDropOut(const ndBrainLayerDropOut& src) - :ndBrainLayerActivation(src) - ,m_droppedNeuron(src.m_droppedNeuron) - ,m_dropFraction(src.m_dropFraction) -{ -} - -ndBrainLayer* ndBrainLayerDropOut::Clone() const -{ - return new ndBrainLayerDropOut(*this); -} - -const char* ndBrainLayerDropOut::GetLabelId() const -{ - return "ndBrainLayerDropOut"; -} - -void ndBrainLayerDropOut::UpdateDropOut() -{ - ndAssert(0); -} - -void ndBrainLayerDropOut::InputDerivative(const ndBrainVector&, const ndBrainVector& outputDerivative, ndBrainVector& inputDerivative) const -{ - ndAssert(outputDerivative.GetCount() == outputDerivative.GetCount()); - inputDerivative.Set(m_droppedNeuron); - inputDerivative.Mul(outputDerivative); -} - -void ndBrainLayerDropOut::MakePrediction(const ndBrainVector& input, ndBrainVector& output) const -{ - ndAssert(input.GetCount() == output.GetCount()); - output.Set(input); - output.Mul(m_droppedNeuron); -} - -void ndBrainLayerDropOut::Save(const ndBrainSave* const loadSave) const -{ - ndAssert(0); - //char buffer[1024]; - // - //sprintf(buffer, "\tinput_width %d\n", m_width); - //loadSave->WriteData(buffer); - // - //sprintf(buffer, "\tinput_heigh %d\n", m_height); - //loadSave->WriteData(buffer); - // - //sprintf(buffer, "\tinput_layers %d\n", m_channels); - //loadSave->WriteData(buffer); -} - -ndBrainLayer* ndBrainLayerDropOut::Load(const ndBrainLoad* const loadSave) -{ - ndAssert(0); - //char buffer[1024]; - //loadSave->ReadString(buffer); - // - //loadSave->ReadString(buffer); - //ndInt32 inputWidth = loadSave->ReadInt(); - // - //loadSave->ReadString(buffer); - //ndInt32 inputHeight = loadSave->ReadInt(); - // - //loadSave->ReadString(buffer); - //ndInt32 inputLayers = loadSave->ReadInt(); - // - //ndBrainLayerDropOut* const layer = new ndBrainLayerDropOut(inputWidth, inputHeight, inputLayers); - //loadSave->ReadString(buffer); - //return layer; - return nullptr; -} \ No newline at end of file diff --git a/newton-4.00/sdk/dBrain/ndBrainLayerDropOut.h b/newton-4.00/sdk/dBrain/ndBrainLayerDropOut.h deleted file mode 100644 index fabb763cd0..0000000000 --- a/newton-4.00/sdk/dBrain/ndBrainLayerDropOut.h +++ /dev/null @@ -1,47 +0,0 @@ -/* Copyright (c) <2003-2022> -* -* This software is provided 'as-is', without any express or implied -* warranty. In no event will the authors be held liable for any damages -* arising from the use of this software. -* -* Permission is granted to anyone to use this software for any purpose, -* including commercial applications, and to alter it and redistribute it -* freely, subject to the following restrictions: -* -* 1. The origin of this software must not be misrepresented; you must not -* claim that you wrote the original software. If you use this software -* in a product, an acknowledgment in the product documentation would be -* appreciated but is not required. -* -* 2. Altered source versions must be plainly marked as such, and must not be -* misrepresented as being the original software. -* -* 3. This notice may not be removed or altered from any source distribution. -*/ - -#ifndef _ND_BRAIN_LAYER_DROPOUT_H__ -#define _ND_BRAIN_LAYER_DROPOUT_H__ - -#include "ndBrainStdafx.h" -#include "ndBrainLayerActivation.h" - -class ndBrainLayerDropOut : public ndBrainLayerActivation -{ - public: - ndBrainLayerDropOut(ndInt32 neurons, ndBrainFloat dropFraction = ndBrainFloat(0.0f)); - ndBrainLayerDropOut(const ndBrainLayerDropOut& src); - ndBrainLayer* Clone() const; - static ndBrainLayer* Load(const ndBrainLoad* const loadSave); - - void UpdateDropOut(); - const char* GetLabelId() const; - void Save(const ndBrainSave* const loadSave) const; - void MakePrediction(const ndBrainVector& input, ndBrainVector& output) const; - void InputDerivative(const ndBrainVector& output, const ndBrainVector& outputDerivative, ndBrainVector& inputDerivative) const; - - ndBrainVector m_droppedNeuron; - ndBrainFloat m_dropFraction; -}; - -#endif - diff --git a/newton-4.00/sdk/dBrain/ndBrainLayerLeakyReluActivation.cpp b/newton-4.00/sdk/dBrain/ndBrainLayerLeakyReluActivation.cpp index fef92d8e91..026cd89c3d 100644 --- a/newton-4.00/sdk/dBrain/ndBrainLayerLeakyReluActivation.cpp +++ b/newton-4.00/sdk/dBrain/ndBrainLayerLeakyReluActivation.cpp @@ -52,7 +52,6 @@ ndBrainLayer* ndBrainLayerLeakyReluActivation::Load(const ndBrainLoad* const loa char buffer[1024]; loadSave->ReadString(buffer); - ndAssert(0); loadSave->ReadString(buffer); ndInt32 inputs = loadSave->ReadInt(); diff --git a/newton-4.00/sdk/dBrain/ndBrainLayerLinear.cpp b/newton-4.00/sdk/dBrain/ndBrainLayerLinear.cpp index 4d0a173e8b..43aebdcbb4 100644 --- a/newton-4.00/sdk/dBrain/ndBrainLayerLinear.cpp +++ b/newton-4.00/sdk/dBrain/ndBrainLayerLinear.cpp @@ -28,14 +28,20 @@ ndBrainLayerLinear::ndBrainLayerLinear(ndInt32 inputs, ndInt32 outputs) :ndBrainLayer() ,m_bias() ,m_weights(outputs, inputs) + ,m_dropOut() + ,m_dropOutProbability(ndBrainFloat(1.0f)) { + m_dropOut.SetCount(outputs); m_bias.SetCount(outputs); + m_dropOut.Set(ndBrainFloat(1.0f)); } ndBrainLayerLinear::ndBrainLayerLinear(const ndBrainLayerLinear& src) :ndBrainLayer(src) ,m_bias(src.m_bias) ,m_weights(src.m_weights) + ,m_dropOut(src.m_dropOut) + ,m_dropOutProbability(src.m_dropOutProbability) { } @@ -253,6 +259,11 @@ ndBrainLayer* ndBrainLayerLinear::Load(const ndBrainLoad* const loadSave) return layer; } +void ndBrainLayerLinear::UpdateDropOut() +{ + ndAssert(0); +} + void ndBrainLayerLinear::MakePrediction(const ndBrainVector& input, ndBrainVector& output) const { m_weights.Mul(input, output); diff --git a/newton-4.00/sdk/dBrain/ndBrainLayerLinear.h b/newton-4.00/sdk/dBrain/ndBrainLayerLinear.h index bf97127d54..ee3bd11aae 100644 --- a/newton-4.00/sdk/dBrain/ndBrainLayerLinear.h +++ b/newton-4.00/sdk/dBrain/ndBrainLayerLinear.h @@ -44,6 +44,7 @@ class ndBrainLayerLinear : public ndBrainLayer virtual ndBrainVector* GetBias(); virtual ndBrainMatrix* GetWeights(); + virtual void UpdateDropOut(); virtual void InitWeightsXavierMethod(); virtual void InitWeights(ndBrainFloat weighVariance, ndBrainFloat biasVariance); @@ -72,6 +73,8 @@ class ndBrainLayerLinear : public ndBrainLayer ndBrainVector m_bias; ndBrainMatrix m_weights; + ndBrainVector m_dropOut; + ndBrainFloat m_dropOutProbability; }; diff --git a/newton-4.00/sdk/dBrain/ndBrainSaveLoad.cpp b/newton-4.00/sdk/dBrain/ndBrainSaveLoad.cpp index d15e687af9..59ef6ca1d7 100644 --- a/newton-4.00/sdk/dBrain/ndBrainSaveLoad.cpp +++ b/newton-4.00/sdk/dBrain/ndBrainSaveLoad.cpp @@ -23,7 +23,6 @@ #include "ndBrain.h" #include "ndBrainSaveLoad.h" #include "ndBrainLayerLinear.h" -#include "ndBrainLayerDropOut.h" #include "ndBrainLayerReluActivation.h" #include "ndBrainLayerTanhActivation.h" #include "ndBrainLayerConvolutional_2d.h" @@ -104,10 +103,6 @@ ndBrain* ndBrainLoad::Load() const { layer = ndBrainLayerLinear::Load(this); } - if (!strcmp(layerType, "ndBrainLayerDropOut")) - { - layer = ndBrainLayerDropOut::Load(this); - } else if (!strcmp(layerType, "ndBrainLayerReluActivation")) { layer = ndBrainLayerReluActivation::Load(this);